Convert-UUlib/UUlib.pm

package Convert::UUlib;

use common::sense;

use Carp;

require Exporter;
require DynaLoader;

our $VERSION = 1.71;

our @ISA = qw(Exporter DynaLoader);

our @_consts = qw(
        ACT_COPYING ACT_DECODING ACT_ENCODING ACT_IDLE ACT_SCANNING

        FILE_DECODED FILE_ERROR FILE_MISPART FILE_NOBEGIN FILE_NODATA
        FILE_NOEND FILE_OK FILE_READ FILE_TMPFILE

        MSG_ERROR MSG_FATAL MSG_MESSAGE MSG_NOTE MSG_PANIC MSG_WARNING

        OPT_RBUF OPT_WBUF
        OPT_BRACKPOL OPT_DEBUG OPT_DESPERATE OPT_DUMBNESS OPT_ENCEXT
        OPT_ERRNO OPT_FAST OPT_IGNMODE OPT_IGNREPLY OPT_OVERWRITE OPT_PREAMB
        OPT_PROGRESS OPT_SAVEPATH OPT_TINYB64 OPT_USETEXT OPT_VERBOSE
        OPT_VERSION OPT_REMOVE OPT_MOREMIME OPT_DOTDOT OPT_AUTOCHECK

        RET_CANCEL RET_CONT RET_EXISTS RET_ILLVAL RET_IOERR RET_NODATA
        RET_NOEND RET_NOMEM RET_OK RET_UNSUP

        B64_ENCODED BH_ENCODED PT_ENCODED QP_ENCODED
        XX_ENCODED UU_ENCODED YENC_ENCODED
);

our @_funcs = qw(
        Initialize CleanUp GetOption SetOption strerror SetMsgCallback
        SetBusyCallback SetFileCallback SetFNameFilter SetFileNameCallback
        FNameFilter LoadFile GetFileListItem GetFileList RenameFile DecodeToTemp
        RemoveTemp DecodeFile InfoFile Smerge QuickDecode EncodeMulti
        EncodePartial EncodeToStream EncodeToFile E_PrepSingle
        E_PrepPartial

        straction strencoding strmsglevel
);

our @EXPORT = @_consts;
our @EXPORT_OK = @_funcs;
our %EXPORT_TAGS = (all => [@_consts,@_funcs], constants => \@_consts);

bootstrap Convert::UUlib $VERSION;

# dummy function for compatiiblity with pre-1.7 versions
sub Initialize { }

# action code -> string mapping
sub straction($) {
   return 'copying'     if $_[0] == &ACT_COPYING;
   return 'decoding'    if $_[0] == &ACT_DECODING;
   return 'encoding'    if $_[0] == &ACT_ENCODING;
   return 'idle'        if $_[0] == &ACT_IDLE;
   return 'scanning'    if $_[0] == &ACT_SCANNING;
   'unknown';
}

# encoding type -> string mapping
sub strencoding($) {
   return 'uuencode'            if $_[0] == &UU_ENCODED;
   return 'base64'              if $_[0] == &B64_ENCODED;
   return 'yenc'                if $_[0] == &YENC_ENCODED;
   return 'binhex'              if $_[0] == &BH_ENCODED;
   return 'plaintext'           if $_[0] == &PT_ENCODED;
   return 'quoted-printable'    if $_[0] == &QP_ENCODED;
   return 'xxencode'            if $_[0] == &XX_ENCODED;
   'unknown';
}

sub strmsglevel($) {
   return 'message'     if $_[0] == &MSG_MESSAGE;
   return 'note'        if $_[0] == &MSG_NOTE;
   return 'warning'     if $_[0] == &MSG_WARNING;
   return 'error'       if $_[0] == &MSG_ERROR;
   return 'panic'       if $_[0] == &MSG_PANIC;
   return 'fatal'       if $_[0] == &MSG_FATAL;
   'unknown';
}

1;
__END__

=head1 NAME

Convert::UUlib - Perl interface to the uulib library (a.k.a. uudeview/uuenview).

=head1 SYNOPSIS

 use Convert::UUlib ':all';
 
 # read all the files named on the commandline and decode them
 # into the CURRENT directory. See below for a longer example.
 LoadFile $_ for @ARGV;

 for my $uu (GetFileList) {
    if ($uu->state & FILE_OK) {
      $uu->decode;
      print $uu->filename, "\n";
    }
 }

=head1 DESCRIPTION

Read the file doc/library.pdf from the distribution for in-depth
information about the C-library used in this interface, and the rest of
this document and especially the non-trivial decoder program at the end.

=head1 EXPORTED CONSTANTS

=head2 Action code constants

  ACT_IDLE      we don't do anything
  ACT_SCANNING  scanning an input file
  ACT_DECODING  decoding into a temp file
  ACT_COPYING   copying temp to target
  ACT_ENCODING  encoding a file

=head2 Message severity levels

  MSG_MESSAGE   just a message, nothing important
  MSG_NOTE      something that should be noticed
  MSG_WARNING   important msg, processing continues
  MSG_ERROR     processing has been terminated
  MSG_FATAL     decoder cannot process further requests
  MSG_PANIC     recovery impossible, app must terminate

=head2 Options

  OPT_VERSION   version number MAJOR.MINORplPATCH (ro)
  OPT_FAST      assumes only one part per file
  OPT_DUMBNESS  switch off the program's intelligence
  OPT_BRACKPOL  give numbers in [] higher precendence
  OPT_VERBOSE   generate informative messages
  OPT_DESPERATE try to decode incomplete files
  OPT_IGNREPLY  ignore RE:plies (off by default)
  OPT_OVERWRITE whether it's OK to overwrite ex. files
  OPT_SAVEPATH  prefix to save-files on disk
  OPT_IGNMODE   ignore the original file mode
  OPT_DEBUG     print messages with FILE/LINE info
  OPT_ERRNO     get last error code for RET_IOERR (ro)
  OPT_PROGRESS  retrieve progress information
  OPT_USETEXT   handle text messages
  OPT_PREAMB    handle Mime preambles/epilogues
  OPT_TINYB64   detect short B64 outside of Mime
  OPT_ENCEXT    extension for single-part encoded files
  OPT_REMOVE    remove input files after decoding (dangerous)
  OPT_MOREMIME  strict MIME adherence
  OPT_DOTDOT    ".."-unescaping has not yet been done on input files
  OPT_RBUF      set default read I/O buffer size in bytes
  OPT_WBUF      set default write I/O buffer size in bytes
  OPT_AUTOCHECK automatically check file list after every loadfile

=head2 Result/Error codes

  RET_OK        everything went fine
  RET_IOERR     I/O Error - examine errno
  RET_NOMEM     not enough memory
  RET_ILLVAL    illegal value for operation
  RET_NODATA    decoder didn't find any data
  RET_NOEND     encoded data wasn't ended properly
  RET_UNSUP     unsupported function (encoding)
  RET_EXISTS    file exists (decoding)
  RET_CONT      continue -- special from ScanPart
  RET_CANCEL    operation canceled

=head2 File States

 This code is zero, i.e. "false":

  UUFILE_READ   Read in, but not further processed

 The following state codes are or'ed together:

  FILE_MISPART  Missing Part(s) detected
  FILE_NOBEGIN  No 'begin' found
  FILE_NOEND    No 'end' found
  FILE_NODATA   File does not contain valid uudata
  FILE_OK       All Parts found, ready to decode
  FILE_ERROR    Error while decoding
  FILE_DECODED  Successfully decoded
  FILE_TMPFILE  Temporary decoded file exists

=head2 Encoding types

  UU_ENCODED    UUencoded data
  B64_ENCODED   Mime-Base64 data
  XX_ENCODED    XXencoded data
  BH_ENCODED    Binhex encoded
  PT_ENCODED    Plain-Text encoded (MIME)
  QP_ENCODED    Quoted-Printable (MIME)
  YENC_ENCODED  yEnc encoded (non-MIME)

=head1 EXPORTED FUNCTIONS

=head2 Initializing and cleanup

Initialize is automatically called when the module is loaded and allocates
quite a small amount of memory for todays machines ;) CleanUp releases that
again.

On my machine, a fairly complete decode with DBI backend needs about 10MB
RSS to decode 20000 files.

=over

=item CleanUp

Release memory, file items and clean up files. Should be called after a
decoidng run, if you want to start a new one.

=back

=head2 Setting and querying options

=over

=item $option = GetOption OPT_xxx

=item SetOption OPT_xxx, opt-value

=back

See the C<OPT_xxx> constants above to see which options exist.

=head2 Setting various callbacks

=over

=item SetMsgCallback [callback-function]

=item SetBusyCallback [callback-function]

=item SetFileCallback [callback-function]

=item SetFNameFilter [callback-function]

=back

=head2 Call the currently selected FNameFilter

=over

=item $file = FNameFilter $file

=back

=head2 Loading sourcefiles, optionally fuzzy merge and start decoding

=over

=item ($retval, $count) = LoadFile $fname, [$id, [$delflag, [$partno]]]

Load the given file and scan it for encoded contents. Optionally tag it
with the given id, and if C<$delflag> is true, delete the file after it
is no longer necessary. If you are certain of the part number, you can
specify it as the last argument.

A better (usually faster) way of doing this is using the C<SetFNameFilter>
functionality.

=item $retval = Smerge $pass

If you are desperate, try to call C<Smerge> with increasing C<$pass>
values, beginning at C<0>, to try to merge parts that usually would not
have been merged.

Most probably this will result in garbled files, so never do this by
default, except:

If the C<OPT_AUTOCHECK> option has been disabled (by default it is
enabled) to speed up file loading, then you I<have> to call C<Smerge -1>
after loading all files as an additional pre-pass (which is normally done
by C<LoadFile>).

=item $item = GetFileListItem $item_number

Return the C<$item> structure for the C<$item_number>'th found file, or
C<undef> of no file with that number exists.

The first file has number C<0>, and the series has no holes, so you can
iterate over all files by starting with zero and incrementing until you
hit C<undef>.

This function has to walk the linear list of fils on each access, so
if you want to iterate over all items, it is usually faster to use
C<GetFileList>.

=item @items = GetFileList

Similar to C<GetFileListItem>, but returns all files in one go, which is
very much faster for large number of items, and has no drawbacks when used
for a small number of items.

=back

=head2 Decoding files

=over

=item $retval = $item->rename ($newname)

Change the ondisk filename where the decoded file will be saved.

=item $retval = $item->decode_temp

Decode the file into a temporary location, use C<< $item->infile >> to
retrieve the temporary filename.

=item $retval = $item->remove_temp

Remove the temporarily decoded file again.

=item $retval = $item->decode ([$target_path])

Decode the file to its destination, or the given target path.

=item $retval = $item->info (callback-function)

=back

=head2 Querying (and setting) item attributes

=over

=item $state    = $item->state

=item $mode     = $item->mode ([newmode])

=item $uudet    = $item->uudet

=item $size     = $item->size

=item $filename = $item->filename ([newfilename})

=item $subfname = $item->subfname

=item $mimeid   = $item->mimeid

=item $mimetype = $item->mimetype

=item $binfile  = $item->binfile

=back

=head2 Information about source parts

=over

=item $parts = $item->parts

Return information about all parts (source files) used to decode the file
as a list of hashrefs with the following structure:

 {
   partno   => <integer describing the part number, starting with 1>,
   # the following member sonly exist when they contain useful information
   sfname   => <local pathname of the file where this part is from>,
   filename => <the ondisk filename of the decoded file>,
   subfname => <used to cluster postings, possibly the posting filename>,
   subject  => <the subject of the posting/mail>,
   origin   => <the possible source (From) address>,
   mimetype => <the possible mimetype of the decoded file>,
   mimeid   => <the id part of the Content-Type>,
 }

Usually you are interested mostly the C<sfname> and possibly the C<partno>
and C<filename> members.

=back

=head2 Functions below are not documented and not very well tested - feedback welcome

  QuickDecode
  EncodeMulti
  EncodePartial
  EncodeToStream
  EncodeToFile
  E_PrepSingle
  E_PrepPartial

=head2 EXTENSION FUNCTIONS

Functions found in this module but not documented in the uulib documentation:

=over

=item $msg = straction ACT_xxx

Return a human readable string representing the given action code.

=item $msg = strerror RET_xxx

Return a human readable string representing the given error code.

=item $str = strencoding xxx_ENCODED

Return the name of the encoding type as a string.

=item $str = strmsglevel MSG_xxx

Returns the message level as a string.

=item SetFileNameCallback $cb

Sets (or queries) the FileNameCallback, which is called whenever the
decoding library can't find a filename and wants to extract a filename
from the subject line of a posting. The callback will be called with
two arguments, the subject line and the current candidate for the
filename. The latter argument can be C<undef>, which means that no
filename could be found (and likely no one exists, so it is safe to also
return C<undef> in this case). If it doesn't return anything (not even
C<undef>!), then nothing happens, so this is a no-op callback:

   sub cb {
      return ();
   }

If it returns C<undef>, then this indicates that no filename could be
found. In all other cases, the return value is taken to be the filename.

This is a slightly more useful callback:

  sub cb {
     return unless $_[1]; # skip "Re:"-plies et al.
     my ($subject, $filename) = @_;
     # if we find some *.rar, take it
     return $1 if $subject =~ /(\w+\.rar)/;
     # otherwise just pass what we have
     return ();
  }

=back

=head1 LARGE EXAMPLE DECODER

The general workflow for decoding is like this:

=over

=item 1. Configure options with C<SetOption> or C<SetXXXCallback>.

=item 2. Load all source files with C<LoadFile>.

=item 3. Optionally C<Smerge>.

=item 4. Iterate over all C<GetFileList> items (i.e. result files).

=item 5. C<CleanUp> to delete files and free items.

=back

What follows is the file C<example-decoder> from the distribution that
illustrates the above worklfow in a non-trivial example.

   #!/usr/bin/perl

   # decode all the files in the directory uusrc/ and copy
   # the resulting files to uudst/

   use Convert::UUlib ':all';

   sub namefilter {
      my ($path) = @_;

      $path=~s/^.*[\/\\]//;

      $path
   }

   sub busycb {
      my ($action, $curfile, $partno, $numparts, $percent, $fsize) = @_;
      $_[0]=straction($action);
      print "busy_callback(", (join ",",@_), ")\n";
      0
   }

   SetOption OPT_RBUF, 128*1024;
   SetOption OPT_WBUF, 1024*1024;
   SetOption OPT_IGNMODE, 1;
   SetOption OPT_IGNMODE, 1;
   SetOption OPT_VERBOSE, 1;
   SetOption OPT_AUTOCHK, 0;

   # show the three ways you can set callback functions. I normally
   # prefer the one with the sub inplace.
   SetFNameFilter \&namefilter;

   SetBusyCallback "busycb", 333;

   SetMsgCallback sub {
      my ($msg, $level) = @_;
      print uc strmsglevel $_[1], ": $msg\n";
   };

   # the following non-trivial FileNameCallback takes care
   # of some subject lines not detected properly by uulib:
   SetFileNameCallback sub {
      return unless $_[1]; # skip "Re:"-plies et al.
      local $_ = $_[0];

      # the following rules are rather effective on some newsgroups,
      # like alt.binaries.games.anime, where non-mime, uuencoded data
      # is very common

      # if we find some *.rar, take it as the filename
      return $1 if /(\S{3,}\.(?:[rstuvwxyz]\d\d|rar))\s/i;

      # one common subject format
      return $1 if /- "(.{2,}?\..+?)" (?:yenc )?\(\d+\/\d+\)/i;

      # - filename.par (04/55)
      return $1 if /- "?(\S{3,}\.\S+?)"? (?:yenc )?\(\d+\/\d+\)/i;

      # - (xxx) No. 1 sayuri81.jpg 756565 bytes
      # - (20 files) No.17 Roseanne.jpg [2/2]
      return $1 if /No\.[ 0-9]+ (\S+\....) (?:\d+ bytes )?\[/;

      # try to detect some common forms of filenames
      return $1 if /([a-z0-9_\-+.]{3,}\.[a-z]{3,4}(?:.\d+))/i;

      # otherwise just pass what we have
      ()
   };

   # now read all files in the directory uusrc/*
   for (<uusrc/*>) {
      my ($retval, $count) = LoadFile ($_, $_, 1);
      print "file($_), status(", strerror $retval, ") parts($count)\n";
   }

   Smerge -1;

   SetOption OPT_SAVEPATH, "uudst/";

   # now wade through all files and their source parts
   for my $uu (GetFileList) {
      print "file ", $uu->filename, "\n";
      print " state ", $uu->state, "\n";
      print " mode ", $uu->mode, "\n";
      print " uudet ", strencoding $uu->uudet, "\n";
      print " size ", $uu->size, "\n";
      print " subfname ", $uu->subfname, "\n";
      print " mimeid ", $uu->mimeid, "\n";
      print " mimetype ", $uu->mimetype, "\n";

      # print additional info about all parts
      print " parts";
      for ($uu->parts) {
         for my $k (sort keys %$_) {
            print " $k=$_->{$k}";
         }
         print "\n";
      }

      $uu->remove_temp;

      if (my $err = $uu->decode) {
         print " ERROR ", strerror $err, "\n";
      } else {
         print " successfully saved as uudst/", $uu->filename, "\n";
      }
   }

   print "cleanup...\n";

   CleanUp;

=head1 PERLMULTICORE SUPPORT

This module supports the perlmulticore standard (see
L<http://perlmulticore.schmorp.de/> for more info) for the following
functions - generally these are functions accessing the disk and/or using
considerable CPU time:

   LoadFile
   $item->decode
   $item->decode_temp
   $item->remove_temp
   $item->info

The perl interpreter will be reacquired/released on every callback
invocation, so for performance reasons, callbacks should be avoided if
that is costly.

Future versions might enable multicore support for more functions.

=head1 BUGS AND LIMITATIONS

The original uulib library this module uses was written at a time where
main memory of measured in megabytes and buffer overflows as a security
thign didn't exist. While a lot of security fixes have been applied over
the years (includign some defense in depth mechanism that can shield
against a lot of as-of-yet undetected bugs), using this library for
security purposes requires care.

Likewise, file sizes when the uulib library was written were tiny compared
to today, so do not expect this library to handle files larger than 2GB.

Lastly, this module uses a very "C-like" interface, which means it doesn't
protect you from invalid points as you might expect from "more perlish"
modules - for example, accessing a file item object after callinbg
C<CleanUp> will likely result in crashes, memory corruption, or worse.

=head1 AUTHOR

Marc Lehmann <schmorp@schmorp.de>, the original uulib library was written
by Frank Pilhofer <fp@informatik.uni-frankfurt.de>, and later heavily
bugfixed by Marc Lehmann.

=head1 SEE ALSO

perl(1), uudeview homepage at L<http://www.fpx.de/fp/Software/UUDeview/>.

=cut

Revision:	1.54
Committed:	Sat Dec 12 10:48:39 2020 UTC (3 years, 5 months ago) by root
Branch:	MAIN
Changes since 1.53:	+3 -1 lines
Log Message:	* empty log message *
#	Content
1	package Convert::UUlib;
2
3	use common::sense;
4
5	use Carp;
6
7	require Exporter;
8	require DynaLoader;
9
10	our $VERSION = 1.71;
11
12	our @ISA = qw(Exporter DynaLoader);
13
14	our @_consts = qw(
15	ACT_COPYING ACT_DECODING ACT_ENCODING ACT_IDLE ACT_SCANNING
16
17	FILE_DECODED FILE_ERROR FILE_MISPART FILE_NOBEGIN FILE_NODATA
18	FILE_NOEND FILE_OK FILE_READ FILE_TMPFILE
19
20	MSG_ERROR MSG_FATAL MSG_MESSAGE MSG_NOTE MSG_PANIC MSG_WARNING
21
22	OPT_RBUF OPT_WBUF
23	OPT_BRACKPOL OPT_DEBUG OPT_DESPERATE OPT_DUMBNESS OPT_ENCEXT
24	OPT_ERRNO OPT_FAST OPT_IGNMODE OPT_IGNREPLY OPT_OVERWRITE OPT_PREAMB
25	OPT_PROGRESS OPT_SAVEPATH OPT_TINYB64 OPT_USETEXT OPT_VERBOSE
26	OPT_VERSION OPT_REMOVE OPT_MOREMIME OPT_DOTDOT OPT_AUTOCHECK
27
28	RET_CANCEL RET_CONT RET_EXISTS RET_ILLVAL RET_IOERR RET_NODATA
29	RET_NOEND RET_NOMEM RET_OK RET_UNSUP
30
31	B64_ENCODED BH_ENCODED PT_ENCODED QP_ENCODED
32	XX_ENCODED UU_ENCODED YENC_ENCODED
33	);
34
35	our @_funcs = qw(
36	Initialize CleanUp GetOption SetOption strerror SetMsgCallback
37	SetBusyCallback SetFileCallback SetFNameFilter SetFileNameCallback
38	FNameFilter LoadFile GetFileListItem GetFileList RenameFile DecodeToTemp
39	RemoveTemp DecodeFile InfoFile Smerge QuickDecode EncodeMulti
40	EncodePartial EncodeToStream EncodeToFile E_PrepSingle
41	E_PrepPartial
42
43	straction strencoding strmsglevel
44	);
45
46	our @EXPORT = @_consts;
47	our @EXPORT_OK = @_funcs;
48	our %EXPORT_TAGS = (all => [@_consts,@_funcs], constants => \@_consts);
49
50	bootstrap Convert::UUlib $VERSION;
51
52	# dummy function for compatiiblity with pre-1.7 versions
53	sub Initialize { }
54
55	# action code -> string mapping
56	sub straction($) {
57	return 'copying' if $_[0] == &ACT_COPYING;
58	return 'decoding' if $_[0] == &ACT_DECODING;
59	return 'encoding' if $_[0] == &ACT_ENCODING;
60	return 'idle' if $_[0] == &ACT_IDLE;
61	return 'scanning' if $_[0] == &ACT_SCANNING;
62	'unknown';
63	}
64
65	# encoding type -> string mapping
66	sub strencoding($) {
67	return 'uuencode' if $_[0] == &UU_ENCODED;
68	return 'base64' if $_[0] == &B64_ENCODED;
69	return 'yenc' if $_[0] == &YENC_ENCODED;
70	return 'binhex' if $_[0] == &BH_ENCODED;
71	return 'plaintext' if $_[0] == &PT_ENCODED;
72	return 'quoted-printable' if $_[0] == &QP_ENCODED;
73	return 'xxencode' if $_[0] == &XX_ENCODED;
74	'unknown';
75	}
76
77	sub strmsglevel($) {
78	return 'message' if $_[0] == &MSG_MESSAGE;
79	return 'note' if $_[0] == &MSG_NOTE;
80	return 'warning' if $_[0] == &MSG_WARNING;
81	return 'error' if $_[0] == &MSG_ERROR;
82	return 'panic' if $_[0] == &MSG_PANIC;
83	return 'fatal' if $_[0] == &MSG_FATAL;
84	'unknown';
85	}
86
87	1;
88	__END__
89
90	=head1 NAME
91
92	Convert::UUlib - Perl interface to the uulib library (a.k.a. uudeview/uuenview).
93
94	=head1 SYNOPSIS
95
96	use Convert::UUlib ':all';
97
98	# read all the files named on the commandline and decode them
99	# into the CURRENT directory. See below for a longer example.
100	LoadFile $_ for @ARGV;
101
102	for my $uu (GetFileList) {
103	if ($uu->state & FILE_OK) {
104	$uu->decode;
105	print $uu->filename, "\n";
106	}
107	}
108
109	=head1 DESCRIPTION
110
111	Read the file doc/library.pdf from the distribution for in-depth
112	information about the C-library used in this interface, and the rest of
113	this document and especially the non-trivial decoder program at the end.
114
115	=head1 EXPORTED CONSTANTS
116
117	=head2 Action code constants
118
119	ACT_IDLE we don't do anything
120	ACT_SCANNING scanning an input file
121	ACT_DECODING decoding into a temp file
122	ACT_COPYING copying temp to target
123	ACT_ENCODING encoding a file
124
125	=head2 Message severity levels
126
127	MSG_MESSAGE just a message, nothing important
128	MSG_NOTE something that should be noticed
129	MSG_WARNING important msg, processing continues
130	MSG_ERROR processing has been terminated
131	MSG_FATAL decoder cannot process further requests
132	MSG_PANIC recovery impossible, app must terminate
133
134	=head2 Options
135
136	OPT_VERSION version number MAJOR.MINORplPATCH (ro)
137	OPT_FAST assumes only one part per file
138	OPT_DUMBNESS switch off the program's intelligence
139	OPT_BRACKPOL give numbers in [] higher precendence
140	OPT_VERBOSE generate informative messages
141	OPT_DESPERATE try to decode incomplete files
142	OPT_IGNREPLY ignore RE:plies (off by default)
143	OPT_OVERWRITE whether it's OK to overwrite ex. files
144	OPT_SAVEPATH prefix to save-files on disk
145	OPT_IGNMODE ignore the original file mode
146	OPT_DEBUG print messages with FILE/LINE info
147	OPT_ERRNO get last error code for RET_IOERR (ro)
148	OPT_PROGRESS retrieve progress information
149	OPT_USETEXT handle text messages
150	OPT_PREAMB handle Mime preambles/epilogues
151	OPT_TINYB64 detect short B64 outside of Mime
152	OPT_ENCEXT extension for single-part encoded files
153	OPT_REMOVE remove input files after decoding (dangerous)
154	OPT_MOREMIME strict MIME adherence
155	OPT_DOTDOT ".."-unescaping has not yet been done on input files
156	OPT_RBUF set default read I/O buffer size in bytes
157	OPT_WBUF set default write I/O buffer size in bytes
158	OPT_AUTOCHECK automatically check file list after every loadfile
159
160	=head2 Result/Error codes
161
162	RET_OK everything went fine
163	RET_IOERR I/O Error - examine errno
164	RET_NOMEM not enough memory
165	RET_ILLVAL illegal value for operation
166	RET_NODATA decoder didn't find any data
167	RET_NOEND encoded data wasn't ended properly
168	RET_UNSUP unsupported function (encoding)
169	RET_EXISTS file exists (decoding)
170	RET_CONT continue -- special from ScanPart
171	RET_CANCEL operation canceled
172
173	=head2 File States
174
175	This code is zero, i.e. "false":
176
177	UUFILE_READ Read in, but not further processed
178
179	The following state codes are or'ed together:
180
181	FILE_MISPART Missing Part(s) detected
182	FILE_NOBEGIN No 'begin' found
183	FILE_NOEND No 'end' found
184	FILE_NODATA File does not contain valid uudata
185	FILE_OK All Parts found, ready to decode
186	FILE_ERROR Error while decoding
187	FILE_DECODED Successfully decoded
188	FILE_TMPFILE Temporary decoded file exists
189
190	=head2 Encoding types
191
192	UU_ENCODED UUencoded data
193	B64_ENCODED Mime-Base64 data
194	XX_ENCODED XXencoded data
195	BH_ENCODED Binhex encoded
196	PT_ENCODED Plain-Text encoded (MIME)
197	QP_ENCODED Quoted-Printable (MIME)
198	YENC_ENCODED yEnc encoded (non-MIME)
199
200	=head1 EXPORTED FUNCTIONS
201
202	=head2 Initializing and cleanup
203
204	Initialize is automatically called when the module is loaded and allocates
205	quite a small amount of memory for todays machines ;) CleanUp releases that
206	again.
207
208	On my machine, a fairly complete decode with DBI backend needs about 10MB
209	RSS to decode 20000 files.
210
211	=over
212
213	=item CleanUp
214
215	Release memory, file items and clean up files. Should be called after a
216	decoidng run, if you want to start a new one.
217
218	=back
219
220	=head2 Setting and querying options
221
222	=over
223
224	=item $option = GetOption OPT_xxx
225
226	=item SetOption OPT_xxx, opt-value
227
228	=back
229
230	See the C<OPT_xxx> constants above to see which options exist.
231
232	=head2 Setting various callbacks
233
234	=over
235
236	=item SetMsgCallback [callback-function]
237
238	=item SetBusyCallback [callback-function]
239
240	=item SetFileCallback [callback-function]
241
242	=item SetFNameFilter [callback-function]
243
244	=back
245
246	=head2 Call the currently selected FNameFilter
247
248	=over
249
250	=item $file = FNameFilter $file
251
252	=back
253
254	=head2 Loading sourcefiles, optionally fuzzy merge and start decoding
255
256	=over
257
258	=item ($retval, $count) = LoadFile $fname, [$id, [$delflag, [$partno]]]
259
260	Load the given file and scan it for encoded contents. Optionally tag it
261	with the given id, and if C<$delflag> is true, delete the file after it
262	is no longer necessary. If you are certain of the part number, you can
263	specify it as the last argument.
264
265	A better (usually faster) way of doing this is using the C<SetFNameFilter>
266	functionality.
267
268	=item $retval = Smerge $pass
269
270	If you are desperate, try to call C<Smerge> with increasing C<$pass>
271	values, beginning at C<0>, to try to merge parts that usually would not
272	have been merged.
273
274	Most probably this will result in garbled files, so never do this by
275	default, except:
276
277	If the C<OPT_AUTOCHECK> option has been disabled (by default it is
278	enabled) to speed up file loading, then you I<have> to call C<Smerge -1>
279	after loading all files as an additional pre-pass (which is normally done
280	by C<LoadFile>).
281
282	=item $item = GetFileListItem $item_number
283
284	Return the C<$item> structure for the C<$item_number>'th found file, or
285	C<undef> of no file with that number exists.
286
287	The first file has number C<0>, and the series has no holes, so you can
288	iterate over all files by starting with zero and incrementing until you
289	hit C<undef>.
290
291	This function has to walk the linear list of fils on each access, so
292	if you want to iterate over all items, it is usually faster to use
293	C<GetFileList>.
294
295	=item @items = GetFileList
296
297	Similar to C<GetFileListItem>, but returns all files in one go, which is
298	very much faster for large number of items, and has no drawbacks when used
299	for a small number of items.
300
301	=back
302
303	=head2 Decoding files
304
305	=over
306
307	=item $retval = $item->rename ($newname)
308
309	Change the ondisk filename where the decoded file will be saved.
310
311	=item $retval = $item->decode_temp
312
313	Decode the file into a temporary location, use C<< $item->infile >> to
314	retrieve the temporary filename.
315
316	=item $retval = $item->remove_temp
317
318	Remove the temporarily decoded file again.
319
320	=item $retval = $item->decode ([$target_path])
321
322	Decode the file to its destination, or the given target path.
323
324	=item $retval = $item->info (callback-function)
325
326	=back
327
328	=head2 Querying (and setting) item attributes
329
330	=over
331
332	=item $state = $item->state
333
334	=item $mode = $item->mode ([newmode])
335
336	=item $uudet = $item->uudet
337
338	=item $size = $item->size
339
340	=item $filename = $item->filename ([newfilename})
341
342	=item $subfname = $item->subfname
343
344	=item $mimeid = $item->mimeid
345
346	=item $mimetype = $item->mimetype
347
348	=item $binfile = $item->binfile
349
350	=back
351
352	=head2 Information about source parts
353
354	=over
355
356	=item $parts = $item->parts
357
358	Return information about all parts (source files) used to decode the file
359	as a list of hashrefs with the following structure:
360
361	{
362	partno => <integer describing the part number, starting with 1>,
363	# the following member sonly exist when they contain useful information
364	sfname => <local pathname of the file where this part is from>,
365	filename => <the ondisk filename of the decoded file>,
366	subfname => <used to cluster postings, possibly the posting filename>,
367	subject => <the subject of the posting/mail>,
368	origin => <the possible source (From) address>,
369	mimetype => <the possible mimetype of the decoded file>,
370	mimeid => <the id part of the Content-Type>,
371	}
372
373	Usually you are interested mostly the C<sfname> and possibly the C<partno>
374	and C<filename> members.
375
376	=back
377
378	=head2 Functions below are not documented and not very well tested - feedback welcome
379
380	QuickDecode
381	EncodeMulti
382	EncodePartial
383	EncodeToStream
384	EncodeToFile
385	E_PrepSingle
386	E_PrepPartial
387
388	=head2 EXTENSION FUNCTIONS
389
390	Functions found in this module but not documented in the uulib documentation:
391
392	=over
393
394	=item $msg = straction ACT_xxx
395
396	Return a human readable string representing the given action code.
397
398	=item $msg = strerror RET_xxx
399
400	Return a human readable string representing the given error code.
401
402	=item $str = strencoding xxx_ENCODED
403
404	Return the name of the encoding type as a string.
405
406	=item $str = strmsglevel MSG_xxx
407
408	Returns the message level as a string.
409
410	=item SetFileNameCallback $cb
411
412	Sets (or queries) the FileNameCallback, which is called whenever the
413	decoding library can't find a filename and wants to extract a filename
414	from the subject line of a posting. The callback will be called with
415	two arguments, the subject line and the current candidate for the
416	filename. The latter argument can be C<undef>, which means that no
417	filename could be found (and likely no one exists, so it is safe to also
418	return C<undef> in this case). If it doesn't return anything (not even
419	C<undef>!), then nothing happens, so this is a no-op callback:
420
421	sub cb {
422	return ();
423	}
424
425	If it returns C<undef>, then this indicates that no filename could be
426	found. In all other cases, the return value is taken to be the filename.
427
428	This is a slightly more useful callback:
429
430	sub cb {
431	return unless $_[1]; # skip "Re:"-plies et al.
432	my ($subject, $filename) = @_;
433	# if we find some *.rar, take it
434	return $1 if $subject =~ /(\w+\.rar)/;
435	# otherwise just pass what we have
436	return ();
437	}
438
439	=back
440
441	=head1 LARGE EXAMPLE DECODER
442
443	The general workflow for decoding is like this:
444
445	=over
446
447	=item 1. Configure options with C<SetOption> or C<SetXXXCallback>.
448
449	=item 2. Load all source files with C<LoadFile>.
450
451	=item 3. Optionally C<Smerge>.
452
453	=item 4. Iterate over all C<GetFileList> items (i.e. result files).
454
455	=item 5. C<CleanUp> to delete files and free items.
456
457	=back
458
459	What follows is the file C<example-decoder> from the distribution that
460	illustrates the above worklfow in a non-trivial example.
461
462	#!/usr/bin/perl
463
464	# decode all the files in the directory uusrc/ and copy
465	# the resulting files to uudst/
466
467	use Convert::UUlib ':all';
468
469	sub namefilter {
470	my ($path) = @_;
471
472	$path=~s/^.*[\/\\]//;
473
474	$path
475	}
476
477	sub busycb {
478	my ($action, $curfile, $partno, $numparts, $percent, $fsize) = @_;
479	$_[0]=straction($action);
480	print "busy_callback(", (join ",",@_), ")\n";
481	0
482	}
483
484	SetOption OPT_RBUF, 128*1024;
485	SetOption OPT_WBUF, 1024*1024;
486	SetOption OPT_IGNMODE, 1;
487	SetOption OPT_IGNMODE, 1;
488	SetOption OPT_VERBOSE, 1;
489	SetOption OPT_AUTOCHK, 0;
490
491	# show the three ways you can set callback functions. I normally
492	# prefer the one with the sub inplace.
493	SetFNameFilter \&namefilter;
494
495	SetBusyCallback "busycb", 333;
496
497	SetMsgCallback sub {
498	my ($msg, $level) = @_;
499	print uc strmsglevel $_[1], ": $msg\n";
500	};
501
502	# the following non-trivial FileNameCallback takes care
503	# of some subject lines not detected properly by uulib:
504	SetFileNameCallback sub {
505	return unless $_[1]; # skip "Re:"-plies et al.
506	local $_ = $_[0];
507
508	# the following rules are rather effective on some newsgroups,
509	# like alt.binaries.games.anime, where non-mime, uuencoded data
510	# is very common
511
512	# if we find some *.rar, take it as the filename
513	return $1 if /(\S{3,}\.(?:[rstuvwxyz]\d\d\|rar))\s/i;
514
515	# one common subject format
516	return $1 if /- "(.{2,}?\..+?)" (?:yenc )?\(\d+\/\d+\)/i;
517
518	# - filename.par (04/55)
519	return $1 if /- "?(\S{3,}\.\S+?)"? (?:yenc )?\(\d+\/\d+\)/i;
520
521	# - (xxx) No. 1 sayuri81.jpg 756565 bytes
522	# - (20 files) No.17 Roseanne.jpg [2/2]
523	return $1 if /No\.[ 0-9]+ (\S+\....) (?:\d+ bytes )?\[/;
524
525	# try to detect some common forms of filenames
526	return $1 if /([a-z0-9_\-+.]{3,}\.[a-z]{3,4}(?:.\d+))/i;
527
528	# otherwise just pass what we have
529	()
530	};
531
532	# now read all files in the directory uusrc/*
533	for (<uusrc/*>) {
534	my ($retval, $count) = LoadFile ($_, $_, 1);
535	print "file($_), status(", strerror $retval, ") parts($count)\n";
536	}
537
538	Smerge -1;
539
540	SetOption OPT_SAVEPATH, "uudst/";
541
542	# now wade through all files and their source parts
543	for my $uu (GetFileList) {
544	print "file ", $uu->filename, "\n";
545	print " state ", $uu->state, "\n";
546	print " mode ", $uu->mode, "\n";
547	print " uudet ", strencoding $uu->uudet, "\n";
548	print " size ", $uu->size, "\n";
549	print " subfname ", $uu->subfname, "\n";
550	print " mimeid ", $uu->mimeid, "\n";
551	print " mimetype ", $uu->mimetype, "\n";
552
553	# print additional info about all parts
554	print " parts";
555	for ($uu->parts) {
556	for my $k (sort keys %$_) {
557	print " $k=$_->{$k}";
558	}
559	print "\n";
560	}
561
562	$uu->remove_temp;
563
564	if (my $err = $uu->decode) {
565	print " ERROR ", strerror $err, "\n";
566	} else {
567	print " successfully saved as uudst/", $uu->filename, "\n";
568	}
569	}
570
571	print "cleanup...\n";
572
573	CleanUp;
574
575	=head1 PERLMULTICORE SUPPORT
576
577	This module supports the perlmulticore standard (see
578	L<http://perlmulticore.schmorp.de/> for more info) for the following
579	functions - generally these are functions accessing the disk and/or using
580	considerable CPU time:
581
582	LoadFile
583	$item->decode
584	$item->decode_temp
585	$item->remove_temp
586	$item->info
587
588	The perl interpreter will be reacquired/released on every callback
589	invocation, so for performance reasons, callbacks should be avoided if
590	that is costly.
591
592	Future versions might enable multicore support for more functions.
593
594	=head1 BUGS AND LIMITATIONS
595
596	The original uulib library this module uses was written at a time where
597	main memory of measured in megabytes and buffer overflows as a security
598	thign didn't exist. While a lot of security fixes have been applied over
599	the years (includign some defense in depth mechanism that can shield
600	against a lot of as-of-yet undetected bugs), using this library for
601	security purposes requires care.
602
603	Likewise, file sizes when the uulib library was written were tiny compared
604	to today, so do not expect this library to handle files larger than 2GB.
605
606	Lastly, this module uses a very "C-like" interface, which means it doesn't
607	protect you from invalid points as you might expect from "more perlish"
608	modules - for example, accessing a file item object after callinbg
609	C<CleanUp> will likely result in crashes, memory corruption, or worse.
610
611	=head1 AUTHOR
612
613	Marc Lehmann <schmorp@schmorp.de>, the original uulib library was written
614	by Frank Pilhofer <fp@informatik.uni-frankfurt.de>, and later heavily
615	bugfixed by Marc Lehmann.
616
617	=head1 SEE ALSO
618
619	perl(1), uudeview homepage at L<http://www.fpx.de/fp/Software/UUDeview/>.
620
621	=cut
622