ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/cvsroot/App-Staticperl/mkbundle
Revision: 1.1
Committed: Mon Dec 6 19:33:57 2010 UTC (13 years, 6 months ago) by root
Branch: MAIN
Log Message:
*** empty log message ***

File Contents

# User Rev Content
1 root 1.1 #!/opt/bin/perl
2    
3     #############################################################################
4     # cannot load modules till after the tracer BEGIN block
5    
6     our $VERBOSE = 1;
7     our $STRIP = "pod"; # none, pod or ppi
8     our $PERL = 0;
9     our $VERIFY = 0;
10     our $STATIC = 0;
11    
12     my $PREFIX = "bundle";
13     my $PACKAGE = "static";
14    
15     my %pm;
16     my @libs;
17     my @static_ext;
18     my $extralibs;
19    
20     @ARGV
21     or die "$0: use 'staticperl help' (or read the sources of staticperl)\n";
22    
23     $|=1;
24    
25     our ($TRACER_W, $TRACER_R);
26    
27     sub find_inc($) {
28     for (@INC) {
29     next if ref;
30     return $_ if -e "$_/$_[0]";
31     }
32    
33     undef
34     }
35    
36     BEGIN {
37     # create a loader process to detect @INC requests before we load any modules
38     my ($W_TRACER, $R_TRACER); # used by tracer
39    
40     pipe $R_TRACER, $TRACER_W or die "pipe: $!";
41     pipe $TRACER_R, $W_TRACER or die "pipe: $!";
42    
43     unless (fork) {
44     close $TRACER_R;
45     close $TRACER_W;
46    
47     unshift @INC, sub {
48     my $dir = find_inc $_[1]
49     or return;
50    
51     syswrite $W_TRACER, "-\n$dir\n$_[1]\n";
52    
53     open my $fh, "<:perlio", "$dir/$_[1]"
54     or warn "ERROR: $dir/$_[1]: $!\n";
55    
56     $fh
57     };
58    
59     while (<$R_TRACER>) {
60     if (/use (.*)$/) {
61     my $mod = $1;
62     eval "require $mod";
63     warn "ERROR: $@ (while loading '$mod')\n"
64     if $@;
65     syswrite $W_TRACER, "\n";
66     } elsif (/eval (.*)$/) {
67     my $eval = $1;
68     eval $eval;
69     warn "ERROR: $@ (in '$eval')\n"
70     if $@;
71     }
72     }
73    
74     exit 0;
75     }
76     }
77    
78     # module loading is now safe
79     use Config;
80    
81     sub trace_module {
82     syswrite $TRACER_W, "use $_[0]\n";
83    
84     for (;;) {
85     <$TRACER_R> =~ /^-$/ or last;
86     my $dir = <$TRACER_R>; chomp $dir;
87     my $name = <$TRACER_R>; chomp $name;
88    
89     $pm{$name} = "$dir/$name";
90    
91     if ($name =~ /^(.*)\.pm$/) {
92     my $auto = "auto/$1";
93     my $autodir = "$dir/$auto";
94    
95     if (-d $autodir) {
96     opendir my $dir, $autodir
97     or die "$autodir: $!\n";
98    
99     for (readdir $dir) {
100     # AutoLoader
101     $pm{"$auto/$_"} = "$autodir/$_"
102     if /\.(?:al|ix)$/;
103    
104     # static ext
105     if (/\Q$Config{_a}\E$/o) {
106     push @libs, "$autodir/$_";
107     push @static_ext, $name;
108     }
109    
110     # extralibs.ld
111     if ($_ eq "extralibs.ld") {
112     open my $fh, "<:perlio", "$autodir/$_"
113     or die "$autodir/$_";
114    
115     local $/;
116     $extralibs .= " " . <$fh>;
117     }
118    
119     # dynamic object
120     warn "WARNING: found shared object - can't link statically ($_)\n"
121     if /\.\Q$Config{dlext}\E$/o;
122    
123     #TODO: extralibs?
124     }
125     }
126     }
127     }
128     }
129    
130     sub trace_eval {
131     syswrite $TRACER_W, "eval $_[0]\n";
132     }
133    
134     sub trace_finish {
135     close $TRACER_W;
136     close $TRACER_R;
137     }
138    
139     #############################################################################
140     # now we can use modules
141    
142     use common::sense;
143     use Digest::MD5;
144    
145     sub dump_string {
146     my ($fh, $data) = @_;
147    
148     if (length $data) {
149     for (
150     my $ofs = 0;
151     length (my $substr = substr $data, $ofs, 80);
152     $ofs += 80
153     ) {
154     $substr =~ s/([^\x20-\x21\x23-\x5b\x5d-\x7e])/sprintf "\\%03o", ord $1/ge;
155     $substr =~ s/\?/\\?/g; # trigraphs...
156     print $fh " \"$substr\"\n";
157     }
158     } else {
159     print $fh " \"\"\n";
160     }
161     }
162    
163     # required for @INC loading, unfortunately
164     trace_module "PerlIO::scalar";
165    
166     #trace_module "Term::ReadLine::readline"; # Term::ReadLine::Perl dependency
167     # URI is difficult
168     #trace_module "URI::http";
169     #trace_module "URI::_generic";
170    
171     sub cmd_boot {
172     $pm{"//boot"} = $_[0];
173     }
174    
175     sub cmd_add {
176     $_[0] =~ /^(.*)(?:\s*(\S+))$/
177     or die "$_[0]: cannot parse";
178    
179     my $file = $1;
180     my $as = defined $2 ? $2 : "/$1";
181    
182     $pm{$as} = $file;
183     }
184    
185     sub cmd_file {
186     open my $fh, "<", $_[0]
187     or die "$_[0]: $!\n";
188    
189     while (<$fh>) {
190     chomp;
191     my ($cmd, $args) = split / /, $_, 2;
192    
193     if ($cmd eq "strip") {
194     $STRIP = $args;
195     } elsif ($cmd eq "eval") {
196     trace_eval $_;
197     } elsif ($cmd eq "use") {
198     trace_module $_
199     for split / /, $args;
200     } elsif ($cmd eq "boot") {
201     cmd_boot $args;
202     } elsif ($cmd eq "static") {
203     $STATIC = 1;
204     } elsif ($cmd eq "add") {
205     cmd_add $args;
206     } elsif (/^\s*#/) {
207     # comment
208     } elsif (/\S/) {
209     die "$_: unsupported directive\n";
210     }
211     }
212     }
213    
214     use Getopt::Long;
215    
216     Getopt::Long::Configure ("bundling", "no_auto_abbrev", "no_ignore_case");
217    
218     GetOptions
219     "strip=s" => \$STRIP,
220     "verbose|v" => sub { ++$VERBOSE },
221     "quiet|q" => sub { --$VERBOSE },
222     "perl" => \$PERL,
223     "eval=s" => sub { trace_eval $_[1] },
224     "use|M=s" => sub { trace_module $_[1] },
225     "boot=s" => sub { cmd_boot $_[1] },
226     "add=s" => sub { cmd_add $_[1] },
227     "static" => sub { $STATIC = 1 },
228     "<>" => sub { cmd_file $_[1] },
229     or exit 1;
230    
231     my $data;
232     my @index;
233     my @order = sort {
234     length $a <=> length $b
235     or $a cmp $b
236     } keys %pm;
237    
238     # sorting by name - better compression, but needs more metadata
239     # sorting by length - faster lookup
240     # usually, the metadata overhead beats the loss through compression
241    
242     for my $pm (@order) {
243     my $path = $pm{$pm};
244    
245     128 > length $pm
246     or die "$pm: path too long (only 128 octets supported)\n";
247    
248     my $src = ref $path
249     ? $$path
250     : do {
251     open my $pm, "<:perlio", $path
252     or die "$path: $!";
253    
254     local $/;
255    
256     <$pm>
257     };
258    
259     if ($pm =~ /^auto\/POSIX\/[^\/]+\.al$/) {
260     if ($src =~ /^ unimpl \"/m) {
261     warn "$pm: skipping (not implemented anyways).\n"
262     if $VERBOSE >= 2;
263     next;
264     }
265     }
266    
267     if ($STRIP =~ /ppi/i) {
268     require PPI;
269    
270     my $ppi = PPI::Document->new (\$src);
271     $ppi->prune ("PPI::Token::Comment");
272     $ppi->prune ("PPI::Token::Pod");
273    
274     # prune END stuff
275     for (my $last = $ppi->last_element; $last; ) {
276     my $prev = $last->previous_token;
277    
278     if ($last->isa (PPI::Token::Whitespace::)) {
279     $last->delete;
280     } elsif ($last->isa (PPI::Statement::End::)) {
281     $last->delete;
282     last;
283     } elsif ($last->isa (PPI::Token::Pod::)) {
284     $last->delete;
285     } else {
286     last;
287     }
288    
289     $last = $prev;
290     }
291    
292     # prune some but not all insignificant whitespace
293     for my $ws (@{ $ppi->find (PPI::Token::Whitespace::) }) {
294     my $prev = $ws->previous_token;
295     my $next = $ws->next_token;
296    
297     if (!$prev || !$next) {
298     $ws->delete;
299     } else {
300     if (
301     $next->isa (PPI::Token::Operator::) && $next->{content} =~ /^(?:,|=|!|!=|==|=>)$/ # no ., because of digits. == float
302     or $prev->isa (PPI::Token::Operator::) && $prev->{content} =~ /^(?:,|=|\.|!|!=|==|=>)$/
303     or $prev->isa (PPI::Token::Structure::)
304     # decrease size, decrease compressability
305     #or ($prev->isa (PPI::Token::Word::)
306     # && (PPI::Token::Symbol:: eq ref $next
307     # || $next->isa (PPI::Structure::Block::)
308     # || $next->isa (PPI::Structure::List::)
309     # || $next->isa (PPI::Structure::Condition::)))
310     ) {
311     $ws->delete;
312     } elsif ($prev->isa (PPI::Token::Whitespace::)) {
313     $ws->{content} = ' ';
314     $prev->delete;
315     } else {
316     $ws->{content} = ' ';
317     }
318     }
319     }
320    
321     # prune whitespace around blocks
322     if (0) {
323     # these usually decrease size, but decrease compressability more
324     for my $struct (PPI::Structure::Block::, PPI::Structure::Condition::) {
325     for my $node (@{ $ppi->find ($struct) }) {
326     my $n1 = $node->first_token;
327     my $n2 = $n1->previous_token;
328     $n1->delete if $n1->isa (PPI::Token::Whitespace::);
329     $n2->delete if $n2 && $n2->isa (PPI::Token::Whitespace::);
330     my $n1 = $node->last_token;
331     my $n2 = $n1->next_token;
332     $n1->delete if $n1->isa (PPI::Token::Whitespace::);
333     $n2->delete if $n2 && $n2->isa (PPI::Token::Whitespace::);
334     }
335     }
336    
337     for my $node (@{ $ppi->find (PPI::Structure::List::) }) {
338     my $n1 = $node->first_token;
339     $n1->delete if $n1->isa (PPI::Token::Whitespace::);
340     my $n1 = $node->last_token;
341     $n1->delete if $n1->isa (PPI::Token::Whitespace::);
342     }
343     }
344    
345     # reformat qw() lists which often have lots of whitespace
346     for my $node (@{ $ppi->find (PPI::Token::QuoteLike::Words::) }) {
347     if ($node->{content} =~ /^qw(.)(.*)(.)$/s) {
348     my ($a, $qw, $b) = ($1, $2, $3);
349     $qw =~ s/^\s+//;
350     $qw =~ s/\s+$//;
351     $qw =~ s/\s+/ /g;
352     $node->{content} = "qw$a$qw$b";
353     }
354     }
355    
356     $src = $ppi->serialize;
357     } elsif ($STRIP =~ /pod/i && $pm ne "Opcode.pm") { # opcode parses it's own pod
358     require Pod::Strip;
359    
360     my $stripper = Pod::Strip->new;
361    
362     my $out;
363     $stripper->output_string (\$out);
364     $stripper->parse_string_document ($src);
365     $src = $out;
366     }
367    
368     if ($VERIFY && $pm =~ /\.pm$/ && $pm ne "Opcode.pm") {
369     if (open my $fh, "-|") {
370     <$fh>;
371     } else {
372     eval "#line 1 \"$pm\"\n$src" or warn "\n\n\n$pm\n\n$src\n$@\n\n\n";
373     exit 0;
374     }
375     }
376    
377     # if ($pm eq "Opcode.pm") {
378     # open my $fh, ">x" or die; print $fh $src;#d#
379     # exit 1;
380     # }
381    
382     warn "adding $pm\n"
383     if $VERBOSE >= 2;
384    
385     push @index, ((length $pm) << 25) | length $data;
386     $data .= $pm . $src;
387     }
388    
389     length $data < 2**25
390     or die "bundle too large (only 32MB supported)\n";
391    
392     my $varpfx = "bundle_" . substr +(Digest::MD5::md5_hex $data), 0, 16;
393    
394     #############################################################################
395     # output
396    
397     print "generating $PREFIX.h... ";
398    
399     {
400     open my $fh, ">", "$PREFIX.h"
401     or die "$PREFIX.h: $!\n";
402    
403     print $fh <<EOF;
404     /* do not edit, automatically created by mkstaticbundle */
405     #include <EXTERN.h>
406     #include <perl.h>
407     #include <XSUB.h>
408    
409     /* public API */
410     EXTERN_C PerlInterpreter *staticperl;
411     EXTERN_C void staticperl_init (void);
412     EXTERN_C void staticperl_cleanup (void);
413     EOF
414     }
415    
416     print "\n";
417    
418     #############################################################################
419     # output
420    
421     print "generating $PREFIX.c... ";
422    
423     open my $fh, ">", "$PREFIX.c"
424     or die "$PREFIX.c: $!\n";
425    
426     print $fh <<EOF;
427     /* do not edit, automatically created by mkstaticbundle */
428    
429     #include <EXTERN.h>
430     #include <perl.h>
431     #include <XSUB.h>
432    
433     #include "bundle.h"
434    
435     /* public API */
436     PerlInterpreter *staticperl;
437    
438     EOF
439    
440     #############################################################################
441     # bundle data
442    
443     my $count = @index;
444    
445     print $fh <<EOF;
446     #include "bundle.h"
447    
448     /* bundle data */
449    
450     static const U32 $varpfx\_count = $count;
451     static const U32 $varpfx\_index [$count + 1] = {
452     EOF
453    
454     my $col;
455     for (@index) {
456     printf $fh "0x%08x,", $_;
457     print $fh "\n" unless ++$col % 10;
458    
459     }
460     printf $fh "0x%08x\n};\n", (length $data);
461    
462     print $fh "static const char $varpfx\_data [] =\n";
463     dump_string $fh, $data;
464    
465     print $fh ";\n\n";;
466    
467     #############################################################################
468     # bootstrap
469    
470     # boot file for staticperl
471     # this file will be eval'ed at initialisation time
472    
473     my $bootstrap = '
474     BEGIN {
475     package ' . $PACKAGE . ';
476    
477     PerlIO::scalar->bootstrap;
478    
479     @INC = sub {
480     my $data = find "$_[1]"
481     or return;
482    
483     $INC{$_[1]} = $_[1];
484    
485     open my $fh, "<", \$data;
486     $fh
487     };
488     }
489     ';
490    
491     $bootstrap .= "require '//boot';"
492     if exists $pm{"//boot"};
493    
494     $bootstrap =~ s/\s+/ /g;
495     $bootstrap =~ s/(\W) /$1/g;
496     $bootstrap =~ s/ (\W)/$1/g;
497    
498     print $fh "const char bootstrap [] = ";
499     dump_string $fh, $bootstrap;
500     print $fh ";\n\n";
501    
502     print $fh <<EOF;
503     /* search all bundles for the given file, using binary search */
504     XS(find)
505     {
506     dXSARGS;
507    
508     if (items != 1)
509     Perl_croak (aTHX_ "Usage: $PACKAGE\::find (\$path)");
510    
511     {
512     STRLEN namelen;
513     char *name = SvPV (ST (0), namelen);
514     SV *res = 0;
515    
516     int l = 0, r = $varpfx\_count;
517    
518     while (l <= r)
519     {
520     int m = (l + r) >> 1;
521     U32 idx = $varpfx\_index [m];
522     int comp = namelen - (idx >> 25);
523    
524     if (!comp)
525     {
526     int ofs = idx & 0x1FFFFFFU;
527     comp = memcmp (name, $varpfx\_data + ofs, namelen);
528    
529     if (!comp)
530     {
531     /* found */
532     int ofs2 = $varpfx\_index [m + 1] & 0x1FFFFFFU;
533    
534     ofs += namelen;
535     res = newSVpvn ($varpfx\_data + ofs, ofs2 - ofs);
536     goto found;
537     }
538     }
539    
540     if (comp < 0)
541     r = m - 1;
542     else
543     l = m + 1;
544     }
545    
546     XSRETURN (0);
547    
548     found:
549     ST (0) = res;
550     sv_2mortal (ST (0));
551     }
552    
553     XSRETURN (1);
554     }
555    
556     /* list all files in the bundle */
557     XS(list)
558     {
559     dXSARGS;
560    
561     if (items != 0)
562     Perl_croak (aTHX_ "Usage: $PACKAGE\::list");
563    
564     {
565     int i;
566    
567     EXTEND (SP, $varpfx\_count);
568    
569     for (i = 0; i < $varpfx\_count; ++i)
570     {
571     U32 idx = $varpfx\_index [i];
572    
573     PUSHs (newSVpvn ($varpfx\_data + (idx & 0x1FFFFFFU), idx >> 25));
574     }
575     }
576    
577     XSRETURN ($varpfx\_count);
578     }
579    
580     static char *args[] = {
581     "staticperl",
582     "-e",
583     "0"
584     };
585    
586     EOF
587    
588     #############################################################################
589     # xs_init
590    
591     print $fh <<EOF;
592     static void
593     xs_init (pTHX)
594     {
595     EOF
596    
597     @static_ext = ("DynaLoader", sort @static_ext);
598    
599     # prototypes
600     for (@static_ext) {
601     s/\.pm$//;
602     (my $cname = $_) =~ s/\//__/g;
603     print $fh " EXTERN_C void boot_$cname (pTHX_ CV* cv);\n";
604     }
605    
606     print $fh <<EOF;
607     char *file = __FILE__;
608     dXSUB_SYS;
609    
610     newXSproto ("$PACKAGE\::find", find, file, "\$");
611     newXSproto ("$PACKAGE\::list", list, file, "");
612     EOF
613    
614     # calls
615     for (@static_ext) {
616     s/\.pm$//;
617    
618     (my $cname = $_) =~ s/\//__/g;
619     (my $pname = $_) =~ s/\//::/g;
620    
621     my $bootstrap = $pname eq "DynaLoader" ? "boot" : "bootstrap";
622    
623     print $fh " newXS (\"$pname\::$bootstrap\", boot_$cname, file);\n";
624     }
625    
626     print $fh <<EOF;
627     Perl_av_create_and_unshift_one (&PL_preambleav, newSVpv (bootstrap, sizeof (bootstrap) - 1));
628     }
629     EOF
630    
631     #############################################################################
632     # optional perl_init/perl_destroy
633    
634     if ($PERL) {
635     print $fh <<EOF;
636    
637     int
638     main (int argc, char *argv [])
639     {
640     extern char **environ;
641     int exitstatus;
642    
643     PERL_SYS_INIT3 (&argc, &argv, &environ);
644     staticperl = perl_alloc ();
645     perl_construct (staticperl);
646    
647     PL_exit_flags |= PERL_EXIT_DESTRUCT_END;
648    
649     exitstatus = perl_parse (staticperl, xs_init, argc, argv, environ);
650     if (!exitstatus)
651     perl_run (staticperl);
652    
653     exitstatus = perl_destruct (staticperl);
654     perl_free (staticperl);
655     PERL_SYS_TERM ();
656    
657     return exitstatus;
658     }
659     EOF
660     } else {
661     print $fh <<EOF;
662    
663     EXTERN_C void
664     staticperl_init (void)
665     {
666     extern char **environ;
667     int argc = sizeof (args) / sizeof (args [0]);
668     char **argv = args;
669    
670     PERL_SYS_INIT3 (&argc, &argv, &environ);
671     staticperl = perl_alloc ();
672     perl_construct (staticperl);
673     PL_origalen = 1;
674     PL_exit_flags |= PERL_EXIT_DESTRUCT_END;
675     perl_parse (staticperl, xs_init, argc, argv, environ);
676    
677     perl_run (staticperl);
678     }
679    
680     EXTERN_C void
681     staticperl_cleanup (void)
682     {
683     perl_destruct (staticperl);
684     perl_free (staticperl);
685     staticperl = 0;
686     PERL_SYS_TERM ();
687     }
688     EOF
689     }
690    
691     print -s "$PREFIX.c", " octets (", (length $data) , " data octets).\n\n";
692    
693     #############################################################################
694     # libs, cflags
695    
696     {
697     print "generating $PREFIX.ccopts... ";
698    
699     my $str = "$Config{ccflags} $Config{optimize} $Config{cppflags} -I$Config{archlibexp}/CORE";
700     $str =~ s/([\(\)])/\\$1/g;
701    
702     print "$str\n\n";
703    
704     open my $fh, ">$PREFIX.ccopts"
705     or die "$PREFIX.ccopts: $!";
706     print $fh $str;
707     }
708    
709     {
710     print "generating $PREFIX.ldopts... ";
711    
712     my $str = $STATIC ? "--static " : "";
713    
714     $str .= "$Config{ccdlflags} $Config{ldflags} @libs $Config{archlibexp}/CORE/$Config{libperl} $Config{perllibs}";
715    
716     my %seen;
717     $str .= " $_" for grep !$seen{$_}++, ($extralibs =~ /(\S+)/g);
718    
719     $str =~ s/([\(\)])/\\$1/g;
720    
721     print "$str\n\n";
722    
723     open my $fh, ">$PREFIX.ldopts"
724     or die "$PREFIX.ldopts: $!";
725     print $fh $str;
726     }
727    
728     if ($PERL) {
729     system "$Config{cc} \$(cat bundle.ccopts\) -o perl bundle.c \$(cat bundle.ldopts\)";
730    
731     unlink "$PREFIX.$_"
732     for qw(ccopts ldopts c h);
733     }
734