Annotation of comics/fetch.pl.new, revision 1.10

1.1       nick        1: #!/usr/bin/perl -w
                      2: 
                      3: use strict;
                      4: use File::Path;
                      5: use Data::Dumper;
1.8       nick        6: use Pod::Usage;
                      7: use Getopt::Long;
1.1       nick        8: 
                      9: ## 
                     10: ## Some default values
                     11: ##
1.10    ! nick       12: my $ver                = q/$Id: fetch.pl.new,v 1.9 2013-02-25 13:48:21 nick Exp $/;
1.1       nick       13: my $comicFile   = "comics.conf";
                     14: my %comics     = &readComicConfig ( $comicFile );
1.8       nick       15: my %opts        = &fetchOptions( );
                     16: my $days_ago    = $opts{'days'} || 0;
1.1       nick       17: my %dates      = &fetchDates();
                     18: my $baseDir     = $comics{'configs'}{'base_directory'} || ".";
                     19: my $imageDir    = $baseDir . "/" . ( $comics{'configs'}{'image_directory'} || "images" ) . 
                     20:                  "/$dates{'mon2'}$dates{'year2'}";
                     21: my $indexDir    = $baseDir . "/" . ( $comics{'configs'}{'index_directory'} || "indexes" );
1.2       nick       22: my $USER_AGENT  = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.18) Gecko/20110628 Ubuntu/10.10 (maverick) Firefox/3.6.18";
1.8       nick       23: my @days        = qw/ Sunday Monday Tuesday Wednesday Thursday Friday Saturday /;
1.1       nick       24: 
                     25: 
                     26: my $DATE=`date`; chomp $DATE;
                     27: print STDOUT "Starting comic fetch at $DATE\n";
                     28: 
                     29: ##
                     30: ## Main program starts here
                     31: ##
                     32: &checkDir ( [ $imageDir, $indexDir ] );
                     33: 
1.5       nick       34: &writeTitle ( \%dates );
1.1       nick       35: 
                     36: foreach my $comic ( sort keys %comics ) {
                     37:   next if ( $comic =~ m/config/ );
                     38:   $comics{$comic}{'error'} = &downloadComic ( \%comics, $comic, \%dates );
                     39:   &writeComic ( \%comics, $comic, \%dates );
                     40: }
                     41: 
1.8       nick       42: print "Finding in $imageDir/*-$dates{'day2'}.jpg\n";
                     43: foreach my $file ( glob( "$imageDir/*-$dates{'day2'}.jpg" ) ) 
1.4       nick       44: {
                     45:        my $size = `/usr/bin/identify $file`;
                     46:        $size =~ s/.*\s(\d+)x\d+.*/$1/;
                     47: 
                     48:        system( "/usr/bin/convert -resize 640 $file $file" )
                     49:                if ( $size > 640 ) 
                     50: }
                     51: 
1.1       nick       52: ## &writeMainIndex ( \%dates );
                     53: 
                     54: &writeFooter( \%dates );
                     55: 
                     56: $DATE=`date`;  chomp( $DATE );
                     57: print STDOUT "Completed comic fetch at $DATE\n";
                     58: 
                     59: ## End
                     60: 
                     61: #######################################################################
                     62: ## Function :  downloadComic
                     63: ##
                     64: ##   Description :
                     65: ##     This function determines the download method being used to 
                     66: ##      retrieve the comic and calls the apprioriate function.
                     67: ##
                     68: ##      If the mode is invalid an error will be returned.
                     69: ##
                     70: #######################################################################
                     71: sub downloadComic ($$) {
                     72:        my ( $comics, $comic, $date ) = @_;
                     73: 
                     74:        SWITCH: {
                     75:                if ( $comics->{$comic}{'mode'} eq 1 ) { 
                     76:                        return indexDownload ( \%comics, $comic, $date );
                     77:                        last SWITCH;
                     78:                }
                     79:                if ( $comics->{$comic}{'mode'} eq 2 ) { 
                     80:                        return directDownload ( \%comics, $comic, $date );
                     81:                        last SWITCH;
                     82:                }
                     83:        }
                     84:         
                     85:        return "ERROR: Unknown download method specified for $comics->{$comic}{'fullName'}.";
                     86: }
                     87: 
                     88: #######################################################################
                     89: #######################################################################
                     90: sub readComicConfig ($$) {
                     91:        my ( $comicFile ) = @_;
                     92:        my %comicConfig   = ( );
                     93:        my %config        = ( );
                     94: 
                     95:        open FILEN, "<$comicFile";
                     96:                while (<FILEN>) {
                     97:                        if ( ( $_ !~ m/^#/ ) && ( $_ =~ m/,.*,/) ){
                     98:                                my @res = split /,/, $_;
                     99:                                $comicConfig{$res[0]}{'url'}      = $res[1];
                    100:                                $comicConfig{$res[0]}{'search'}   = $res[2];
                    101:                                $comicConfig{$res[0]}{'mode'}     = $res[3];
                    102:                                $comicConfig{$res[0]}{'fullName'} = $res[4];
                    103:                                $comicConfig{$res[0]}{'ext'}      = $res[5];
                    104:                                $comicConfig{$res[0]}{'error'}    = 0;
                    105:                        }
                    106:                        elsif ( $_ =~ m/(.*)\s+=\s+(.*)/ ) {
                    107:                                $comicConfig{'configs'}{$1} = $2;
                    108:                        }
                    109:                }
                    110:        close (FILEN);
                    111: 
                    112:        return %comicConfig;
                    113: }
                    114: 
                    115: #######################################################################
                    116: #######################################################################
                    117: sub writeComic ($$) {
                    118:        my ( $comics, $comic, $date ) = @_;
1.9       nick      119:        my $indexFile = $indexDir . "/index-" . $days[$date->{'dow'}] .
                    120:                         "-" . $date->{'year2'} . $date->{'mon2'} .
1.1       nick      121:                         $date->{'day2'} . ".html";
                    122:        my $content = <<EOF;
                    123: 
                    124: <!-- ********* Begin $comic ($comics->{$comic}{'fullName'}) ******* -->
                    125:   <tr>
                    126:     <td align="left">
                    127: <font color="blue"><b>$comics->{$comic}{'fullName'}</b></font> &nbsp; &nbsp; 
                    128: <font size="-2">
                    129:        <a href="$comics->{$comic}{'url'}">
                    130:                $comics->{$comic}{'url'}
                    131:        </a>
                    132: </font><br/>
                    133: <img src="../images/$date->{'mon2'}$date->{'year2'}/$comic-$date->{'day2'}.jpg" alt="$comic-$date->{'day2'}" />
                    134: <br/><br/>
                    135: </td></tr>
                    136: <!-- ********* Finish $comic ($comics->{$comic}{'fullName'}) ******* -->
                    137: 
                    138: EOF
                    139:        open INDEX, ">>$indexFile";
                    140: 
                    141:        print INDEX $content if ( ! $comics->{$comic}{'error'} );
                    142: 
                    143:        print INDEX <<EOF
                    144: <font color="blue"><b>$comics->{$comic}{'fullName'}</b></font> &nbsp; &nbsp;
                    145: <font size="-2"><
                    146:         <a href="$comics->{$comic}{'url'}">
                    147:                 $comics->{$comic}{'url'}
                    148:         </a>
                    149: </font><br/>
                    150: <font color="red"><b>$comic :  $comics->{$comic}{'error'}</b></font><br/>
                    151:   </td>
                    152: </tr>
                    153: EOF
                    154:                if ( $comics->{$comic}{'error'} );
                    155: 
                    156:        close (INDEX);
                    157: 
                    158:        return 0;
                    159: }
                    160: 
                    161: 
                    162: #######################################################################
                    163: #######################################################################
                    164: sub writeMainIndex ($$) {
                    165:        my ( $date ) = @_;
                    166: 
                    167: }
                    168: 
                    169: 
                    170: #######################################################################
                    171: #######################################################################
                    172: sub writeFooter {
                    173:        my ( $date ) = @_;
1.10    ! nick      174:        my $indexFile = $indexDir . "/index-" . $days[$date->{'dow'}] . 
        !           175:                        "-" . $date->{'year2'} . $date->{'mon2'} .
1.1       nick      176:                         $date->{'day2'} . ".html";
                    177:        my $sysDate = `date`;
                    178: 
                    179:        open INDEX, ">>$indexFile";
                    180:        print INDEX <<EOF;
                    181: </table>
1.3       nick      182: <center>
                    183: <font size="2">
                    184: Generated on: <font color="green">$sysDate</font><br/>
1.7       nick      185: Version: <font color="green">$ver</font><br />
                    186: CVS: <a href="http://demandred.dyndns.org/cgi-bin/cvsweb/comics/">http://demandred.dyndns.org/cgi-bin/cvsweb/comics/</a>
1.1       nick      187:   <p>
                    188:     <a href="http://validator.w3.org/check?uri=referer"><img
                    189:       src="http://www.w3.org/Icons/valid-xhtml10-blue" alt="Valid XHTML 1.0 Transitional" height="31" width="88" border="0" /></a>
                    190:   </p>
                    191: </center>
                    192: 
                    193: </body>
                    194: </html>
                    195: EOF
                    196:        close( INDEX );
                    197: }
                    198: 
                    199: #######################################################################
                    200: #######################################################################
                    201: sub checkDir ($$) {
                    202:        my @dir = @_;
                    203: 
                    204:        foreach ( @dir ) {
                    205:                if ( ! -d $_ ) { mkpath( $_ ); }
                    206:        }
                    207: }
                    208: 
                    209: #######################################################################
                    210: #######################################################################
                    211: sub writeTitle ($$) {
                    212:        my ( $date ) = @_;
1.10    ! nick      213:        my $indexFile = $indexDir . "/index-" . $days[$date->{'dow'}] . 
        !           214:                        "-" . $date->{'year2'} . $date->{'mon2'} .
1.1       nick      215:                          $date->{'day2'} . ".html";
1.8       nick      216:        my $today     = $days[$date->{'dow'}] . " " . $date->{'mon'} . "/" . $date->{'day'} . "/" . $date->{'year'};
1.1       nick      217: 
                    218:        open INDEX, ">$indexFile";
                    219:        print INDEX <<EOF;
                    220: <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
                    221: 
                    222: <html xmlns="http://www.w3.org/1999/xhtml">
                    223: <head>
                    224: <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
                    225:     <title>Daily Comics for $today</title>
                    226:   </head>
                    227: <body bgcolor="#FFFFFF">
                    228: <h1>Daily Comics for $today</h1>
                    229: <table align="center" cellpadding="5" cellspacing="0">
                    230: EOF
                    231:        close (INDEX);
                    232: }
                    233: 
                    234: #######################################################################
                    235: #######################################################################
                    236: sub directDownload ($$) {
                    237:        my ( $comics, $comic, $date ) = @_;
                    238:        my $file = &parseComic ( $comics, $comic, $date );
                    239: 
                    240:         ##
                    241:         ## Save the file to the appropriate directory
                    242:         ##
                    243:         my $cDir  = $date->{'mon2'} . $date->{'year2'};
                    244:         my $cDate = $date->{'day2'};
                    245: 
1.3       nick      246:        my $cmd = "wget -q $file --referer=\"" . $comics->{$comic}{'url'} ."\" --user-agent=\"$USER_AGENT\" -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg";
1.1       nick      247:         return system($cmd);
                    248: }
                    249: 
                    250: #######################################################################
                    251: #######################################################################
                    252: sub indexDownload ($$) {
                    253:        my ( $comics, $comic, $date ) = @_;
                    254:        my ( @lines, $comicLine, $mainURL );
                    255:        my $comicIndex = "indexes/index.$comic";
                    256: 
                    257:        `wget -q $comics->{$comic}{'url'} -O $comicIndex`;
                    258: 
                    259:        if ( ! open FILEN, "<$comicIndex" ) {  
                    260:                return "ERROR: Can't open index file for " . $comics->{$comic}{'fullName'} . 
                    261:                       " (" . $comics->{$comic}{'url'} . ")"; 
                    262:        } 
                    263:                @lines = <FILEN>;
                    264:        close (FILEN);  
                    265: 
                    266:        unlink ("$comicIndex");
                    267: 
                    268:        $mainURL = $comics->{$comic}{'url'};
                    269:        ## I need to figure out how to merge these two in to one regex.
                    270:        $mainURL =~ s/(http:\/\/.*)(?:\/.*\/){1,}.*/$1/;
                    271:        $mainURL =~ s/([a-z])\/.*/$1/i;
                    272: 
                    273:        ##
                    274:        ## Find the comic strip URL based on the specified regex in the search
                    275:        ##
                    276:        foreach my $line (@lines) {
                    277:                if ( $line =~ m/$comics->{$comic}{'search'}/ ) {
                    278:                        $comicLine = $1; chomp $comicLine;
                    279:                }
                    280:         }
                    281: 
                    282:        ##
                    283:        ## Save the file to the appropriate directory
                    284:        ##
                    285:        my $cDir    = $date->{'mon2'} . $date->{'year2'};
                    286:        my $cDate   = $date->{'day2'};
                    287: 
                    288:        if ( $comicLine ) {
                    289:                if ( $comicLine =~ m/(gif|jpg|png)/i ) { $comics->{$comic}{'ext'} = $1; }
                    290:                my $comicURL = ( $comicLine =~ m/http/ ) ? $comicLine : $mainURL . $comicLine;
1.3       nick      291:                my $cmd = "wget --user-agent=\"$USER_AGENT\" --referer=\"" . $comics->{$comic}{'url'} . "\" -q $comicURL -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg";
1.1       nick      292:                system( $cmd );
                    293:                return 0;
                    294:        }
                    295: 
                    296:        unlink "index.html";
                    297: 
                    298:        return "ERROR: Could not download comic $comics->{$comic}{'fullName'}";
                    299: }
                    300: 
                    301: #######################################################################
                    302: #######################################################################
                    303: sub parseComic ($$) {
                    304:        my ( $comics, $comic, $date ) = @_;
                    305:        my $string = $comics->{$comic}{'search'};
                    306: 
                    307:        $string =~ s/__year__/$date->{'year'}/g;
                    308:        $string =~ s/__year2__/$date->{'year2'}/g;
                    309:        $string =~ s/__mon__/$date->{'mon'}/g;
                    310:        $string =~ s/__mon2__/$date->{'mon2'}/g;
                    311:        $string =~ s/__day__/$date->{'day'}/g;
                    312:        $string =~ s/__day2__/$date->{'day2'}/g;
                    313:        $string =~ s/__ext__/$comics->{$comic}{'ext'}/g;
                    314:        chomp $string;
                    315: 
                    316:        return $string;
                    317: }
                    318: 
                    319: #######################################################################
                    320: #######################################################################
                    321: sub fetchDates () {
                    322:        my %dates = ();
                    323: 
1.8       nick      324:        ($dates{'day'}, $dates{'mon'}, $dates{'year'}, $dates{'dow'}) = (localtime(time - (86400 * $days_ago )))[3,4,5,6];
1.1       nick      325: 
                    326:        $dates{'year'} += 1900;
                    327:        $dates{'year2'} = substr $dates{'year'}, 2, 2;
                    328:        $dates{'day2'}  = ( $dates{'day'} < 10 ) ? "0" . $dates{'day'} : $dates{'day'}; 
                    329:        $dates{'mon'}++;
                    330:        $dates{'mon2'}  = ( $dates{'mon'} < 10 ) ? "0".$dates{'mon'} : $dates{'mon'};
                    331: 
                    332:        return %dates;
                    333: }
1.8       nick      334: 
                    335: ###############################################################################
                    336: ##
                    337: ## &fetchOptions( );
                    338: ##
                    339: ##      Grab our command line arguments and toss them in to a hash
                    340: ##
                    341: ###############################################################################
                    342: sub fetchOptions {
                    343:         my %opts;
                    344: 
                    345:         &GetOptions(
                    346:                         "days:i"        => \$opts{'days'},
                    347:                         "help|?"        => \$opts{'help'},
                    348:                         "man"           => \$opts{'man'},
                    349:                    ) || &pod2usage( );
                    350:         &pod2usage( ) if defined $opts{'help'};
                    351:         &pod2usage( { -verbose => 2, -input => \*DATA } ) if defined $opts{'man'};
                    352: 
                    353:         return %opts;
                    354: }
                    355: 
                    356: __END__
                    357: 
                    358: =head1 NAME
                    359: 
                    360: fetch.pl - Fetches comics and places them all locally in a single html file.
                    361: 
                    362: =head1 SYNOPSIS
                    363: 
                    364: fetch.pl [options]
                    365: 
                    366: Options:
                    367:         --days,d        Fetch comics from X days ago
                    368:         --help,?        Display the basic help menu
                    369:         --man,m         Display the detailed man page
                    370: 
                    371: =head1 DESCRIPTION
                    372: 
                    373: =head1 HISTORY
                    374: 
                    375: =head1 AUTHOR
                    376: 
                    377: Nicholas DeClario <nick@declario.com>
                    378: 
                    379: =head1 BUGS
                    380: 
                    381: This is a work in progress.  Please report all bugs to the author.
                    382: 
                    383: =head1 SEE ALSO
                    384: 
                    385: =head1 COPYRIGHT
                    386: 
                    387: =cut
                    388: 
                    389: 

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>