[BACK]Return to fetch.pl.new CVS log [TXT] [DIR] Up to [Local Repository] / comics

Diff for /comics/fetch.pl.new between versions 1.13 and 1.23

version 1.13, 2013/03/12 13:09:34 version 1.23, 2018/01/26 13:05:27
Line 1 Line 1
 #!/usr/bin/perl -w  #!/usr/bin/perl -w
   
   ###############################################################################
   # $Log$
   # Revision 1.23  2018/01/26 13:05:27  nick
   # Added a new config option to remove all newline from the resulting index.html
   # file.  This allows for easier parsing for certain comics.  I then updated
   # the URLs to search for and enabled the newline removal for a handful
   # of uComics.
   #
   # I believe I've also properly fixed the Comic Config version displayed on
   # the webpage itself.
   #
   # Revision 1.22  2017/12/05 13:37:40  nick
   # Added the CVS config version to the outpuit.
   #
   # Revision 1.21  2015/10/26 14:25:40  nick
   # Fixed a bug that was improperly including the day of week string preventing the weekend comics from fetching proproperly.
   #
   # Revision 1.20  2015/10/22 12:58:44  nick
   # Added the ability for Sunday only comics.  Stonesoup is no longer weekdays, this has been added to Sunday only.  I also added Foxtrot Classics for weekdays and Foxtrot for Sundays.
   #
   # Revision 1.19  2015/07/13 12:56:58  nick
   # Added Sally Forth and Pearls Before Swine.  Adding Sally Forth required a change in the 'wget' command for fetching the index file to include 'user-agent' and 'referer'.
   #
   # Revision 1.18  2015/05/07 12:31:43  nick
   # Added favicon
   #
   # Revision 1.17  2015/02/19 14:56:10  nick
   # Fixed a problem that forced everything to JPG.  This would kill GIF animations, but would not display the gifs either because 'convert' appends an index number to the end of the file name for each from of the GIF animation.  I fixed this to maintain GIF compatibilty as well as rewritting how the script fetches the size of the file.  Additionally, I updated the configuration for Questionable Content to search for GIF or JPG, which is what triggered this entire update.
   #
   # Revision 1.16  2015/02/05 18:05:58  nick
   # Changed the background and added a fancy title.
   #
   # Revision 1.15  2015/01/19 13:46:19  nick
   # *** empty log message ***
   #
   ###############################################################################
   
 use strict;  use strict;
 use File::Path;  use File::Path;
 use Data::Dumper;  use Data::Dumper;
 use Pod::Usage;  use Pod::Usage;
 use Getopt::Long;  use Getopt::Long;
   
   use Date::Calc qw/Date_to_Text_Long Today Day_of_Week Day_of_Week_to_Text/;
   
 ##   ## 
 ## Some default values  ## Some default values
 ##  ##
 my $ver         = q/$Id$/;  my $ver         = '$Id$';
 my $comicFile   = "comics.conf";  my $comicFile   = "comics.conf";
   my $comicConfigVer = "Unknown";
 my %comics      = &readComicConfig ( $comicFile );  my %comics      = &readComicConfig ( $comicFile );
 my %opts        = &fetchOptions( );  my %opts        = &fetchOptions( );
 my $days_ago    = $opts{'days'} || 0;  my $days_ago    = $opts{'days'} || 0;
Line 22  my $indexDir    = $baseDir . "/" . ( $co Line 62  my $indexDir    = $baseDir . "/" . ( $co
 my $USER_AGENT  = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.18) Gecko/20110628 Ubuntu/10.10 (maverick) Firefox/3.6.18";  my $USER_AGENT  = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.18) Gecko/20110628 Ubuntu/10.10 (maverick) Firefox/3.6.18";
 my @days        = qw/ Sunday Monday Tuesday Wednesday Thursday Friday Saturday /;  my @days        = qw/ Sunday Monday Tuesday Wednesday Thursday Friday Saturday /;
   
   
 my $DATE=`date`; chomp $DATE;  my $DATE=`date`; chomp $DATE;
 print STDOUT "Starting comic fetch at $DATE\n";  print STDOUT "Starting comic fetch at $DATE\n";
   
Line 34  print STDOUT "Starting comic fetch at $D Line 73  print STDOUT "Starting comic fetch at $D
 &writeTitle ( \%dates );  &writeTitle ( \%dates );
   
 foreach my $comic ( sort keys %comics ) {  foreach my $comic ( sort keys %comics ) {
   
     ## Skip if this is Sunday and the comic is weekdays only
   next if ( $comic =~ m/config/ );    next if ( $comic =~ m/config/ );
     if (($dates{'wday'} eq "Sunday") && 
         ($comics{$comic}{'sunday'} == 0)) {
       print "Skipping '$comic'; Weekdays only.\n";
       next;
     }
   
     ## Skip if Sunday only comic and it's not Sunday.
     if (($dates{'wday'} ne "Sunday") &&
         ($comics{$comic}{'sunday_only'} == 1)) {
       print "Skipping '$comic' ($comics{$comic}{'sunday_only'}); Sunday only.\n";
       next
     }
   
   $comics{$comic}{'error'} = &downloadComic ( \%comics, $comic, \%dates );    $comics{$comic}{'error'} = &downloadComic ( \%comics, $comic, \%dates );
   &writeComic ( \%comics, $comic, \%dates );    &writeComic ( \%comics, $comic, \%dates );
 }  
   
 print "Finding in $imageDir/*-$dates{'day2'}.jpg\n";      my $file = "$imageDir/$comic-$dates{'day2'}.$comics{$comic}{'ext'}";
 foreach my $file ( glob( "$imageDir/*-$dates{'day2'}.jpg" ) )       my $size = 0;    
 {  
         my $size = `/usr/bin/identify $file`;      my $cmd = "/usr/bin/identify -verbose $file|";
         $size =~ s/.*\s(\d+)x\d+.*/$1/;      open(IMG, $cmd) || die ("Can't open: $!\n");
       while(<IMG>) {
           if ($_ =~ m/^\s+geometry:\s+(\d+)x\d+.*/i) {
               $size = $1 if ( $size == 0);
           }
       }
       close(IMG);
   
   
         system( "/usr/bin/convert -resize 640 $file $file" )          system( "/usr/bin/convert -resize 640 $file $file" )
                 if ( $size > 640 )                   if ( $size > 640 ) 
Line 92  sub readComicConfig ($$) { Line 152  sub readComicConfig ($$) {
         my %comicConfig   = ( );          my %comicConfig   = ( );
         my %config        = ( );          my %config        = ( );
   
       my ($year, $mon, $day) =( localtime(time))[5,4,3];
       $year += 1900;
       $mon = sprintf("%02d", ($mon + 1));
       $day = sprintf("%02d", $day);
   
         open FILEN, "<$comicFile";          open FILEN, "<$comicFile";
                 while (<FILEN>) {                  while (<FILEN>) {
               #if ($_ =~ m/^#.* \$Id$/) {
               if ($_ =~ m/^#.* \$Id$$/) {
                   $comicConfigVer = $1;
               }
                         if ( ( $_ !~ m/^#/ ) && ( $_ =~ m/,.*,/) ){                          if ( ( $_ !~ m/^#/ ) && ( $_ =~ m/,.*,/) ){
                   $_ =~ s/__YEAR__/$year/g;
                   $_ =~ s/__MON__/$mon/g;
                   $_ =~ s/__DAY__/$day/g;
                   
                                 my @res = split /,/, $_;                                  my @res = split /,/, $_;
                                 $comicConfig{$res[0]}{'url'}      = $res[1];                                  $comicConfig{$res[0]}{'url'}      = $res[1];
                                 $comicConfig{$res[0]}{'search'}   = $res[2];                                  $comicConfig{$res[0]}{'search'}   = $res[2];
                                 $comicConfig{$res[0]}{'mode'}     = $res[3];                                  $comicConfig{$res[0]}{'mode'}     = $res[3];
                                 $comicConfig{$res[0]}{'fullName'} = $res[4];                                  $comicConfig{$res[0]}{'fullName'} = $res[4];
                                 $comicConfig{$res[0]}{'ext'}      = $res[5];                                  $comicConfig{$res[0]}{'ext'}      = $res[5];
                   $comicConfig{$res[0]}{'sunday'}   = sprintf("%d", $res[6] || 1);
                   $comicConfig{$res[0]}{'sunday_only'} = sprintf("%d", $res[7] || 0);
                   $comicConfig{$res[0]}{'remove_newlines'} = sprintf("%d", $res[8] || 0);
                                 $comicConfig{$res[0]}{'error'}    = 0;                                  $comicConfig{$res[0]}{'error'}    = 0;
                         }                          }
                         elsif ( $_ =~ m/(.*)\s+=\s+(.*)/ ) {                          elsif ( $_ =~ m/(.*)\s+=\s+(.*)/ ) {
Line 131  sub writeComic ($$) { Line 207  sub writeComic ($$) {
                 $comics->{$comic}{'url'}                  $comics->{$comic}{'url'}
         </a>          </a>
 </font><br/>  </font><br/>
 <img src="../images/$date->{'mon2'}$date->{'year2'}/$comic-$date->{'day2'}.jpg" alt="$comic-$date->{'day2'}" />  <img src="../images/$date->{'mon2'}$date->{'year2'}/$comic-$date->{'day2'}.$comics->{$comic}{'ext'}" alt="$comic-$date->{'day2'}" />
 <br/><br/>  <br/><br/>
 </td></tr>  </td></tr>
 <!-- ********* Finish $comic ($comics->{$comic}{'fullName'}) ******* -->  <!-- ********* Finish $comic ($comics->{$comic}{'fullName'}) ******* -->
Line 185  sub writeFooter { Line 261  sub writeFooter {
 <font size="2">  <font size="2">
 Generated on: <font color="green">$sysDate</font><br/>  Generated on: <font color="green">$sysDate</font><br/>
 Version: <font color="green">$ver</font><br />  Version: <font color="green">$ver</font><br />
   Config Version: <font color="green">$comicConfigVer</font><br />
 CVS: <a href="http://demandred.dyndns.org/cgi-bin/cvsweb/comics/">http://demandred.dyndns.org/cgi-bin/cvsweb/comics/</a>  CVS: <a href="http://demandred.dyndns.org/cgi-bin/cvsweb/comics/">http://demandred.dyndns.org/cgi-bin/cvsweb/comics/</a>
   <p>    <p>
     <a href="http://validator.w3.org/check?uri=referer"><img      <a href="http://validator.w3.org/check?uri=referer"><img
Line 217  sub writeTitle ($$) { Line 294  sub writeTitle ($$) {
                         $date->{'mon2'} . $date->{'day2'} . "-" .                           $date->{'mon2'} . $date->{'day2'} . "-" . 
                         $sd . ".html";                          $sd . ".html";
         my $today     = $days[$date->{'dow'}] . " " . $date->{'mon'} . "/" . $date->{'day'} . "/" . $date->{'year'};          my $today     = $days[$date->{'dow'}] . " " . $date->{'mon'} . "/" . $date->{'day'} . "/" . $date->{'year'};
       my $today_long = Date_to_Text_Long(Today());
   
         open INDEX, ">$indexFile";          open INDEX, ">$indexFile";
         print INDEX <<EOF;          print INDEX <<EOF;
Line 226  sub writeTitle ($$) { Line 304  sub writeTitle ($$) {
 <head>  <head>
 <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />  <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
 <link href="/daily/comics/comics.css" type="text/css" rel="stylesheet" media="screen">  <link href="/daily/comics/comics.css" type="text/css" rel="stylesheet" media="screen">
   <link rel="shortcut icon" href="./favicon.ico">
     <title>Daily Comics for $today</title>      <title>Daily Comics for $today</title>
   </head>    </head>
 <body bgcolor="#FFFFFF">  <body bgcolor="#FFFFFF">
 <h1>Daily Comics for $today</h1>  
 <table align="center" cellpadding="5" cellspacing="0">  <table align="center" cellpadding="5" cellspacing="0">
   <tr><td>
   <table cellpadding="0" cellspacing="0" border="0">
   <tr><td align="Left"><img src="images/daily_comics_heading01.png"></td></tr>
   <tr><td align="left">$today_long</td></tr>
   <tr><td>&nbsp;</td></tr>
   </td</tr>
   
 EOF  EOF
         close (INDEX);          close (INDEX);
 }  }
Line 247  sub directDownload ($$) { Line 332  sub directDownload ($$) {
         my $cDir  = $date->{'mon2'} . $date->{'year2'};          my $cDir  = $date->{'mon2'} . $date->{'year2'};
         my $cDate = $date->{'day2'};          my $cDate = $date->{'day2'};
   
         my $cmd = "wget -q $file --referer=\"" . $comics->{$comic}{'url'} ."\" --user-agent=\"$USER_AGENT\" -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg";          my $cmd = "wget -q $file --referer=\"" . $comics->{$comic}{'url'} ."\" --user-agent=\"$USER_AGENT\" -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg";
   
         return system($cmd);          return system($cmd);
 }  }
   
Line 258  sub indexDownload ($$) { Line 344  sub indexDownload ($$) {
         my ( @lines, $comicLine, $mainURL );          my ( @lines, $comicLine, $mainURL );
         my $comicIndex = "indexes/index.$comic";          my $comicIndex = "indexes/index.$comic";
   
         `wget -q $comics->{$comic}{'url'} -O $comicIndex`;      my $wget_cmd = "wget -q --referer=\"$comics->{$comic}{'url'}\" " .
                      "--user-agent=\"$USER_AGENT\" " .
                      "$comics->{$comic}{'url'} -O $comicIndex";
       system($wget_cmd);
   
         if ( ! open FILEN, "<$comicIndex" ) {            if ( ! open FILEN, "<$comicIndex" ) {  
                 return "ERROR: Can't open index file for " . $comics->{$comic}{'fullName'} .                   return "ERROR: Can't open index file for " . $comics->{$comic}{'fullName'} . 
                        " (" . $comics->{$comic}{'url'} . ")";                          " (" . $comics->{$comic}{'url'} . ")"; 
         }           } 
                 @lines = <FILEN>;      while (<FILEN>) {
           my $line = $_;
           $line =~ s/\R|\ \ +|\t//g if ( $comics->{$comic}{'remove_newliens'} );
           push @lines, $line;
       }
         close (FILEN);            close (FILEN);  
   
         unlink ("$comicIndex");          unlink ("$comicIndex");
Line 278  sub indexDownload ($$) { Line 371  sub indexDownload ($$) {
         ## Find the comic strip URL based on the specified regex in the search          ## Find the comic strip URL based on the specified regex in the search
         ##          ##
         foreach my $line (@lines) {          foreach my $line (@lines) {
                 if ( $line =~ m/$comics->{$comic}{'search'}/ ) {                  if ( $line =~ m/$comics->{$comic}{'search'}/i ) {
                         $comicLine = $1; chomp $comicLine;                          $comicLine = $1; chomp $comicLine;
                 }                  }
         }      }
   
         ##          ##
         ## Save the file to the appropriate directory          ## Save the file to the appropriate directory
Line 292  sub indexDownload ($$) { Line 385  sub indexDownload ($$) {
         if ( $comicLine ) {          if ( $comicLine ) {
                 if ( $comicLine =~ m/(gif|jpg|png)/i ) { $comics->{$comic}{'ext'} = $1; }                  if ( $comicLine =~ m/(gif|jpg|png)/i ) { $comics->{$comic}{'ext'} = $1; }
                 my $comicURL = ( $comicLine =~ m/http/ ) ? $comicLine : $mainURL . $comicLine;                  my $comicURL = ( $comicLine =~ m/http/ ) ? $comicLine : $mainURL . $comicLine;
                 my $cmd = "wget --user-agent=\"$USER_AGENT\" --referer=\"" . $comics->{$comic}{'url'} . "\" -q $comicURL -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg";                  my $cmd = "wget --user-agent=\"$USER_AGENT\" --referer=\"" . $comics->{$comic}{'url'} . "\" -q $comicURL -O images/$cDir/$comic-$cDate.$comics->{$comic}{'ext'}";
                 system( $cmd );                  system( $cmd );
                 return 0;                  return 0;
         }          }
Line 332  sub fetchDates () { Line 425  sub fetchDates () {
         $dates{'day2'}  = ( $dates{'day'} < 10 ) ? "0" . $dates{'day'} : $dates{'day'};           $dates{'day2'}  = ( $dates{'day'} < 10 ) ? "0" . $dates{'day'} : $dates{'day'}; 
         $dates{'mon'}++;          $dates{'mon'}++;
         $dates{'mon2'}  = ( $dates{'mon'} < 10 ) ? "0".$dates{'mon'} : $dates{'mon'};          $dates{'mon2'}  = ( $dates{'mon'} < 10 ) ? "0".$dates{'mon'} : $dates{'mon'};
       my @days = qw/ Sunday Monday Tuesday Wednesday Thursday Friday Saturday /;
       $dates{'wday'} = $days[$dates{'dow'}];
   
         return %dates;          return %dates;
 }  }
Removed from v.1.13  
changed lines
  Added in v.1.23

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>