--- comics/fetch.pl.new 2012/12/20 16:48:06 1.6 +++ comics/fetch.pl.new 2017/12/05 13:37:40 1.22 @@ -1,23 +1,57 @@ #!/usr/bin/perl -w +############################################################################### +# $Log: fetch.pl.new,v $ +# Revision 1.22 2017/12/05 13:37:40 nick +# Added the CVS config version to the outpuit. +# +# Revision 1.21 2015/10/26 14:25:40 nick +# Fixed a bug that was improperly including the day of week string preventing the weekend comics from fetching proproperly. +# +# Revision 1.20 2015/10/22 12:58:44 nick +# Added the ability for Sunday only comics. Stonesoup is no longer weekdays, this has been added to Sunday only. I also added Foxtrot Classics for weekdays and Foxtrot for Sundays. +# +# Revision 1.19 2015/07/13 12:56:58 nick +# Added Sally Forth and Pearls Before Swine. Adding Sally Forth required a change in the 'wget' command for fetching the index file to include 'user-agent' and 'referer'. +# +# Revision 1.18 2015/05/07 12:31:43 nick +# Added favicon +# +# Revision 1.17 2015/02/19 14:56:10 nick +# Fixed a problem that forced everything to JPG. This would kill GIF animations, but would not display the gifs either because 'convert' appends an index number to the end of the file name for each from of the GIF animation. I fixed this to maintain GIF compatibilty as well as rewritting how the script fetches the size of the file. Additionally, I updated the configuration for Questionable Content to search for GIF or JPG, which is what triggered this entire update. +# +# Revision 1.16 2015/02/05 18:05:58 nick +# Changed the background and added a fancy title. +# +# Revision 1.15 2015/01/19 13:46:19 nick +# *** empty log message *** +# +############################################################################### + use strict; use File::Path; use Data::Dumper; +use Pod::Usage; +use Getopt::Long; + +use Date::Calc qw/Date_to_Text_Long Today Day_of_Week Day_of_Week_to_Text/; ## ## Some default values ## -my $ver = q/$Id: fetch.pl.new,v 1.6 2012/12/20 16:48:06 nick Exp $/; +my $ver = '$Id: fetch.pl.new,v 1.22 2017/12/05 13:37:40 nick Exp $'; my $comicFile = "comics.conf"; +my $comicConfigVer = "Unknown"; my %comics = &readComicConfig ( $comicFile ); -my $days_ago = 0; # Set this to fetch comics from X days ago +my %opts = &fetchOptions( ); +my $days_ago = $opts{'days'} || 0; my %dates = &fetchDates(); my $baseDir = $comics{'configs'}{'base_directory'} || "."; my $imageDir = $baseDir . "/" . ( $comics{'configs'}{'image_directory'} || "images" ) . "/$dates{'mon2'}$dates{'year2'}"; my $indexDir = $baseDir . "/" . ( $comics{'configs'}{'index_directory'} || "indexes" ); my $USER_AGENT = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.18) Gecko/20110628 Ubuntu/10.10 (maverick) Firefox/3.6.18"; - +my @days = qw/ Sunday Monday Tuesday Wednesday Thursday Friday Saturday /; my $DATE=`date`; chomp $DATE; print STDOUT "Starting comic fetch at $DATE\n"; @@ -30,17 +64,37 @@ print STDOUT "Starting comic fetch at $D &writeTitle ( \%dates ); foreach my $comic ( sort keys %comics ) { + + ## Skip if this is Sunday and the comic is weekdays only next if ( $comic =~ m/config/ ); + if (($dates{'wday'} eq "Sunday") && + ($comics{$comic}{'sunday'} == 0)) { + print "Skipping '$comic'; Weekdays only.\n"; + next; + } + + ## Skip if Sunday only comic and it's not Sunday. + if (($dates{'wday'} ne "Sunday") && + ($comics{$comic}{'sunday_only'} == 1)) { + print "Skipping '$comic' ($comics{$comic}{'sunday_only'}); Sunday only.\n"; + next + } + $comics{$comic}{'error'} = &downloadComic ( \%comics, $comic, \%dates ); &writeComic ( \%comics, $comic, \%dates ); -} -my $D = `date +%d`; chomp $D; -print "Finding in $imageDir/*-$D.jpg\n"; -foreach my $file ( glob( "$imageDir/*-$D.jpg" ) ) -{ - my $size = `/usr/bin/identify $file`; - $size =~ s/.*\s(\d+)x\d+.*/$1/; + my $file = "$imageDir/$comic-$dates{'day2'}.$comics{$comic}{'ext'}"; + my $size = 0; + + my $cmd = "/usr/bin/identify -verbose $file|"; + open(IMG, $cmd) || die ("Can't open: $!\n"); + while() { + if ($_ =~ m/^\s+geometry:\s+(\d+)x\d+.*/i) { + $size = $1 if ( $size == 0); + } + } + close(IMG); + system( "/usr/bin/convert -resize 640 $file $file" ) if ( $size > 640 ) @@ -89,15 +143,29 @@ sub readComicConfig ($$) { my %comicConfig = ( ); my %config = ( ); + my ($year, $mon, $day) =( localtime(time))[5,4,3]; + $year += 1900; + $mon = sprintf("%02d", ($mon + 1)); + $day = sprintf("%02d", $day); + open FILEN, "<$comicFile"; while () { + if ($_ =~ m/^#.* \$Id: fetch.pl.new,v 1.22 2017/12/05 13:37:40 nick Exp $/) { + $comicConfigVer = $1; + } if ( ( $_ !~ m/^#/ ) && ( $_ =~ m/,.*,/) ){ + $_ =~ s/__YEAR__/$year/g; + $_ =~ s/__MON__/$mon/g; + $_ =~ s/__DAY__/$day/g; + my @res = split /,/, $_; $comicConfig{$res[0]}{'url'} = $res[1]; $comicConfig{$res[0]}{'search'} = $res[2]; $comicConfig{$res[0]}{'mode'} = $res[3]; $comicConfig{$res[0]}{'fullName'} = $res[4]; $comicConfig{$res[0]}{'ext'} = $res[5]; + $comicConfig{$res[0]}{'sunday'} = sprintf("%d", $res[6] || 1); + $comicConfig{$res[0]}{'sunday_only'} = sprintf("%d", $res[7] || 0); $comicConfig{$res[0]}{'error'} = 0; } elsif ( $_ =~ m/(.*)\s+=\s+(.*)/ ) { @@ -113,8 +181,10 @@ sub readComicConfig ($$) { ####################################################################### sub writeComic ($$) { my ( $comics, $comic, $date ) = @_; - my $indexFile = $indexDir . "/index-" . $date->{'year2'} . $date->{'mon2'} . - $date->{'day2'} . ".html"; + my $sd = substr( join( '', $days[$date->{'dow'}] ), 0, 3 ); + my $indexFile = $indexDir . "/index-" . $date->{'year2'} . + $date->{'mon2'} . $date->{'day2'} . "-" . + $sd . ".html"; my $content = <{$comic}{'fullName'}) ******* --> @@ -126,7 +196,7 @@ sub writeComic ($$) { $comics->{$comic}{'url'}
-$comic-$date->{'day2'} +$comic-$date->{'day2'}

@@ -167,8 +237,10 @@ sub writeMainIndex ($$) { ####################################################################### sub writeFooter { my ( $date ) = @_; - my $indexFile = $indexDir . "/index-" . $date->{'year2'} . $date->{'mon2'} . - $date->{'day2'} . ".html"; + my $sd = substr( join( '', $days[$date->{'dow'}] ), 0, 3 ); + my $indexFile = $indexDir . "/index-" . $date->{'year2'} . + $date->{'mon2'} . $date->{'day2'} . "-" . + $sd . ".html"; my $sysDate = `date`; open INDEX, ">>$indexFile"; @@ -177,7 +249,9 @@ sub writeFooter {
Generated on: $sysDate
-Version: $ver +Version: $ver
+Config Version: $comicConfigVer
+CVS: http://demandred.dyndns.org/cgi-bin/cvsweb/comics/

Valid XHTML 1.0 Transitional @@ -204,9 +278,12 @@ sub checkDir ($$) { ####################################################################### sub writeTitle ($$) { my ( $date ) = @_; - my $indexFile = $indexDir . "/index-" . $date->{'year2'} . $date->{'mon2'} . - $date->{'day2'} . ".html"; - my $today = $date->{'mon'} . "/" . $date->{'day'} . "/" . $date->{'year'}; + my $sd = substr( join( '', $days[$date->{'dow'}] ), 0, 3 ); + my $indexFile = $indexDir . "/index-" . $date->{'year2'} . + $date->{'mon2'} . $date->{'day2'} . "-" . + $sd . ".html"; + my $today = $days[$date->{'dow'}] . " " . $date->{'mon'} . "/" . $date->{'day'} . "/" . $date->{'year'}; + my $today_long = Date_to_Text_Long(Today()); open INDEX, ">$indexFile"; print INDEX < + + Daily Comics for $today -

Daily Comics for $today

+
+ + + + + + EOF close (INDEX); } @@ -236,7 +321,8 @@ sub directDownload ($$) { my $cDir = $date->{'mon2'} . $date->{'year2'}; my $cDate = $date->{'day2'}; - my $cmd = "wget -q $file --referer=\"" . $comics->{$comic}{'url'} ."\" --user-agent=\"$USER_AGENT\" -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg"; + my $cmd = "wget -q $file --referer=\"" . $comics->{$comic}{'url'} ."\" --user-agent=\"$USER_AGENT\" -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg"; + return system($cmd); } @@ -247,7 +333,10 @@ sub indexDownload ($$) { my ( @lines, $comicLine, $mainURL ); my $comicIndex = "indexes/index.$comic"; - `wget -q $comics->{$comic}{'url'} -O $comicIndex`; + my $wget_cmd = "wget -q --referer=\"$comics->{$comic}{'url'}\" " . + "--user-agent=\"$USER_AGENT\" " . + "$comics->{$comic}{'url'} -O $comicIndex"; + system($wget_cmd); if ( ! open FILEN, "<$comicIndex" ) { return "ERROR: Can't open index file for " . $comics->{$comic}{'fullName'} . @@ -267,10 +356,10 @@ sub indexDownload ($$) { ## Find the comic strip URL based on the specified regex in the search ## foreach my $line (@lines) { - if ( $line =~ m/$comics->{$comic}{'search'}/ ) { + if ( $line =~ m/$comics->{$comic}{'search'}/i ) { $comicLine = $1; chomp $comicLine; } - } + } ## ## Save the file to the appropriate directory @@ -281,7 +370,7 @@ sub indexDownload ($$) { if ( $comicLine ) { if ( $comicLine =~ m/(gif|jpg|png)/i ) { $comics->{$comic}{'ext'} = $1; } my $comicURL = ( $comicLine =~ m/http/ ) ? $comicLine : $mainURL . $comicLine; - my $cmd = "wget --user-agent=\"$USER_AGENT\" --referer=\"" . $comics->{$comic}{'url'} . "\" -q $comicURL -O - | /usr/bin/convert - jpeg:images/$cDir/$comic-$cDate.jpg"; + my $cmd = "wget --user-agent=\"$USER_AGENT\" --referer=\"" . $comics->{$comic}{'url'} . "\" -q $comicURL -O images/$cDir/$comic-$cDate.$comics->{$comic}{'ext'}"; system( $cmd ); return 0; } @@ -314,16 +403,71 @@ sub parseComic ($$) { sub fetchDates () { my %dates = (); - ($dates{'day'}, $dates{'mon'}, $dates{'year'}, $dates{'dow'}) = (localtime)[3,4,5,6]; - - ## If you missed a day or two, reflect it here: - $dates{'day'} -= $days_ago; ## <-- 5 days ago + ($dates{'day'}, $dates{'mon'}, $dates{'year'}, $dates{'dow'}) = (localtime(time - (86400 * $days_ago )))[3,4,5,6]; $dates{'year'} += 1900; $dates{'year2'} = substr $dates{'year'}, 2, 2; $dates{'day2'} = ( $dates{'day'} < 10 ) ? "0" . $dates{'day'} : $dates{'day'}; $dates{'mon'}++; $dates{'mon2'} = ( $dates{'mon'} < 10 ) ? "0".$dates{'mon'} : $dates{'mon'}; + my @days = qw/ Sunday Monday Tuesday Wednesday Thursday Friday Saturday /; + $dates{'wday'} = $days[$dates{'dow'}]; return %dates; } + +############################################################################### +## +## &fetchOptions( ); +## +## Grab our command line arguments and toss them in to a hash +## +############################################################################### +sub fetchOptions { + my %opts; + + &GetOptions( + "days:i" => \$opts{'days'}, + "help|?" => \$opts{'help'}, + "man" => \$opts{'man'}, + ) || &pod2usage( ); + &pod2usage( ) if defined $opts{'help'}; + &pod2usage( { -verbose => 2, -input => \*DATA } ) if defined $opts{'man'}; + + return %opts; +} + +__END__ + +=head1 NAME + +fetch.pl - Fetches comics and places them all locally in a single html file. + +=head1 SYNOPSIS + +fetch.pl [options] + +Options: + --days,d Fetch comics from X days ago + --help,? Display the basic help menu + --man,m Display the detailed man page + +=head1 DESCRIPTION + +=head1 HISTORY + +=head1 AUTHOR + +Nicholas DeClario + +=head1 BUGS + +This is a work in progress. Please report all bugs to the author. + +=head1 SEE ALSO + +=head1 COPYRIGHT + +=cut + +
$today_long