# Generate RSS and thread from a feed description file. # # This module generates RSS feeds and thread indexes of newly-published pages # or change notes for a web site maintained with App::DocKnot::Spin. # # SPDX-License-Identifier: MIT ############################################################################## # Modules and declarations ############################################################################## package App::DocKnot::Spin::RSS 7.00; use 5.024; use autodie; use parent qw(App::DocKnot); use warnings FATAL => 'utf8'; use App::DocKnot::Spin::Thread; use App::DocKnot::Util qw(print_checked print_fh); use Carp qw(croak); use Date::Language (); use Date::Parse qw(str2time); use Path::Tiny qw(path); use POSIX qw(strftime); ############################################################################## # Utility functions ############################################################################## # Escapes &, <, and > characters for HTML or XML output. # # $string - Input string # # Returns: Escaped string sub _escape { my ($string) = @_; $string =~ s{ & }{&}xmsg; $string =~ s{ < }{<}xmsg; $string =~ s{ > }{>}xmsg; return $string; } # List intersection. # # $one - First list # $two - Second list # # Returns: Common elements of both lists as a list sub _intersect { my ($one, $two) = @_; my %one = map { $_ => 1 } $one->@*; return grep { $one{$_} } $two->@*; } # Construct an absolute URL from a relative URL and a base URL. This plays # fairly fast and loose with schemes and the like, since we don't need to be # precise for our purposes. # # $url - Relative URL # $base - Base URL to which it is relative # # Returns: Absolute URL sub _absolute_url { my ($url, $base) = @_; # If $url is already absolute, return it. return $url if $url =~ m{ \A [[:lower:]]+ : }xms; # If $url starts with /, take only the scheme and host from the base URL. if ($url =~ m{ \A / }xms) { $base =~ s{ \A ( [[:lower:]]+ :// [^/]+ ) .* }{$1}xms; return $base . $url; } # Otherwise, strip the last component off the base URL, and then strip # more trailing components off the base URL for every ../ element in the # relative URL. Then glue them together. This does not deal with the # case where there are more ../ elements than there are elements in the # base URL. $base =~ s{ [^/]+ \z }{}xms; while ($url =~ s{ \A [.][.]/+ }{}xms) { $base =~ s{ [^/]+ /+ \z }{}xms; } return $base . $url; } # Construct a relative URL from an absolute URL and a base URL. If there is # no base URL or if the URLs cannot be made relative to each other, return the # relative URL unchanged. # # $url - Absolute URL # $base - URL to which it should be relative # # Returns: Relative URL sub _relative_url { my ($url, $base) = @_; return $url if !$base; # Remove the protocol and host portion from the base URL and ensure that # portion matches. if ($base =~ s{ \A ( https?:// [^/]+ ) /* }{}xms) { my $host = $1; if ($url !~ s{ \A \Q$host\E /* }{}xms) { return $url; } } else { return $url; } # Split the base URL into path segments. While the input URL starts with # a matching segment, remove it. When we run out of matching segments, # the relative URL is a number of ../ strings equal to the number of # remaining base segments, plus the remaining input URL. my @base = split(m{ /+ }xms, $base); while ($url && @base) { my $segment = shift(@base); if ($url !~ s{ \A \Q$segment\E (?: /+ | \z ) }{}xms) { return ('../' x (scalar(@base) + 1)) . $url; } } return ('../' x scalar(@base)) . $url; } # Spin a file into HTML. # # $file - Path::Tiny path to the file # # Returns: Rendered HTML as a list with one element per line sub _spin_file { my ($self, $file) = @_; my $source = $file->slurp_utf8(); my $page = $self->{spin}->spin_thread($source, $file); return map { "$_\n" } split(m{ \n }xms, $page); } # Report an action to standard output. # # $action - String description of the action # $output - Output file generated # $base - Base path for all output sub _report_action { my ($self, $action, $output) = @_; my $shortout = $output->relative($self->{base} // path(q{.})); print_checked("$action .../$shortout\n"); return; } ############################################################################## # Parsing ############################################################################## # Read key/value blocks in an RFC-2822-style file. # # $file - File to read # # Returns: List of hashes corresponding to the blocks in the file. sub _read_rfc2822_file { my ($self, $file) = @_; my $key; my @blocks = ({}); my $current = $blocks[0]; # Parse the file. $key holds the last key seen, used to append # continuation values to the previous key. $current holds the current # block being parsed and @blocks all blocks seen so far. my $fh = $file->openr_utf8(); while (defined(my $line = <$fh>)) { if ($line =~ m{ \A \s* \z }xms) { if ($key) { $current = {}; push(@blocks, $current); undef $key; } } elsif ($line =~ m{ \A (\S+): [ \t]+ ([^\n]+) \Z }xms) { my ($new_key, $value) = ($1, $2); $value =~ s{ \s+ \z }{}xms; $key = lc($new_key); $current->{$key} = $value; } elsif ($line =~ m{ \A (\S+): \s* \z }xms) { my $new_key = $1; $key = lc($new_key); $current->{$key} //= q{}; } elsif ($line =~ m{ \A \s }xms) { if (!$key) { die "$file:$.: invalid continuation line\n"; } my $value = $line; $value =~ s{ \A \s }{}xms; if ($value =~ m{ \A [.] \s* \Z }xms) { $value = "\n"; } if ($current->{$key} && $current->{$key} !~ m{ \n\z }xms) { $current->{$key} .= "\n"; } $current->{$key} .= $value; } else { die "$file:$.: cannot parse line\n"; } } close($fh); # If the file ends in a blank line, we'll have a stray empty block. # Remove it. if (!$key) { pop(@blocks); } # Return the parsed blocks. return \@blocks; } # Parse a change file. Save the metadata into the provided hash reference and # the changes into the provided array reference. Each element of the array # will be a hash with keys title, date, link, and description. # # $file - Path::Tiny path to file to read # # Returns: List of reference to metadata hash and reference to a list of # hashes of changes sub _parse_changes { my ($self, $file) = @_; my $blocks_ref = $self->_read_rfc2822_file($file); # The first block is our metadata. recent defaults to 15. my $metadata_ref = shift($blocks_ref->@*); if (!defined($metadata_ref->{recent})) { $metadata_ref->{recent} = 15; } # Canonicalize the data for the rest of the blocks, and check for # duplicate GUIDs. my %guids; my $base = $metadata_ref->{base}; for my $block_ref ($blocks_ref->@*) { $block_ref->{date} = str2time($block_ref->{date}) or die qq{cannot parse date "$block_ref->{date}"\n}; # Relative links are relative to the base URL in the metadata. if ($block_ref->{link} && $base) { if ($block_ref->{link} eq q{/}) { $block_ref->{link} = $base; } else { $block_ref->{link} = $base . $block_ref->{link}; } } # If no GUID was given, take it from the link for journal and review # entries, and otherwise from the date. Then ensure it's unique. my $guid = $block_ref->{guid}; if (!$guid) { if ($block_ref->{journal} || $block_ref->{review}) { $guid = $block_ref->{link}; } else { $guid = $block_ref->{date}; } } if ($guids{$guid}) { die "duplicate GUID for entry $guid\n"; } $block_ref->{guid} = $guid; # Determine the tags. my @tags = $block_ref->{tags} ? split(q{ }, $block_ref->{tags}) : (); if ($block_ref->{review}) { push(@tags, 'review'); } $block_ref->{tags} = \@tags; } # Return the results. return ($metadata_ref, $blocks_ref); } ############################################################################## # RSS output ############################################################################## # Format a journal post into HTML for inclusion in an RSS feed. This depends # heavily on my personal layout for journal posts. # # $file - Path::Tiny path to the journal post # # Returns: HTML suitable for including in an RSS feed sub _rss_journal { my ($self, $file) = @_; my @page = $self->_spin_file($file); # Remove the parts that don't go into the RSS feed. while (@page and $page[0] !~ m{
}xms) { pop(@page); } pop(@page); while (@page and $page[-1] =~ m{ \A \s* \z }xms) { pop(@page); } # Return the rest. return join(q{}, @page) . "\n"; } # Format a review into HTML for inclusion in an RSS feed. This depends even # more heavily on my personal layout for review posts. # # $file - Path::Tiny path to the review # # Returns: HTML suitable for inclusion in an RSS feed sub _rss_review { my ($self, $file, $metadata) = @_; my @page = $self->_spin_file($file); # Find the title and author because we'll add them back in laater, and # remove the preamble of the page not included in the RSS feed. my ($title, $author); while (@page && $page[0] !~ m{
}xmsg; $page =~ s{ |
}xms; $page =~ s{ (.*?) }{$1}xmsg; # Add the author and title to the top of the HTML because we stripped out # the top-level heading where this would normally have been. $page = "
Review: $title, $author
\n\n" . $page; # Return the cleaned-up page. return $page . "\n"; } # Print out the RSS version of the changes information given. Lots of this is # hard-coded. Use the date of the last change as}xms; $description =~ s{ \n* \z }{
\n}xms; } elsif ($entry_ref->{journal}) { my $path = path($entry_ref->{journal})->absolute($base); $description = $self->_rss_journal($path); } elsif ($entry_ref->{review}) { my $path = path($entry_ref->{review})->absolute($base); $description = $self->_rss_review($path); } # Make all relative URLs absolute. $description =~ s{ ( < (?:a [ ] href | img [ ] src) = \" ) (?!http:) ( [./\w] [^\"]+ ) \" }{ $1 . _absolute_url($2, $entry_ref->{link}) . qq{\"} }xmsge; # Convert this into an object suitable for the output template. #<<< my $formatted_ref = { date => $date, description => $description, guid => $entry_ref->{guid}, link => $entry_ref->{link}, title => $entry_ref->{title}, }; #>>> push(@formatted_entries, $formatted_ref); } # Generate the RSS output using the template. #<<< my %vars = ( base => $metadata_ref->{base}, description => $metadata_ref->{description}, docknot_version => $App::DocKnot::VERSION, entries => \@formatted_entries, language => $metadata_ref->{language}, latest => $latest, now => $now, title => $metadata_ref->{title}, url => $url, ); #>>> my $result; $self->{template}->process($self->{templates}{rss}, \%vars, \$result) or croak($self->{template}->error()); # Write the result to the output file. $file->spew_utf8($result); return; } ############################################################################## # Thread output ############################################################################## # Print out the thread version of the recent changes list. # # $file - Path::Tiny output path # $metadata_ref - RSS feed metadata # $entries_ref - Entries sub _thread_output { my ($self, $file, $metadata_ref, $entries_ref) = @_; # The entries are in a flat list, but we want a two-level list of entries # by month so that the template can add appropriate month headings. # Restructure the entry list accordingly. my (@entries_by_month, $last_month); for my $entry_ref ($entries_ref->@*) { my $month = strftime('%B %Y', localtime($entry_ref->{date})); my $date = strftime('%Y-%m-%d', localtime($entry_ref->{date})); # Copy the entry with a reformatted description. my $description = $entry_ref->{description}; $description =~ s{ ^ }{ }xmsg; $description =~ s{ \\ }{\\\\}xmsg; #<<< my $formatted_ref = { date => $date, description => $description, link => $entry_ref->{link}, title => $entry_ref->{title}, }; #<<< # Add the entry to the appropriate month. if (!$last_month || $month ne $last_month) { my $month_ref = { heading => $month, entries => [$formatted_ref] }; push(@entries_by_month, $month_ref); $last_month = $month; } else { push($entries_by_month[-1]{entries}->@*, $formatted_ref); } } # Generate the RSS output using the template. #<<< my %vars = ( prefix => $metadata_ref->{'thread-prefix'}, entries => \@entries_by_month, ); #>>> my $result; $self->{template}->process($self->{templates}{changes}, \%vars, \$result) or croak($self->{template}->error()); # Write the result to the output file. $file->spew_utf8($result); return; } ############################################################################## # Index output ############################################################################## # Translate the thread of a journal entry for inclusion in an index page. # # $file - Path::Tiny to the journal entry # # Returns: Thread to include in the index page sub _index_journal { my ($self, $file, $url) = @_; my $fh = $file->openr_utf8(); # Skip to the first \h1 and exclude it. while (defined(my $line = <$fh>)) { last if $line =~ m{ \\h1 }xms; } # Skip an initial blank line. my $text = <$fh>; $text =~ s{ \A \s* \z}{}xms; # Grab the rest of the entry until the \date command that ends it. while (defined(my $line = <$fh>)) { last if $line =~ m{ \A \\date }xms; $text .= $line; } # All done. close($fh); return $text; } # Translate the thread of a book review for inclusion into an index page. # # $file - Path::Tiny to the book review # # Returns: Thread to include in the index page sub _index_review { my ($self, $file) = @_; my $title; my $author; # Regex to match a single "character" in a macro argument. my $char = qr{ (?: [^\]\\] | \\entity \[ [^\]]+ \] ) }xms; # Scan for the author information and save it. Handle the case where the # \header or \edited line is continued on the next line. my $fh = $file->openr_utf8(); while (defined(my $line = <$fh>)) { if ($line =~ m{ \\ (?:header|edited) \s* \[ $char+ \] \s* \z }xms) { $line .= <$fh>; } if ($line =~ m{ \\(header|edited)\s*\[($char+)\]\s*\[($char+)\] }xms) { ($title, $author) = ($2, $3); if ($1 eq 'edited') { $author .= ' (ed.)'; } last; } } if (!defined($author)) { die "cannot find author in review $file\n"; } # Add the prefix saying what's being reviewed. my $text; if ($file =~ m{ /magazines/ }xms) { $text = "Review: \\cite[$title], $author\n\n"; } else { $text = "Review: \\cite[$title], by $author\n\n"; } # Add the metadata table. $text .= "\\table[][\n"; while (defined(my $line = <$fh>)) { last if $line =~ m{ \A \\div [(]review[)] \[ }xms; if ($line =~ m{ \A \s* \\data \[($char+)\] \s* \[($char+)\] }xms) { $text .= " \\tablerow[$1][$2]\n"; } } $text .= "]\n\n"; # Add the rest of the review. while (defined(my $line = <$fh>)) { last if $line =~ m{ \A \\done }xms; $line =~ s{ \\story \[ \d+ \] }{\\strong}xmsg; $line =~ s{ \\rating \s* \[($char+)\] }{Rating: $1 out of 10}xms; $text .= $line; } close($fh); return $text; } # Print out the index version of the recent changes list. # # $file - Path::Tiny path to the output file # $base - Base Path::Tiny path for input files # $metadata_ref - RSS feed metadata # $entries_ref - Entries sub _index_output { my ($self, $file, $base, $metadata_ref, $entries_ref) = @_; # Format each entry. my @formatted_entries; for my $entry_ref ($entries_ref->@*) { my @time = localtime($entry_ref->{date}); my $date = strftime('%Y-%m-%d %H:%M', @time); my $day = strftime('%Y-%m-%d', @time); # Get the text of the entry. my $text; if ($entry_ref->{journal}) { my $path = path($entry_ref->{journal})->absolute($base); $text = $self->_index_journal($path); } elsif ($entry_ref->{review}) { my $path = path($entry_ref->{review})->absolute($base); $text = $self->_index_review($path); } else { die "unknown entry type\n"; } # Make all the URLs absolute and then convert images back to relative # based on the URL of the file we're creating. This handles # correcting links from thread from elsewhere in the tree. $text =~ s{ ( \\ (?: link | image ) \s* \[ ) ( [^\]]+ ) \] }{ $1 . _absolute_url($2, $entry_ref->{link}) . ']' }xmsge; $text =~ s{ ( \\ image \s* \[ ) ( [^\]]+ ) \] }{$1 . _relative_url($2, $metadata_ref->{'index-base'}) . ']' }xmsge; # Add the entry to the list. #<<< my $formatted_ref = { date => $date, day => $day, link => $entry_ref->{link}, title => $entry_ref->{title}, text => $text, }; #>>> push(@formatted_entries, $formatted_ref); } # Generate the RSS output using the template. #<<< my %vars = ( prefix => $metadata_ref->{'index-prefix'}, suffix => $metadata_ref->{'index-suffix'}, entries => \@formatted_entries, ); #>>> my $result; $self->{template}->process($self->{templates}{index}, \%vars, \$result) or croak($self->{template}->error()); # Write the result to the output file. $file->spew_utf8($result); return; } ############################################################################## # Public interface ############################################################################## # Create a new RSS generator object. # # $args_ref - Anonymous hash of arguments with the following keys: # base - Path::Tiny base path for output files # # Returns: Newly created object sub new { my ($class, $args_ref) = @_; # Create and return the object. my $base = defined($args_ref->{base}) ? path($args_ref->{base}) : undef; my $tt = Template->new({ ABSOLUTE => 1, ENCODING => 'utf8' }) or croak(Template->error()); #<<< my $self = { base => $base, spin => App::DocKnot::Spin::Thread->new(), template => $tt, }; bless($self, $class); $self->{templates} = { changes => $self->appdata_path('templates', 'changes.tmpl'), index => $self->appdata_path('templates', 'index.tmpl'), rss => $self->appdata_path('templates', 'rss.tmpl'), }; #>>> return $self; } # Generate specified output files from an .rss input file. # # $source - Path::Tiny path to the .rss file # $base - Optional Path::Tiny base path for output sub generate { my ($self, $source, $base) = @_; $source = path($source); $base //= $self->{base}; $base = defined($base) ? path($base) : path(q{.}); # Read in the changes. my ($metadata_ref, $changes_ref) = $self->_parse_changes($source); # The output key tells us what files to write out. my @output = ('*:rss:index.rss'); if ($metadata_ref->{output}) { @output = split(q{ }, $metadata_ref->{output}); } # Iterate through each specified output file. for my $output (@output) { my ($tags, $format, $file) = split(m{ : }xms, $output); $file = path($file); if ($file->is_relative()) { $file = $file->absolute($base); } # If the output file is newer than the input file, do nothing. next if ($file->exists() && -M "$file" <= -M "$source"); # Find all the changes of interest to this output file. my @entries; if ($tags eq q{*}) { @entries = $changes_ref->@*; } else { my @tags = split(m{ , }xms, $tags); @entries = grep { _intersect($_->{tags}, \@tags) } $changes_ref->@*; } # Write the output. if ($format eq 'thread') { $self->_report_action('Generating thread file', $file); $self->_thread_output($file, $metadata_ref, \@entries); } elsif ($format eq 'rss') { if (scalar(@entries) > $metadata_ref->{recent}) { splice(@entries, $metadata_ref->{recent}); } $self->_report_action('Generating RSS file', $file); $self->_rss_output($file, $base, $metadata_ref, \@entries); } elsif ($format eq 'index') { if (scalar(@entries) > $metadata_ref->{recent}) { splice(@entries, $metadata_ref->{recent}); } $self->_report_action('Generating index file', $file); my $index_base = $source->parent(); $self->_index_output($file, $index_base, $metadata_ref, \@entries); } } return; } ############################################################################## # Module return value and documentation ############################################################################## 1; __END__ =for stopwords Allbery DocKnot MERCHANTABILITY NONINFRINGEMENT RSS TimeDate YYYY-MM-DD sublicense hoc rss =head1 NAME App::DocKnot::Spin::RSS - Generate RSS and thread from a feed description file =head1 SYNOPSIS use App::DocKnot::Spin::RSS; my $rss = App::DocKnot::Spin::RSS->new({ base => 'path/to/tree' }); $rss->generate('path/to/tree/.rss'); =head1 REQUIREMENTS Perl 5.24 or later and the modules Date::Language, Date::Parse (both part of the TimeDate distribution), List::SomeUtils, Path::Tiny, and Perl6::Slurp, both of which are available from CPAN. =head1 DESCRIPTION App::DocKnot::Spin::RSS reads as input a feed description file consisting of simple key/value pairs and writes out either thread (for input to App::DocKnot::Spin::Thread) or RSS. The feed description consists of a leading block of metadata and then one block per entry in the feed. Each block can either include the content of the entry or can reference an external thread file, in several formats, for the content. The feed description file defines one or more output files in the Output field of the metadata. Output files are only regenerated if they are older than the input feed description file. App::DocKnot::Spin::RSS is designed for use with App::DocKnot::Spin. It relies on App::DocKnot::Spin::Thread to convert thread to HTML, both for inclusion in RSS feeds and for post-processing of generated thread files. App::DocKnot::Spin::RSS is invoked automatically by App::DocKnot::Spin when it encounters an F<.rss> file in a directory it is processing. See L for the details of the language in which F<.rss> files are written. =head1 CLASS METHODS =over 4 =item new(ARGS) Create a new App::DocKnot::Spin::RSS object. ARGS should be a hash reference with one or more of the following keys, all of which are optional: =over 4 =item base By default, App::DocKnot::Spin::RSS output files are relative to the current working directory. If the C