#!/usr/bin/perl -w

=pod

=head1 NAME

tv_grab_za_dstv - Grab TV listings for South Africa, from dstv.com.

=head1 SYNOPSIS

tv_grab_za_dstv --help

tv_grab_za_dstv [--config-file FILE] --configure [--gui OPTION]

tv_grab_za_dstv [--config-file FILE] [--output FILE] [--days N]
           [--quiet] [--cacheage N]

tv_grab_za_dstv --list-channels

=head1 DESCRIPTION

Output TV listings for several channels available in South Africa.
The data comes from www.dstv.com which is the company providing 
satellite TV in South Africa.  The grabber relies on parsing HTML so 
it might stop working at any time.

First run B<tv_grab_za_dstv --configure> to choose, which channels you want
to download. Then running B<tv_grab_za_dstv> with no arguments will output
listings in XML format to standard output.

B<--configure> Prompt for which channels,
and write the configuration file.

B<--config-file FILE> Set the name of the configuration file, the
default is B<~/.xmltv/tv_grab_za_dstv.conf>.  This is the file written by
B<--configure> and read when grabbing.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of XMLTV::ProgressBar.

B<--output FILE> write to FILE rather than standard output.

B<--days N> grab N days.  The default is eight.  
The accepted values here are 1, 8, 15 or 31.

B<--quiet> suppress the progress messages normally written to standard
error.

B<--cacheage N> will try and cache downloads, and reuse them if the file is
younger than N days old.  The default is to cache downloads for 1 day.

B<--list-channels> write output giving <channel> elements for every
channel available (ignoring the config file), but no programmes.

B<--help> print a help message and exit.

=head1 SEE ALSO

L<xmltv(5)>.

=head1 AUTHOR

Norman Rasmussen, norman@rasmussen.org.  
Based on tv_grab_fi by Matti Airas, mairas@iki.fi.  
tv_grab_fi originally based on tv_grab_sn by Stefan G:orling.

=head1 BUGS

The granularity of the days option is dependant on the website and the
channels are identified by satellite number rather than the RFC2838 form
recommended by the XMLTV DTD.  The last programme for each channel
does not get a stop date.

=cut

######################################################################
# initializations

use strict;
use XMLTV::Version '$Id: tv_grab_za_dstv,v 1.01 2004/12/21 03:15:39 normanr Exp $ ';
use Getopt::Long;
use Date::Manip;
use HTML::TreeBuilder;
use HTML::Entities; # parse entities
use IO::File;
use HTTP::Cookies;

use XMLTV;
use XMLTV::Memoize;
use XMLTV::ProgressBar;
use XMLTV::Ask;
use XMLTV::Config_file;
use XMLTV::Get_nice;
use XMLTV::Mode;
use XMLTV::Date;
use XMLTV::Usage <<END
$0: get South African television listings from www.dstv.com in XMLTV format
To configure: $0 --configure [--config-file FILE]
To grab listings: $0 [--config-file FILE] [--output FILE] [--days N]
        [--quiet] [--cacheage N]
To list channels: $0 --list-channels
END
  ;

# Attributes of the root element in output.
my $HEAD = { 'source-info-url' => 'http://www.dstv.com/',
	     'source-data-url'     => 'http://www.dstv.com/dstv-guide/',
	     'generator-info-name' => 'XMLTV',
	     'generator-info-url'  => 'http://membled.com/work/apps/xmltv/',
	   };

# default language
my $LANG="en-ZA";

# Global channel data.
our @ch_all;
our %ch_ids;

######################################################################
# get options

# Get options, including undocumented --cache option.
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
my ($opt_days, $opt_help, $opt_output, $opt_cacheage, 
    $opt_configure, $opt_config_file, $opt_gui,
    $opt_quiet, $opt_list_channels);
$opt_days   = 8; # default
$opt_cacheage = 1; # default
$opt_quiet  = 0; # default
GetOptions('days=i'    => \$opt_days,
	   'cacheage=i'    => \$opt_cacheage,
	   'help'          => \$opt_help,
	   'configure'     => \$opt_configure,
	   'config-file=s' => \$opt_config_file,
	   'gui:s'         => \$opt_gui,
	   'output=s'      => \$opt_output,
	   'quiet'         => \$opt_quiet,
	   'list-channels' => \$opt_list_channels,
	  )
  or usage(0);
die 'number of days must be 1, 8, 15 or 31'
  if (defined $opt_days && ($opt_days != 1) && ($opt_days != 8) && ($opt_days != 15) && ($opt_days != 31));
usage(1) if $opt_help;

XMLTV::Ask::init($opt_gui);

my $mode = XMLTV::Mode::mode('grab', # default
			     $opt_configure => 'configure',
			     $opt_list_channels => 'list-channels',
			    );

# File that stores which channels to download.
my $config_file
  = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_za_dstv', $opt_quiet);

my @config_lines; # used only in grab mode
if ($mode eq 'configure') {
    XMLTV::Config_file::check_no_overwrite($config_file);
}
elsif ($mode eq 'grab') {
    @config_lines = XMLTV::Config_file::read_lines($config_file);
}
elsif ($mode eq 'list-channels') {
    # Config file not used.
}
else { die }

$XMLTV::Get_nice::get = \&cached_get;

my $cookie_jar = HTTP::Cookies->new;
$cookie_jar->set_cookie(undef, 'SelectionPage', 'true',
						      '/', 'www.dstv.com', undef,
						      0, 0, (time() - $HTTP::Cookies::EPOCH_OFFSET)+86400, 0);

$LWP::Simple::ua->cookie_jar($cookie_jar);

# Whatever we are doing, we need the channels data.
my %channels = get_channels(); # sets @ch_all
my @channels;

######################################################################
# write configuration

if ($mode eq 'configure') {
    open(CONF, ">$config_file") or die "cannot write to $config_file: $!";

    # Ask about each channel.
    my @chs = sort {join('.', reverse split(/\./, $a)) cmp join('.', reverse split(/\./, $b))} keys %channels;
    my @names = map { $channels{$_} } @chs;
    my @qs = map { "add channel $_?" } @names;
    my @want = ask_many_boolean(1, @qs);
    foreach (@chs) {
	my $w = shift @want;
	warn("cannot read input, stopping channel questions"), last
	  if not defined $w;
	# No need to print to user - XMLTV::Ask is verbose enough.

	# Print a config line, but comment it out if channel not wanted.
	print CONF '#' if not $w;
	my $name = shift @names;
	print CONF "channel $_ $name\n";
	# TODO don't store display-name in config file.
    }

    close CONF or warn "cannot close $config_file: $!";
    say("Finished configuration.");

    exit();
}

# Not configuration, we must be writing something, either full
# listings or just channels.
#
die if $mode ne 'grab' and $mode ne 'list-channels';

# Options to be used for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    my $fh = new IO::File(">$opt_output");
    die "cannot write to $opt_output: $!" if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
#$w_args{encoding} = 'UTF-8';
$w_args{encoding} = 'ISO-8859-1';
my $writer = new XMLTV::Writer(%w_args);
$writer->start($HEAD);

if ($mode eq 'list-channels') {
    # Write channels mode.
    $writer->write_channel($_) foreach @ch_all;
    $writer->end();
    exit();
}

######################################################################
# We are producing full listings.
die if $mode ne 'grab';

# Read configuration.
my $line_num = 1;
foreach (@config_lines) {
    ++ $line_num;
    next if not defined;
    if (/^channel:?\s+(\S+)\s+([^\#]+)/) {
		my $ch_did = $1;
		my $ch_name = $2;
		$ch_name =~ s/\s*$//;
		push @channels, $ch_did;
		$channels{$ch_did} = $ch_name;
    }
    else {
		warn "$config_file:$line_num: bad line\n";
    }
}

######################################################################
# begin main program

die "No channels specified, run me with --configure\n"
  if not keys %channels;
my @to_get;

# the order in which we fetch the channels matters
foreach my $ch_did (@channels) {
    my $ch_name=$channels{$ch_did};
    my $ch_xid=$ch_did;
    $writer->write_channel({ id => $ch_xid,
			     'display-name' => [ [ $ch_name ] ] });
	push @to_get, [ undef, $ch_xid, $ch_ids{$ch_did} ];
}

# This progress bar is for both downloading and parsing.  Maybe
# they could be separate stages.
#
my $bar = new XMLTV::ProgressBar('getting listings', scalar @to_get)
  if not $opt_quiet;
foreach (@to_get) {
    foreach (process_table($_->[0], $_->[1], $_->[2])) {
		$writer->write_programme($_);
    }
    update $bar if not $opt_quiet;
}
$bar->finish() if not $opt_quiet;
$writer->end();

######################################################################
# subroutine definitions

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
	*t = sub {};
	*d = sub { '' };
    }
    else {
	*t = \&Log::TraceMessages::t;
	*d = \&Log::TraceMessages::d;
	Log::TraceMessages::check_argv();
    }
}

####
# process_table: fetch a URL and process it
#
# arguments:
#    Date::Manip object giving the day to grab
#    xmltv id of channel
#    dstv id of channel
#
# returns: list of programme hashes to write
#
sub process_table {
    my ($date, $ch_xmltv_id, $ch_dstv_id) = @_;

	my $sType = do { 
		   if	($opt_days ==  1) {0}
		elsif	($opt_days ==  8) {1}
		elsif	($opt_days == 15) {2}
		elsif	($opt_days == 31) {3}
		else 	{ die "bad number of days" } };
	
    my $url = "http://www.dstv.com/dstv-guide/default.asp?Submit=true&sType=$sType&ChannelID=$ch_dstv_id";
    t "getting URL: $url";
    my $data=get_nice($url);
    if (not defined $data) {
		die "could not fetch $url, aborting\n";
    }
    local $SIG{__WARN__} = sub {
		warn "$url: $_[0]";
    };

    # parse the page to a document object
    my $tree = HTML::TreeBuilder->new();
    $tree->parse($data);
    my @program_data = get_program_data($tree);
    
    my @r;
	while (@program_data) {
		my $cur = shift @program_data;
		my $next = shift @program_data;
		unshift @program_data,$next if $next;
		push @r, make_programme_hash($date, $ch_xmltv_id, $ch_dstv_id, $cur, $next);
    }
    return @r;
}

sub make_programme_hash {
    my ($date, $ch_xmltv_id, $ch_dstv_id, $cur, $next) = @_;
	$date = $cur->{date};
    my %prog;

    $prog{channel}=$ch_xmltv_id;
    $prog{title}=[ [ $cur->{title}, $LANG ] ];

    my $cur_time = $cur->{time};
    t 'raw time for programme: ' . d $cur_time;
    my $start=parse_date("$date $cur_time");
    t 'parse_date() returned: ' . d $start;
    $prog{start}=UnixDate($start, '%q') . " +0200";
    t 'set programme start time to: ' . d $prog{start};

    my $next_time = $next ? $next->{time} : undef;
    if (defined $next_time) {
		t '$cur_time=' . d $cur_time;
		t '$next_time=' . d $next_time;
		my $stop_date;
		if ($next_time lt $cur_time) {
		    # Must span midnight.  (Don't worry about start being
		    # summer time and stop being winter: we assume the site is
		    # sane enough to put them both in the same timezone and
		    # avoid looking like stop < start.)
		    #
		    t '$next_time appears sooner, must be next day';
		    $stop_date = nextday($date);
		}
		else {
		    $stop_date = $date;
		}
		t '$stop_date set to: ' . d $stop_date;
		my $stop = parse_date("$stop_date $next_time");
		t 'stop time in CAT: ' . d $stop;
		$prog{stop}=UnixDate($stop, '%q') . " +0200";
    }	

    # FIXME: parse description field further
    $prog{desc}=[ [ $cur->{desc}, $LANG ] ] if defined $cur->{desc};

    $prog{rating}=[ [ $cur->{rating} ] ] if defined $cur->{rating};

    return \%prog;
}

# program data is split as follows:
# - a complete day's worth of programs sits between two images.
#   each day starts with a hdr-bg-lblue.gif image, and ends with the next rslts or content image
# - td with class=srch-head has the time in bold
# - in the same b after the time is the program
# - its second sibling td with class=srch-head (two to the right)
#   that has a br as its first child is the rating
# - td two rows down with the padding (and outside the previous table) is the description
sub get_program_data {
    my $tree = shift;
    t "get_program_data() ENTRY for tree: $tree";
    my @data;

	my @date_elems = get_date_elems($tree);
    t 'date elements: ' . d \@date_elems . ' of ' . scalar @date_elems . ' elements';
      
    my $date;
    for (my $i = 0; $i < @date_elems; $i++)
    {
    	my $elem = $date_elems[$i];
    	my $table;
    	if (ref($elem) ne "") { $table = $elem->find('table'); }
    	if (defined $table) {

	    	my $tableid = $table->id();
	    	if (!defined $tableid) {next;}
	    		    	
	    	if ($tableid eq "Table4") {
	    		my $b = $table->find('b');
	    		$date = get_content($b->content_list());
	    	} elsif ($tableid eq "Table3") {
	    		my $delem = $date_elems[$i + 2];

		    	my %h;
		    
		    	$h{date} = $date;
				$h{time} = get_time_str($elem);
				$h{title} = get_name_str($elem);
				$h{rating} = get_rating_str($elem);
				$h{desc} = get_desc_str($delem);
				
				t 'got data: ' . d \%h;
				push @data, \%h;
	    	}
	    }
    }

    t 'get_program_data() RETURNING ' . d \@data;
    return @data;
}
sub get_date_elems {
    my ($tree) = @_;
    my @date_elems = $tree->look_down
      ("class"=>"srch-head-main")->parent->parent->content_list;
    return @date_elems;
}
sub get_time_str {
    my ($elem) = @_;
    my $time = get_content(($elem->find('b'))[0]->content_list);
    $time =~ s/ .+$//;
    $time = UnixDate(parse_date($time), '%R');
    return $time;
}
sub get_name_str {
    my ($elem) = @_;
    my $name = get_content(($elem->find('b'))[0]->content_list);
    $name =~ s/^.+? //;
    $name =~ s/ +$//;
    return $name;
}
sub get_rating_str {
    my ($elem) = @_;
    my $rating = get_content(($elem->find('b'))[1]->content_list);
    return $rating;
}
sub get_desc_str {
    my ($delem) = @_;
    my $desc = get_content(($delem->content_list)[0]->content_list);
    $desc =~ s/^ +//;
    $desc =~ s/ +$//;
    return $desc;
}
sub get_content {
  my (@l) = @_;
  while (@l) {
    my $e = shift @l;
    return $e if !ref($e);
  }
  return undef;
}

# get channel listing
sub get_channels {
    my $bar = new XMLTV::ProgressBar('getting list of bouquets', 1)
      if not $opt_quiet;
    my %channels;
    my $url="http://www.dstv.com/dstv-guide/default.asp";
    my $local_data=get_nice($url);
    die "could not get bouquet listing $url, aborting\n"
      if not defined $local_data;

    # FIXME commonize this
    local $SIG{__WARN__} = sub {
	warn "$url: $_[0]";
    };
    local $SIG{__DIE__} = sub {
	die "$url: $_[0]";
    };
    
    update $bar if not $opt_quiet;
    $bar->finish() if not $opt_quiet;

    my $tree = HTML::TreeBuilder->new();
    $tree->parse($local_data);

    # all bouquet elements are option tags within the select named Bouquet

    my @select_elems = $tree->look_down(_tag => 'select',
					name => 'Bouquet');
    my @ch_op_elems = map { $_->look_down(_tag => 'option') } @select_elems;

    $bar = new XMLTV::ProgressBar('getting list of channels', scalar @ch_op_elems)
      if not $opt_quiet;

    foreach my $elem (@ch_op_elems) {
		my $bouquet_id = $elem->attr('value');
	    my @children=$elem->content_list;
	    if (scalar(@children) != 1 || ref($children[0])) {
			warn "Illegal bouquet name";
			next;
	    }
	    for ($children[0]) {
			s/^\s+//; s/\s+$//;
			my $bouquet_name=$_;
			
			get_channels_for_bouquet(\%channels, $bouquet_id, $bouquet_name);
			
		    update $bar if not $opt_quiet;
	    }
    }
    die "no channels could be found" if not keys %channels;
	$bar->finish() if not $opt_quiet;
    return %channels;
}

# get channel listing
sub get_channels_for_bouquet {
    my ($channels, $bouquet_id, $bouquet_name) = @_;
    my $bouquet_prefix = "." . join('.', reverse split(/\W/, lc $bouquet_name)) . ".dstv.com";
    my $url="http://www.dstv.com/dstv-guide/default.asp?Bouquet=".$bouquet_id;
    my $local_data=get_nice($url);
    die "could not get channel listing $url, aborting\n"
      if not defined $local_data;

    # FIXME commonize this
    local $SIG{__WARN__} = sub {
	warn "$url: $_[0]";
    };
    local $SIG{__DIE__} = sub {
	die "$url: $_[0]";
    };
    
    my $tree = HTML::TreeBuilder->new();
    $tree->parse($local_data);

    # all channel elements are option tags within the select named ChannelID

    my @select_elems = $tree->look_down(_tag => 'select',
					name => 'ChannelID');
    my @ch_op_elems = map { $_->look_down(_tag => 'option') } @select_elems;
    foreach my $elem (@ch_op_elems) {
		my $channel_id = $elem->attr('value');
	    my @children=$elem->content_list;
	    if (scalar(@children) != 1 || ref($children[0])) {
			warn "Illegal channel name";
			next;
	    }
	    for ($children[0]) {
			s/^\s+//; s/\s+$//;
			/^(\d+) - (.*)$/;
			my $channel_number=$1;
			my $channel_name=$bouquet_name." - ".$2;
			t sprintf('%3d', $channel_id).":".sprintf('%3d', $channel_number).":".$channel_name;
			$$channels{sprintf('%3d', $channel_number).$bouquet_prefix}=$channel_name;
			$ch_ids{$1.$bouquet_prefix}=$channel_id;
			push @ch_all, { 'display-name' => [ [ $channel_name, $LANG ] ],
							'id' => $1.$bouquet_prefix };
	    }
    }
}

# Bump a YYYYMMDD date by one.
sub nextday {
    my $d = shift;
    my $p = parse_date($d);
    my $n = DateCalc($p, '+ 1 day');
    return UnixDate($n, '%Q');
}

sub cached_get {
	my $url = shift;

	my $home = $ENV{HOME} || 
               #$ENV{HOMEDRIVE} . $ENV{HOMEPATH} ||
               ".";
               
	my $cachepath = $ENV{CACHE_PATH} || "$home/.xmltv/cache";

	if (not -d($cachepath)) {
		if (not -e($cachepath)) {
			mkdir($cachepath);
		} else {
			die "$cachepath expected to be a directory";
		}
    }
	
	my $cachefile = $url;
	$cachefile =~ s/\:/_/g;
	$cachefile =~ s/\//_/g;
	$cachefile =~ s/\?/_/g;
	$cachefile = "$cachepath/$cachefile.txt";
	
	# if the cache file doesn't exist or it's older than the allowed cache age
	if ((!-f($cachefile)) || ((-M($cachefile) > $opt_cacheage))) {
		my $result = LWP::Simple::getstore($url,$cachefile);
		return undef if !HTTP::Status::is_success($result);
		$XMLTV::Get_nice::Delay = 5;
	} else {
		$XMLTV::Get_nice::Delay = 0;	
	}

	if (-f($cachefile))	
	{
		local(*INPUT, $/);
        open (INPUT, $cachefile)     || die "can't open $cachefile: $!";
		my $data = <INPUT>;
		return $data;
	}
	
	return undef;
}
