#!/usr/bin/perl -w
#
# This is a script for scraping slashdot comments so you can have a
# personal archive.
#
# TODO: fetch parent and 1 level of replies, so context is preserved.
#
# Copyright 2005 Ryan Underwood, <nemesis@icequake.net>
#
# This is free software and may be modified and redistributed for any
# purpose whatsoever, under the condition that you do not initiate any
# legal action involving the use of this software against the author(s).
#

use strict;
use LWP::UserAgent;
use HTML::Entities;
use HTML::TokeParser::Simple;
use HTTP::Cookies;
use Time::Local;

##############################################################################
# Configuration
##############################################################################
my $UID = 609077;
# The user's UID who you want to scrape comment history from.
my $SCRAPE_UID = $UID;
# This needs to be updated with the current value of your login cookie.
my $SLASHDOT_COOKIE = "user=".$UID."::2Bbcc5vt1LySZzj8VuKrHy";
# This should be the user agent that the cookie was created with.
my $USER_AGENT = 'Opera/7.54 (X11; Linux i586; U)  [en]';
# The HTML template for the output.  template() uses this.
my $tpl = qq{
<html>
<head><title>=TITLE=</title></head>
<body>
=TEXT=
<p>
=SCORE=
</body>
</html>
};

##############################################################################

my $ua = LWP::UserAgent->new;
$ua->default_headers->push_header('Cookie', $SLASHDOT_COOKIE);
$ua->agent($USER_AGENT);

my $url = "http://slashdot.org/users.pl?op=userinfo&uid=$SCRAPE_UID";
my $response = $ua->get($url);

if (!$response->is_success) {
	die $response->status_line;
}

my $page = $response->content;

my $stream = HTML::TokeParser::Simple->new(\$page);

my $num_comments = 0;
if ($page =~ /of ([0-9]+) [Cc]omments/) {
	$num_comments = $1;
}
else {
	die "Couldn't determine number of comments!";
}

$SIG{__DIE__} = sub { warn $page };

sub template($) {

	my $vars = shift;

	foreach my $var (keys %$vars) {
		$tpl =~ s/=$var=/$vars->{$var}/g;
	}
	return $tpl;
}

sub scrape_comment($$$$) {
	my $page_ref = shift;
	my $label = shift;
	my $sid = shift;
	my $cid = shift;
	my $stream = HTML::TokeParser::Simple->new($page_ref);
	my $found_date = 0;
	my $year = undef;
	my $mon = undef;
	my $mday = undef;
	my $title = undef;
	my $pollbooth = 0;

	while (my $tag = $stream->get_tag('a')) {
		if (!$found_date) {
			my $href = $tag->return_attr("href");
			if (defined($href)) {
				if ($href =~ /article\.pl\?sid=([0-9]+)\/([0-9]+)\/([0-9]+)\//) {
					$year = $1;
					$mon = $2;
					$mday = $3;
					while (!$tag->is_text) { $tag = $stream->get_token; }
					$title = $tag->as_is;
					$found_date = 1;
					#warn "Found the date, cool!";
				}
				elsif ($href =~ /pollBooth\.pl\?qid=[0-9]+/) {
					$pollbooth = 1;
					$found_date = 1;
					next;
				}
				next;
			}
		}
		my $name = $tag->return_attr("name");
		if (!defined($name) || $name eq '') { next; }
		if ($tag->return_attr("name") ne $cid) { next; }
		# We have located the comment area
		while (!$tag->is_end_tag('a')) { $tag = $stream->get_token; }
		$tag = $stream->get_token;
		my $score = $tag->as_is;
		$score =~ /Score:([\-0-9]{1,2})(?:, ([A-Za-z]+))?/;
		$score = $1;
		my $score_what = $2;
		if (!defined($score_what)) { $score_what = ''; }
		else { $score_what = ', '.$score_what; }

		my $ctime = time;
		if (!$pollbooth) {
			my $date_collect = '';
			my $date_reg = '([A-Z][a-z]+)\s+([A-Z][a-z]+)\s+([0-9]+),\s+@([0-9]+\:[0-9]+)([A-Z]{2})';
			while (1) {
				$tag = $stream->get_token;
				if (!$tag->is_text) {
					$date_collect = '';
					next;
				}
				$date_collect .= $tag->as_is;
				if ($date_collect =~ /$date_reg/) { last; }
			}
			$date_collect =~ /$date_reg/;
			my $time = $4;
			my $ampm = $5;
			my $hour = undef;
			my $min = undef;
			if ($time =~ /^([0-9]+):([0-9]+)$/) {
				$hour = $1;
				$min = $2;
			}
			else {
				warn "Failed to parse time: $time";
			}
			$ctime = timelocal(0,$min,$hour,$mday,$mon-1,$year);
		}

		# find the comment text
		my $text = '';
		while (!$tag->is_start_tag('td')) { $tag = $stream->get_token; }
		$tag = $stream->get_token;
		while (!$tag->is_end_tag('td')) {
			$text .= $tag->as_is;
			$tag = $stream->get_token;
		}

		# TODO: If there is a parent, fetch it so context is preserved.

		my $fname = "$cid\_$label.html";
		#have to deal with bad filename characters here
		$fname =~ s/[\/]/_/g;
		open my $file, ">$fname" or die $!.": ".$fname;
		print $file template({'TITLE' => $title, 'TEXT' => $text, 'SCORE' => 'Score:'.$score.$score_what});
		close $file;
		utime $ctime, $ctime, $fname;
	}
}

sub grab_comment($$$) {
	my $label = shift;
	my $sid = shift;
	my $cid = shift;
	my $url = "http://www.slashdot.org/comments.pl?sid=$sid&cid=$cid";

	print "Grabbing comment $label ($url)\n";

	my $ua = LWP::UserAgent->new;
	$ua->agent($USER_AGENT);
	my $response = $ua->get($url);
	if (!$response->is_success) {
		warn $response->status_line;
		return;
	}
	my $page = $response->content;
	scrape_comment(\$page, $label, $sid, $cid);
}

for (my $i = 0; $i < $num_comments;) {
	while (my $tag = $stream->get_tag('a'))
	{
		next unless $tag->return_attr("href") ne "";
		my $link = $tag->return_attr("href");
		if ($link =~ /slashdot\.org\/comments.*sid=([0-9]+).*cid=([0-9]+)/) {
			my $sid = $1;
			my $cid = $2;
			while (!$tag->is_text) { $tag = $stream->get_token; }
			my $label = $tag->as_is;
			#if ($i > 55) {
			grab_comment($label,$sid,$cid);
			#}
			$i++;
		}
	}

	my $nexturl = $url."&min_comment=".$i;
	print "Going to next page: $nexturl\n";

	# below variables are intentionally from global scope
	$ua = LWP::UserAgent->new;
	$ua->default_headers->push_header('Cookie', $SLASHDOT_COOKIE);
	$ua->agent($USER_AGENT);
	$response = $ua->get($nexturl);

	if (!$response->is_success) {
		die $response->status_line;
	}

	$page = $response->content;
	$stream = HTML::TokeParser::Simple->new(\$page);
}

