#!/usr/bin/perl

use strict qw(subs refs);
use warnings;
use LWP::UserAgent;
use XML::Parser;
use Data::Dumper;
use XML::XPath;
use IO::Compress::Gzip qw(gzip $GzipError) ;

use constant HAS_PREV => 1;

my %config;

sub set_config {
	$config{'map'} = "sitemap.xml";
	$config{'ua'} = LWP::UserAgent->new( 
		agent => "Eds static getter",
		max_redirect => 0 );
}

sub save_url {
	my $url = shift;
	my $path = shift;

	# $ua->max_redirect = 0;
	
	print( "Getting $url\n" );
	my $resp = $config{'ua'}->get( $url );

	if( !$resp->is_success ) {
		return( $resp->code );
	}

	if( $path eq $config{'map'} ) {
		open( F, ">${path}" ) or die "$path:$!";
		print( F $resp->content );
		close(F);
		print STDERR "Got $url\n";
	}

	my $z = new IO::Compress::Gzip( "${path}.gz", { -Level => 9 } ) or die( "${path}.gz:$!" );
	$z->write( $resp->content );
	$z->close();

	open( F, ">${path}.header" ) or die( "${path}.header:$!" );
	foreach( qw( Date Server Vary Content-Type Last-Modified ) ) {
		if( defined( $resp->headers->header( $_ ) ) ) {
			print( F $_, ": ", $resp->headers->header( $_ ), "\n" );
		}
		print( F "X-Mirror-Cache: http://www.s5h.net/\n" );
	}
	close(F);

	# does this page have a previous page link?
	# this will probably have to be replaced with different themes
	#if( $resp->content =~ /<a href="([^"])+">Previous Entries<\/a><\/span>/ ) {
	if( $resp->content =~ /Previous Entries/ ) {
		return( HAS_PREV );
	}

	return( undef );
}

sub main {
	# save the sitemap
	my $e = save_url( "http://www.s5h.net/sitemap.xml", $config{'map'} );
	die $e if( defined( $e ) );

	my $xp = XML::XPath->new( filename => $config{'map'} );
	my $nodeset = $xp->find('//url/loc');
	
	if( my @nodelist = $nodeset->get_nodelist ) {
		my @urls = map( $_->string_value, @nodelist );	
		
		foreach( @urls ) {
			my $new = $_;
			$new =~ s%http://[^/]*/(.*)%$1%g;
			
			`mkdir -p $new` unless( $new eq "" );

			my $ret = save_url( "http://www.s5h.net/$new", "./$new/index.html" );

			# now we get all the /page/2 whatever urls
			for( my $i = 1 ; defined( $ret ) && $ret == HAS_PREV ; $i++ ) {
				`mkdir -p ./${new}page/$i`;
				$ret = save_url( "http://www.s5h.net/${new}page/$i/", "./${new}page/$i/index.html" );
				# next if( !defined( $ret ) );
				if( defined( $ret ) && $ret != HAS_PREV ) {
					`rm -rf ./$new/page/$i`;
					last;
				}
			}

		}
	}

}

set_config;
main;

