#!/usr/local/bin/perl
#
#   This is a script for the RePEc::Index module.  It does update of
#   the RePEc-Index database.
#
#   Copyright (c) 2000-2002 Ivan Kurmanov, RePEc project.  All rights
#   reserved.
#
#   This program is free software; you can redistribute it and/or
#   modify it under the same terms as Perl itself.
#


use Events;

use RePEc::Iterator;

use strict;


use RePEc::Index::Config;


use RePEc::Index::Log;

RePEc::Index::Log::set_printout_level ( 4 );
RePEc::Index::Log::set_printout_level ( 5 );

sub log_it {
    RePEc::Index::Log::log( @_ );
}

sub p {
    log_it( 1, @_ );
}

sub error   { log_it( 8, @_ ); }

sub warning { log_it( 6, @_ ); }
    

my $repec_directory = '/home/ivan/data/RePEc';
my $authority = "RePEc";

$repec_directory = $RePEc::Index::Config::REPECDIR;
$authority       = $RePEc::Index::Config::AUTHORITY;

my $Session_Time = time ();   # getting current time

Events-> register_event_handler( 'REPEC::ITERATOR::*',  
'my ($s) = ($_[0] =~ m/^REPEC::ITERATOR::(.+)/); print STDERR "it $s: ", $_[1], "\n"; ');

Events->register_event_handler( 'REDIF::ARCHIVE::STARTED', \&archive_started);
Events->register_event_handler( 'REDIF::ARCHIVE::FINISHED',\&archive_finished);

# Events-> register_event_handler( 'REDIF::FILE::*',     \& print_event );
# Events->register_event_handler( 'REDIF::FILE::*', \&redif_file_handler );
Events->register_event_handler( 'REDIF::FILE::ARCHIVE',\&redif_file_handler );
Events->register_event_handler( 'REDIF::FILE::SERIES', \&redif_file_handler );
Events->register_event_handler( 'REDIF::FILE::NORMAL', \&redif_file_handler );


# Events-> register_event_handler( 'REDIF::ARCHIVE::STARTED', \& print_event );
# Events-> register_event_handler( 'REDIF::ARCHIVE::STARTED', \& print_event );
# Events-> register_event_handler( 'REPEC::ITERATOR::?',  \& print_event );

sub print_event {
    my $event = shift;
    $event =~ s/::/ /g;
    p "EVENT: $event  [ ", join ( ', ', @_ ), " ] ";
    return 1;
}

# Events-> register_event_handler( 'REDIF::FILE::NORMAL',
#  ' $::files++; print "*"; '  );

Events-> register_event_handler( 'REDIF::ARCHIVE::FINISHED',  
'&::p( "Archive finished ($::ARCHIVE_NAME): $::files"); $::files = 0; '.
'&::p( "File-records stored: $::stored_files"); $::stored_files = 0; '  );

# '&::p( "Archive finished ($::ARCHIVE_NAME): $::files"); $::files = 0; '.
# '&::p( "File-records stored: $::stored_files"); $::stored_files = 0; '  );

$::files = 0;
$::stored_files = 0;

log_it( 7, "$0, starting update process" );

RePEc::Iterator::main ( $repec_directory, $authority );

log_it( 7, "$0, finished update" );

RePEc::Index -> store_index_dat ( 
				  { 
				      LAST_UPDATE => $Session_Time , 
				      OK => 1, 
				  } 
				  );


p "Saved Index db status. update timestamp: $Session_Time";

############## main processing subroutines ####################################

use RePEc::Index;

use Carp::Assert;

use vars qw( $ARCHIVE_NAME );

my %ARC_FILES;

########

sub archive_started {
    $ARCHIVE_NAME = $_[1];
    %ARC_FILES = ();

    warning "Archive $ARCHIVE_NAME starts";

}

########

sub archive_finished {

    warning "Archive $ARCHIVE_NAME finished";

    my $hash = RePEc::Index -> iterate_archive_files( $ARCHIVE_NAME );

    warning "lost files check";
    while(  my $file = each ( %$hash )  ) {
	if( not $ARC_FILES{$file} ) {
	    warning "File not present: $file";
	    ####### !!!!!!!
	    ### XXX
	} else {
	    p "file $file confirmed";
	}
    }

}

########

sub redif_file_handler {

    my $filename = $_[1];
    my $short_filename = $_[2];

#    p "file: $filename, short: $short_filename";

    my $record = RePEc::Index 
          	-> look_up_file ( $short_filename, $ARCHIVE_NAME );

    log_it( 6, "Double file ($short_filename)" )   # probably impossible
	if ( $ARC_FILES{$short_filename} );
#    die "Double file ($short_filename)" if ( $ARC_FILES{$short_filename} );
    $ARC_FILES{$short_filename} = 1;

    my $readfile = 0;

    if ( not defined $record ) {

	log_it 5, "file $short_filename: new";

	$record = RePEc::Index -> create_new_file_record ();
	$record -> filename_set ( $short_filename );
	$record -> first_observed_set ( $Session_Time );
	$record -> templates_list_set ( [] );
	$record -> present_set ( 1 );
	$record -> last_observed_set ( $Session_Time );
	
	$readfile = 1;

    } else {

	if ( -e $filename ) {

	    my $mark = $record -> last_observed ();
	    if ( ( stat $filename ) [9] > $mark ) {
		### file has changed since last update
		log_it 5, "file $short_filename: known, changed";

		$record -> last_changed_set ( $Session_Time );
		
		$readfile = 1;
	    } else {
		log_it 3, "file $short_filename: known, unchanged";
	    }
	    
	    $record -> present_set ( 1 );
	    $record -> last_observed_set ( $Session_Time );
	    

	} else {
	    #p
	    log_it 9, "file $short_filename: known, dissapeared ($filename)";
	    use Carp qw( &cluck );
	    
	    cluck;
	    $record -> present_set ( 0 );
	    
	}
    }

    if( $readfile ) {

	my $handles_expected = $record -> templates_list(); 
	my @handles; 
	my %handles;
	
	use ReDIF::Parser;
	&ReDIF::Parser::open_file( $filename );
	
	my $t;
	while( $t = ReDIF::Parser::get_next_template() ) {
	    my $ha = $t->{handle}[0];

	    push @handles, $ha;
	    $handles{$ha} = 1;
	    
	    my $hr = RePEc::Index -> look_up_handle( $ha );

	    my $type = $t->{'template-type'}[0];

	    if( not defined $hr ) {
		$hr = RePEc::Index -> create_new_template_record();
		$hr -> handle_set( $ha );
		$hr -> first_observed_set( $Session_Time );
	    }
	    $hr -> object_set( $t );
	    $hr -> present_set( 1 );
	    $hr -> filename_set( $short_filename );
	    $hr -> redif_type_set( $type );
	    $hr -> last_observed_set( $Session_Time );

	    my $OK = RePEc::Index -> store_template_record ( $hr );
	    if ( $OK ) { $::stored_files ++; }
	    else { 
		error "Template storage operation failed ($ha)"; 
	    };
	    
	}
	$record -> templates_list_set( \@handles );

	### also need to set as absent the older templates
	### XXX
    }


    my $OK = RePEc::Index -> store_file_record ( $record, $ARCHIVE_NAME );
    if ( $OK ) { $::stored_files ++; }
    else { 
	error "Storage operation failed $short_filename"; 
#	warn "Storage operation failed"; 
    };
    return 0;
}    

__END__






