#!/usr/bin/env perl

use Koha;
use C4::Context;
use C4::Items qw(GetMarcWithItems);
use WebService::Solr;
use Koha::Solr::IndexStrategy::MARC;
use Koha::Solr::Document::MARC;
use Koha::Changelog::DBLog;
use Koha::Authority;
use File::Slurp;
use DateTime;
use DateTime::TimeZone;
use Try::Tiny;
use Getopt::Long qw(GetOptions);
use Pod::Usage;
use Log::Dispatch;
use Log::Dispatch::Screen;
use Carp;
use Parallel::ForkManager;
use Time::HiRes qw(gettimeofday);
use List::MoreUtils qw(natatime);
use POSIX qw();

my %dispatch = (
    biblio => {
        name => 'biblio',
        acquire_all_ids => sub {
            return C4::Context->dbh->selectcol_arrayref(
                'SELECT biblionumber FROM biblio');
        },
        acquire_one => \&GetMarcWithItems,
        rules_file => C4::Context->config('solr')->{biblio_rules},
        prefix => 'bib',
        rtype => 'biblio',
        tspref => 'SolrBibUpdateTS',
    },
    auth => {
        name => 'auth',
        acquire_all_ids => sub {
            return C4::Context->dbh->selectcol_arrayref(
                'SELECT authid FROM auth_header');
        },
        acquire_one => sub { Koha::Authority->new(id => $_)->marc },
        rules_file => C4::Context->config('solr')->{auth_rules},
        prefix => 'auth',
        rtype => 'auth',
        tspref => 'SolrAuthUpdateTS',
    },
);

my @verbosities = qw(warn info);
my @queues;
my %opts = (
    all => undef,
    ids => undef,
    timestamp => undef,
    to_stdout => undef,
    workers => 1,
    batch_size => 100,
    continuous => 0,
    sleeptime => 30,
    verbosity => 0,
    commit => 0,
    optimize => 0,
    delete => 0,
    empty => 0,
    help => undef,
    );

GetOptions(
    'bibs!' => sub { push @queues, 'biblio' },
    'auths!' => sub { push @queues, 'auth' },
    'a|all!' => \$opts{all},
    'ids:s' => \$opts{ids},
    's|since:s' => \$opts{timestamp},
    'w|workers:i' => \$opts{workers},
    'b|batch-size:i' => \$opts{batch_size},
    'c|continuous!' => \$opts{continuous},
    't|sleeptime:i' => \$opts{sleeptime},
    'delete!' => \$opts{delete},
    'empty!' => \$opts{empty},
    'commit!' => \$opts{commit},
    'optimize!' => \$opts{optimize},
    'o|stdout!' => \$opts{to_stdout},
    'v|verbose' => sub { $opts{verbosity}++ },
    'h|help' => \$opts{help},
) or pod2usage(1);

pod2usage(0) if $opts{help};

croak '"all" and "continuous" options are not compatible'
    if ( $opts{continuous} && $opts{all} );

my $log = Log::Dispatch->new();
$log->add( Log::Dispatch::Screen->new(
               min_level=>$verbosities[$opts{verbosity}] // 'debug',
               newline=>1, stderr=>1));

my $ltz = DateTime::TimeZone->new(name => 'local');

if ( $opts{empty} ) {
    my $solr = WebService::Solr->new( C4::Context->config('solr')->{url},
                                      { autocommit => 0 } );
    $log->info( 'Emptying index.' );
    $solr->delete( {query => 'rtype:bib'} ) if grep {/^biblio$/} @queues;
    $solr->delete( {query => 'rtype:auth'} ) if grep {/^auth$/} @queues;
}

while (1) {
    $log->info('Starting new queue run at '. DateTime->now(time_zone => $ltz));
    for ( @queues ) {
        my $rtype = $dispatch{$_};
        $log->info("Dispatching $rtype->{name}");
        $rtype->{opts} = \%opts;
        run_queue( $rtype );
        $log->info("Finished $rtype->{name}");
    }
    last unless $opts{continuous};
    $log->info('Sleeping until next run');
    sleep $opts{sleeptime};
}

exit;


sub run_queue {
    my $rtype = shift;

    $SIG{HUP} = $SIG{TERM} = $SIG{INT} = $SIG{QUIT} = sub {
        $log->warn('Caught signal. Exiting.');
        my $todos = [ {stamp => C4::Context->preference( $rtype->{tspref} )} ];
        update_timestamp( $rtype, $todos );
        exit 1;
    };

    my $todos = get_todos($rtype);

    update_timestamp($rtype, $todos);

    $log->info('Processing '.@$todos.' changelog entries');
    my $forker = Parallel::ForkManager->new($rtype->{opts}{workers});
    my $todoitr = natatime(@$todos / $rtype->{opts}{workers}, @$todos);
    while (my @todo_part = $todoitr->()) {
        next if $forker->start;
        $SIG{HUP} = $SIG{TERM} = $SIG{INT} = $SIG{QUIT} = undef;
        $C4::Context::context->{dbh} = C4::Context->dbh->clone;
        process_list($rtype, \@todo_part);
        $forker->finish;
    }
    $forker->wait_all_children;
    $SIG{HUP} = $SIG{TERM} = $SIG{INT} = $SIG{QUIT} = undef;

    if ($opts{optimize}) {
        my $solr = WebService::Solr->new( C4::Context->config('solr')->{url} );
        $log->info( 'Optimizing.' );
        $solr->optimize;
    }
    if ($opts{commit}) {
        my $solr = WebService::Solr->new( C4::Context->config('solr')->{url} );
        $log->info( 'Committing.' );
        $solr->commit;
    }

    return;
}

sub get_todos {
    my ($rtype) = @_;

    my $opts = $rtype->{opts};
    if ( $opts->{all} || $opts->{ids} ) {
        my @ids;
        if ( $opts->{all} ) {
            $log->info('Collecting all ids');
            @ids = @{$rtype->{acquire_all_ids}->()};
        }
        else {
            $log->info('Collecting select ids');
            for ( split /,/, $opts->{ids} ) {
                push @ids, /(\d+)-(\d+)/ ? $1..$2 : $_;
            }
        }
        my $tstamp = DateTime->now(time_zone => $ltz)->ymd
            . ' ' . DateTime->now(time_zone => $ltz)->hms;
        my $action = $opts{delete} ? 'delete' : 'update';
        return [
            map { {id=>$_, rtype=>$rtype->{name},
                   action=>$action, stamp=>$tstamp }
            } @ids
        ];
    }
    else {
        my $tstamp = $opts->{timestamp} // get_timestamp($rtype);
        $log->info("Collecting ids updated since $tstamp");
        return Koha::Changelog::DBLog
            ->new( rtype => $rtype->{name} )
            ->get_todos( $tstamp );
    }
}

sub s2hms {
    my $s = shift;
    my $hms;
    if ($s > 5400) {
        $hms = sprintf '%.02fh', $s / 3600;
    }
    elsif ($s > 120) {
        $hms = sprintf '%.02fm', $s / 60;
    }
    else {
        $hms = sprintf '%.02fs', $s;
    }
    return $hms;
}

sub process_list {
    our ($rtype, $todos) = @_;
    return unless @$todos;

    our $solr = WebService::Solr->new( C4::Context->config('solr')->{url},
                                      { autocommit => 0 } );
    my %deletes = map { $_->{id} => 1} grep { $_->{action} ~~ 'delete' } @$todos;
    my %updates = map { $_->{id} => 1} grep { $_->{action} ~~ 'update' } @$todos;

    # Remove deleted items from updates list
    %updates = map { $_ => 1 } grep { ! exists $deletes{$_} } keys %updates;

    if ( my @delete_ids = map { $rtype->{prefix}.'_'.$_} keys %deletes ) {
        $log->info('Deleting '. @delete_ids .' docs');
        $solr->delete( {id => \@delete_ids} );
        $log->info('Delete complete');
    }

    our $r = Koha::Solr::IndexStrategy::MARC->new(
        rules_text => scalar read_file($rtype->{rules_file}) );
    our ($update_count, $render_errors, $update_errors) = (0, 0, 0);
    our ($start_time, undef) = gettimeofday();
    our $queue_size = scalar keys %updates;
    $log->info("Have $queue_size updates in queue");

    sub doc_with_catch {
        my $id = shift;
        $log->debug("Expressing $id");
        my $doc = try {
            my $record = $rtype->{acquire_one}->($id);
            return unless $record;
            $update_count++;
            Koha::Solr::Document::MARC->new(
                record => $record, strategy => $r );
        }
        catch {
            $log->warn("Error rendering $id into Solr doc: $_");
            $render_errors++;
            $update_count--;
            undef;
        };
        return $doc;
    };

    sub add_with_catch {
        my $update_docs = shift;
        return unless @$update_docs;

        if ($rtype->{opts}{to_stdout}) {
            say join('', @$update_docs);
            return;
        }

        my ($now, undef) = gettimeofday();
        my $total_elapsed_seconds = ($now - $start_time) || 1;
        my $dps_all = int($update_count / $total_elapsed_seconds);
        my $docs_remaining = $queue_size - $update_count;
        my $est_seconds_remaining
            = ($dps_all) ? int($docs_remaining / $dps_all) : 0;

        my $elapsed = s2hms($total_elapsed_seconds);
        my $remaining = s2hms($est_seconds_remaining);
        my $msg = sprintf q{completed %d/%d, %d d/s, %s elapsed, %s remaining},
            $update_count, $queue_size, $dps_all, $elapsed, $remaining;
        $log->info($msg);

        try {
            if ($rtype->{opts}{to_stdout}) {
                say join("\n", @$update_docs);
            }
            else {
                $solr->add( $update_docs );
            }
        }
        catch {
            if (@$update_docs == 1) {
                $log->warn('Unable to upload doc: ' . substr($_, 0, 255));
                $update_errors++;
                return;
            }
            $log->warn("Batch upload error. Splitting batch.");
            my @ud2 = splice $update_docs, @$update_docs/2;
            add_with_catch( $update_docs );
            add_with_catch( \@ud2 );
            undef;
        };
        $log->debug('Batch complete');
    };

    my $update_docs = [];
    for ( sort {$a <=> $b} keys %updates ) {
        my $doc = doc_with_catch($_);
        next unless defined $doc;
        push $update_docs, $doc;
        if (@$update_docs >= $rtype->{opts}{batch_size}) {
            add_with_catch( $update_docs );
            $update_docs = [];
        }
    }
    add_with_catch( $update_docs );

    $log->info(
        "Updated $update_count records with $render_errors render errors ".
        "and $update_errors update errors.");
    return;
}

sub get_timestamp {
    my $rtype = shift;
    return C4::Context->preference($rtype->{tspref})
        // '1970-01-01 00:00:00';
}

sub update_timestamp {
    my ($rtype, $todos) = @_;
    return unless @$todos;

    return if $rtype->{opts}{all}
           || $rtype->{opts}{ids}
           || $rtype->{opts}{to_stdout};

    $todos = [ sort { $a->{stamp} cmp $b->{stamp} } @$todos ];
    my $youngest_entry = pop @$todos;
    my $youngest_ts = $youngest_entry->{stamp};

    $log->info('Updated timestamp is '.($youngest_ts ? $youngest_ts : '(undef)'));
    return unless $youngest_ts;

    C4::Context->preference_set($rtype->{tspref}, $youngest_ts);
    return;
}

__END__

=head1 NAME

solrsync - Synchronize Koha bibliographic data to Solr index

=head1 SYNOPSIS

solrsync [options]

  Options:
    --help | -h
    --verbose | -v
    --[no-]bibs
    --[no-]auths
    --all | -a
    --since=T | -s T
    --workers=N | -w N
    --batch-size=N | -b N
    --[no-]continuous | -c
    --sleeptime=S | -t S
    --ids=N,N,N-M
    --stdout | -o

=head1 TYPICAL USE CASES

Sync all bibs:

=over

C<solrsync --bibs --all>

=back

Sync all auths and bibs verbosely using four worker threads:

=over

C<solrsync --auths --bibs --all --workers=4 -v>

=back

Sync only bibs updated since last sync:

=over

C<solrsync --bibs>

=back

Sync only bibs changed since date:

=over

C<solrsync --bibs -s 2012-10-03T10:00:00>

=back

Sync only auth with authid between 1234 and 2345:

=over

C<solrsync --auths --ids=1234-2345>

=back

Sync all newly modified bibs and auths every 30 seconds:

=over

C<solrync --bibs --auths --continuous --sleeptime=30>

=back

=head1 OPTIONS

=over

=item B<--verbose>

Increase verbosity. Mulitple invocations are cumulative.

=back

=over

=item B<--bibs>

Process bibliographic records. Default is --no-bibs.

=back

=over

=item B<--auths>

Process authority records. Default is --no-auths.

=back

=over

=item B<--all>

Process all bib and/or auth records in database. Default is --no-all.

=back

=over

=item B<--since=T>

Process all bib and/or auth records written to changelog since time T.
T is an ISO formatted datetime stamp. Default is the value stored in the
appropriate syspref, either SolrBibUpdateTS for bibs or SolrAuthUpdateTS
for auths.

=back

=over

=item B<--workers=N>

Deploy N workers to process the queue contents in parallel. Defaults to 1.

=back

=over

=item B<--batch-size=N>

Queue up N expressed docs before uploading them to the Solr server. Defaults
to 100.

=back

=over

=item B<--continuous>

Run as a persistent process running in a loop. Default is to run a single
time then exit.

=back

=over

=item B<--sleeptime=S>

Sleep for S seconds between queue runs. Only effective when running in
continuous mode. Default is 20.

=back

=over

=item B<--ids=N,N,N-M>

Process only the biblionumbers or authids given in the value of this option.
IDs must be comma separated. A sequence can be given with dashes. For example,
C<--ids=11,17,93-95,22> would enqueue 11, 17, 93, 94, 95, and 22.

=back

=over

=item B<--stdout>

For debugging purposes, mostly. Writes the expressed document to stdout
instead of sending it to the Solr server.

=back

=cut
