#!/usr/local/bin/perl -w

use strict;

use GO::Parser;
use GO::AppHandle;
use Data::Stag;
use Getopt::Long;

if ($ARGV[0] =~ /^\-h/) {
    system("perldoc $0");
    exit;
}
my $apph = GO::AppHandle->connect(\@ARGV);

# 200000 lines equates roughly to a 500mb xsltproc process
use constant ASSOCFILE_LINE_LIMIT => $ENV{GO_ASSOCFILE_LINE_LIMIT} || 150000;

my $errf;
my $replace;
my $append;
my $fill_count;
my $fill_path;
my $no_fill_path;
my $no_optimize;
my $no_clear_cache;
my $add_root;
my $ev;
my $handler_class = 'godb';

my $fmt_arg = "";
if ($ARGV[0] =~ /^\-format/) {
    shift @ARGV;
    $fmt_arg = "-p " . shift @ARGV;
}

# parse global arguments
my @args = ();
while (@ARGV) {
    my $arg = shift @ARGV;
    if ($arg =~ /^\-e$/ || $arg =~ /^\-error$/) {
        $errf = shift @ARGV;
        next;
    }
    push(@args, $arg);
}


my $is_parsed_ontology;

# send XML errors to STDERR by default
my $errhandler = Data::Stag->getformathandler('xml');
if ($errf) {
    $errhandler->file($errf);
}
else {
    $errhandler->fh(\*STDERR);
}

my $generic_parser = new GO::Parser ({handler=>$handler_class});
$generic_parser->errhandler($errhandler);

# uncompress any compressed files
my @files = $generic_parser->normalize_files(@args);

# gene assoc files are too big for xslt processing; split into smaller files
@files =
  map {
      split_large_assocfile($_);
  } @files;

# parse file list and local arguments
my $dtype;
while (@files) {

    my $fn = shift @files;
    if ($fn =~ /^\-fill_count/) {
	$fill_count = 1;
	next;
    }
    if ($fn =~ /^\-fill_path/) {
	$fill_path = 1;
	next;
    }
    if ($fn =~ /^\-no_fill_path/) {
	$no_fill_path = 1;
	next;
    }
    if ($fn =~ /^\-no_optimize/) {
	$no_optimize = 1;
	next;
    }
    if ($fn =~ /^\-no_clear_cache/) {
	$no_clear_cache = 1;
	next;
    }
    if ($fn =~ /^\-add_root/) {
	$add_root = 1;
	next;
    }
    if ($fn =~ /^\-append/) {
	$append = 1;
	next;
    }
    if ($fn =~ /^\-replace/) {
	$replace = 1;
	next;
    }
    if ($fn =~ /^\-datatype/) {
        $dtype = shift @files;
        $fmt_arg = "-p $dtype"; # deprecated
        next;
    }
    if ($fn =~ /^\-ev/) {
        $ev = shift @files;
        next;
    }
    if ($fn =~ /^\-handler/) {
        $handler_class = shift @files;
        next;
    }
    if ($replace) {
        die "not implemented yet!";
        #$apph->remove_term_details($fn);
    }

    # if ANY of the files is an ontology
    $is_parsed_ontology = 1
      if !$dtype || $dtype eq 'go_ont' || $dtype =~ /obo/;

    my $this_file_is_an_ontology;
    if ($dtype) {
        $this_file_is_an_ontology = 
          $dtype eq 'go_ont' || $dtype =~ /obo/;
    }

    my $load_parser = GO::Parser->new({format=>$dtype,handler=>$handler_class});
    if ($handler_class eq 'chadodb') {
        $load_parser->xslt('oboxml_to_chadoxml')
          unless $dtype && $dtype eq 'chadoxml';
    }
    else {
        # default: GO DB
        $load_parser->xslt('oboxml_to_godb_prestore');
    }
    $load_parser->handler->apph($apph);
    $load_parser->errhandler($errhandler);

    showtime();

    # Set DBStag optimisations in GO::Handlers::godb
    $load_parser->handler->optimize_by_dtype($dtype,$append) unless $no_optimize;

    # provide parser with an index of term IDs derived
    # from db - this way the
    # parser is aware of illegal entries in xref and assoc files
    $apph->reset_acc2name_h;
    $load_parser->acc2name_h($apph->acc2name_h);
    printf STDERR "Valid terms in checklist: %d\n",
      scalar(keys %{$load_parser->acc2name_h || {}});
    
    # fire events from file to $handler
    $load_parser->parse($fn);
    #$fh->close || die ("problem with pipe: $cmd");
    
    # show statistics
    $load_parser->handler->show_cache;

    # avoid cumulatively large caches
    $load_parser->handler->clear_cache
      unless $no_clear_cache;

    $apph->commit;
    showtime();
    print STDERR "LOADED: $fn\n";
}

# add global root
if ($add_root) {
    $apph->add_root('all');
}

if ($fill_path ||
    ($is_parsed_ontology && !$no_fill_path)) {
    eval {
        $apph->fill_path_table;
    };
    if ($@) {
        $generic_parser->err("(FAO Developers: Error making paths: $@");
    }
}
if ($fill_count) {
    eval {
        $apph->fill_count_table;
    };
    if ($@) {
        $generic_parser->err(msg=>"(FAO Developers: Error making counts: $@");
    }
}
print "\nDone!\n";
exit 0;

sub showtime {
    my $t=time;
    my $ppt = localtime $t;
    print STDERR "$t $ppt ";
}

# break up large assocfiles into a directory of smaller files
#
# breaks occur every ASSOCFILE_LINE_LIMIT, but do NOT
# break in the middle of lines dealing with a gene product
sub split_large_assocfile {
    my $f= shift;
    if ($f !~ /gene_association\.(\S+)/) {
        return $f;
    }
    my $orgdb = $1;
    my ($wc) = (`wc $f` =~ /^\s*(\d+)/);
    chomp $wc;
    print STDERR "WC=$wc\n";
    if ($wc <= ASSOCFILE_LINE_LIMIT) {
        return $f;
    }
    my $splitdir = "$f-splitfiles";
    # remove any previous instance
    if (-d $splitdir) {
        if (system("rm -rf $splitdir")) {
            die "cannot remove $splitdir";
        }
    }
    if (system("mkdir $splitdir")) {
        die "cannot mkdir $splitdir";
    }
    my $fh = FileHandle->new("$f") || die("cannot open $f");
    my $n = 0;
    my $line = 1;
    my $last_gp;
    my $ofh;
    my @new_files = ();
    while (<$fh>) {
        next if /^\!/;
        my ($db, $gp) = /(\w+)\t(\S+)/;

        # if this is the first line OR
        # we have two many lines AND we are no breaking up a gene product
        # THEN start a new part
        if (!$ofh || 
            ($line > ASSOCFILE_LINE_LIMIT && $gp ne $last_gp)) {
            
            $n++;
            $line = 1;
            my $of = "$splitdir/gene_association.$orgdb.part-$n";
            push(@new_files, $of);
            $ofh = FileHandle->new(">$of") || die("cannot write to $of");
        }
        print $ofh $_;
        $last_gp = $gp;
        $line++;
    }
    $fh->close;
    $ofh->close;
    return @new_files;
}

__END__

=head1 NAME

load-go.pl

=head1 SYNOPSIS

  load-go.pl -d go -h mydbserver -datatype go_ont *.ontology

=head1 DESCRIPTION

Loads GO data (ontology files, def files, xref files, assoc files)
into a GO database. Will also perform additional housekeeping tasks on
database if required

=head1 MODULES AND SOFTWARE REQUIRED

You will need the 'xsltproc' executable, which is part of libxslt

(You will have this if you have already installed XML::LibXSLT)

You need to have B<both> go-perl and go-db-perl installed

L<http://www.godatabase.org/dev> contains further details on these two modules

This site also has details on the GO database

=head1 ARGUMENTS

  -d DBNAME

  -h DBSERVER

  -datatype FORMAT
     (see below)

  -schema SCHEMA
     by default: godb
     Other values: chado

     Support for the chado schema is in beta. See http://www.gmod.org/chado

  -dbms DRIVER
     by default: mysql
     other values: Pg

     Support for PostgreSQL is in beta

  -append
     by default this script assumes you are loading a dataset for
     the FIRST time. it performs only SQL INSERTs in certain
     cases rather than checking with SELECT if it needs to update.
     
     if you are loading the same file for the second time, use
     this option. the loading will be slightly slower, but it
     will append to existing data

     You should use this option if you are loading multiple
     ontology files in one go!

  -no_optimize
     by default, loading will be optimized; certain primary keys in
     the db will be cached, and certain tables will be INSERTED straight
     into without doing an initial SELECT (the presumption is that these
     datatypes would only be loaded once). See L<GO::Handlers::godb> for
     details.
     If this is turned off, then all data will follow the 
     SELECT followed by UPDATE or INSERT pattern
     This will be slower, but will use less memoty as no cache is required

  -no_clear_cache
     by default the in-memory cache (which reduced SQL lookups) is cleared
     after every single file is loaded. This is to prevent massive caches
     when we load all association files in a single command line.
     If you have plenty of memory, or aren't loading too many assoc
     files you may wish to use this option

  -fill_path
     (TRUE by default, IF an ontology file is parsed)
     
     populates the graph_path transitive closure table on completion

     this option can be used without any files as arguments to
     fill the path table in an already term-populated db

  -no_fill_path

     prevents graph_path table being populated after the ontologies
     have been loaded

  -fill_count

     populates the gene_product_count after all files have been loaded

  -add_root

     adds an explicit root term

     this may be necessary for loading from gene_ontology.obo which
     has 3 ontologies - it can be useful to make a fake root term
     covering these

     NOT FUNCTIONAL - CURRENTLY DONE AUTOMATICALLY

  -append

     you must use this option if you wish to append to data of the
     same type in an already loaded database; it switches off
     bulkloading option

  -replace

     removes all data of the same datatype before loading

  -ev
   
     filters based on an evidence type
     to filter out IEAs, use the not '!' prefix

         -ev '!IEA'
     

=head1 DATATYPES

specify these with the B<-datatype> option

=head2 go_ont

A GO ontology file.

After loading is completed, the path/closure table will be built

=head2 go_def

A GO.defs definitions file

=head2 go_xref

A GO xrefs file; eg ec2go

=head2 go_assoc

A gene_associations file

If you also specify the -fill_count option the gene_product_count table will also get populated (this is done at 

You can also specify the -ev command to filter out specific evidence codes; for example

  load-go.pl -d go -h mydbserver -datatype go-ontology *.ontology
  
=head2 obo

An obo formatted file

=head1 HOW IT WORKS

First the input file is converted into its native XML format (eg
OBO-XML). That native XML format is transformed to an XML format
isomorphic to the GO relational database using an XSLT
stylesheet. This transformed XML is then loaded using DBIx::DBStag

=head1 SEE ALSO

  go-dev/xml/xsl/oboxml_to_godb_prestore.xsl
  L<DBIx::DBStag>
  L<GO::Parser>

=head1 NOTES

When loading gene_association files, will split large files into
multiple smaller files and load these

=cut