=head1 NAME

Data::Downloader -- Download and organize files using RSS feeds and templates

=head1 SYNOPSIS

    #!/usr/bin/env perl

    use Data::Downloader -init_logging => "DEBUG";
    use strict;

    # Get pictures of apples.  Store them using md5's, make symlinks using date and tags

    my $images = Data::Downloader::Repository->new(
        name           => "images",
        storage_root   => "/usr/local/datastore/data",
        cache_strategy => "Keep",
        feeds          => [
            {
                name => "flickr",
                feed_template => 'http://api.flickr.com/services/feeds/photos_public.gne?tags=<tags>&lang=en-us&format=rss_200',
                file_source => {
                    url_xpath      => 'media:content/@url',
                    filename_xpath => 'media:content/@url',
                    filename_regex => '/([^/]*)$',
                },
                metadata_sources => [
                    { name => 'date_taken', xpath => 'dc:date.Taken' },
                    { name => 'tags',       xpath => 'media:category' },
                ],
            },
        ],
        metadata_transformations => [
            {
                input         => "tags",
                output        => "tag",
                function_name => "split",
            },
        ],
        linktrees => [
            {
                root          => "/usr/local/datastore/by_tag",
                condition     => undef,
                path_template => "<tag>"
            },
            {
                root          => "/usr/local/datastore/by_date",
                condition     => undef,
                path_template => "<date_taken:%Y/%m/%d>"
            },
        ],
    );

    $images->load(speculative => 1) or $images->save;

    for my $feed ($images->feeds) {
        $feed->refresh(tags => "apples");
    }

    $images->download_all;

=head1 DESCRIPTION

Data::Downloader allows one to maintain local repositories of files
and file metadata based on RSS feeds which describe the files.  It
stores files uniquely using MD5 sums, but allows for the creation
and maintenance of trees of symbolic links to the downloaded files.

An SQLite database is used to store both the configuration and information
about the files that have been downloaded.  This file is automatically
created in $HOME/.data_downloader.db (by default).

This distribution also conatins a command-line tool, L<dado>, for
managing the repository.

=head2 CONFIGURATION

See L<Data::Downloader::Config>.

=head2 UPDATING RSS FEEDS

Read about this in L<Data::Downloader::Feed>.

=head2 DOWNLOADING FILES

Take a look at L<Data::Downloader::File>.

=head2 MAINTAINING SYMLINKS

Check out L<Data::Downloader::Linktree>.

=head1 SEE ALSO

L<dado>

L<Data::Downloader::Config>

L<Data::Downloader::Repository>

L<Data::Downloader::Feed>

L<Data::Downloader::DB>

L<Data::Downloader::Cache>

L<Data::Downloader::Linktree>

L<Rose>

=cut

package Data::Downloader;
use Rose::DB::Object::Loader;
use Lingua::EN::Inflect qw/def_noun/;
use Log::Log4perl qw(:easy);

use Data::Downloader::DB;
use Data::Downloader::DB::Object;
use Data::Downloader::DB::Object::Cached;
use Data::Downloader::Config;
use Data::Downloader::MetadataPivot;
use Data::Downloader::FileMetadata;
use strict;

def_noun "metadatum" => "metadata";

our $VERSION = '0.9902';
our $db;
our $useProgressBars;   # set during import to turn on Smart::Comments
our $setupDone;

sub import {
    my ($class,@params) = @_;
    return if $setupDone;
    $setupDone = 1;
    $db = Data::Downloader::DB->new("main");
    if (@params && grep /-init_logging/, @params) {
        my ($level) = grep { /^(DEBUG|INFO|WARN|ERROR|FATAL)$/ } @params;
        $level ||= "INFO";
        my $init = <<"EOT";
           log4perl.rootLogger = $level, Screen
           log4perl.appender.Screen = Log::Log4perl::Appender::ScreenColoredLevels
           log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
           log4perl.appender.Screen.layout.ConversionPattern = @{[ $ENV{HARNESS_ACTIVE} ? '#' : '' ]} [%-5p] %d %F{1} (%L) %m %n
EOT
        Log::Log4perl::init(\$init);
    }
    if (@params && grep /-use_progress_bars/, @params) {
        $useProgressBars = 1;
    }
    if (! -s $db->database) {
       INFO "initializing database ".$db->database."\n";
       _initialize_db();
    }
   _setup_classes();
}

sub _initialize_db {
    local $/ = ';';
    while (<DATA>) {
        my $stmt = $_;
        if (/CREATE TRIGGER/) {
            do {
                $stmt .= ( $_ = <DATA> );
            } until /END/ || !defined($_);
        }
        TRACE "SQL:\n$stmt\n";
        next unless $stmt =~ /\w/;
        $db->dbh->do($stmt) or die "failed to execute '$stmt'\n\n".$db->dbh->errstr;
    }
}

sub _setup_classes {
    our $cm = Rose::DB::Object::ConventionManager->new();
    $cm->tables_are_singular(1);
    $cm->singular_to_plural_function(\&Lingua::EN::Inflect::PL);

    my $loader_dynamic = Rose::DB::Object::Loader->new(
        base_class         => "Data::Downloader::DB::Object",
        db                 => $db,
        db_class           => "Data::Downloader::DB",
        class_prefix       => "Data::Downloader",
        convention_manager => $cm,
        post_init_hook     => sub { shift->error_mode('return'); },
        # TODO use log4perl for error mode somehow
    );

    my $loader_cached = Rose::DB::Object::Loader->new(
        base_class         => "Data::Downloader::DB::Object::Cached",
        db                 => $db,
        db_class           => "Data::Downloader::DB",
        class_prefix       => "Data::Downloader",
        convention_manager => $cm,
        post_init_hook     => sub { shift->error_mode('return'); },
    );

    my @config_tables = qw/repository feed disk feed_parameter file_source metadata_source metadata_transformation/;
    my @classes = $loader_dynamic->make_classes(exclude_tables => \@config_tables)
        or LOGDIE "Error: unable to load classes from ".$db->dbh->database;

    push @classes, $loader_cached->make_classes(include_tables => \@config_tables);

    for (@classes) {
        eval "use $_";
        no strict 'refs';
        *{"${_}::init_db"} = sub { Data::Downloader::DB->new_or_cached("main") };
        die "Errors using $_ : $@" if $@ && $@ !~ /Can't locate/;
    }

    Data::Downloader::MetadataPivot->do_setup;
    Data::Downloader::FileMetadata->do_setup;
}

1;

<<"=head1 SCHEMA"; # my trick to put __DATA__ into pod

=head1 SCHEMA

__DATA__

-- See also L<Rose::DB::Object::Loader>

 /*
  * Every table below corresponds to a class named Data::Downloader::<TableName>.
  * Every class has accessors and mutators whose names are the same as the column names.
  * Foreign keys also beget methods in both the parent and child classes, e.g.
  * e.g. $repository->files and $file->repository_obj.
  */

   /*** static tables : populated once during configuration ***/

--- A L<repository|Data::Downloader::Repository> has a L<cache|Data::Downloader::Cache> strategy.

    create table repository (
        id                integer primary key,
        name              varchar(255) not null unique,
        storage_root      text not null,
        file_url_template text,
        cache_strategy    text not null, -- e.g. "LRU" corresponds to Data::Downloader::Cache::LRU
        cache_max_size    integer -- in bytes
    );

--- A L<repository|Data::Downloader::Repository> has many L<feed|Data::Downloader::Feed>s.

    create table feed (
        id                integer primary key,
        repository        integer not null references repository(id),
        name              varchar(255) not null unique,
        feed_template     text
    );

--- A L<repository|Data::Downloader::Repository> has many L<disk|Data::Downloader::Disk>s.

    create table disk (
        id                integer primary key,
        repository        integer not null references repository(id),
        root              varchar(255) not null unique
    );

--- A L<feed|Data::Downloader::Feed> has many default parameters

    create table feed_parameter (
       id    integer primary key,
       feed  integer not null references feed(id),
       name  text not null,
       default_value text
    );

--- A L<feed|Data::Downloader::Feed> has a file source.

    create table file_source (
        feed             integer primary key not null references feed(id),
        url_xpath        text,
        urn_xpath        text,  -- unique identifier for files
        md5_xpath        text,
        filename_xpath   text,
        filename_regex   text   -- apply to whatever is extracted from the xpath
    );

--- A L<feed|Data::Downloader::Feed> has many metadata sources.

    create table metadata_source (
        id               integer primary key,
        feed             integer references feed(id),
        name             text not null unique,
        xpath            text
    );

--- A L<repository|Data::Downloader::Repository> has many L<linktree|Data::Downloader::Linktree>s.

    create table linktree (
        id               integer primary key,
        repository       integer references repository(id),
        root             text,
        condition        text,      -- SQL::Abstract clause
        path_template    text,      -- String::Template string
        unique(root)
    );

--- A L<repository|Data::Downloader::Repository> has many L<metadata_transformations|Data::Downloader::MetadataTransformation>s.

    create table metadata_transformation (
        id               integer primary key,
        input            text, -- references metadata_source.name or this table
        output           text not null,
        repository       integer references repository(id),
        function_name    text, -- function to apply
        function_params  text,
        order_key        integer not null default 1,
        unique(order_key)
    );

   /*** dynamic tables  : populated when files are downloaded, rss feeds are updated, etc. ***/

    create table stat_info (
        id integer primary key,
        repository integer not null references repository(id),
        last_stat_update datetime,
        last_fsck datetime,
        unique(repository)
    );

--- metadata

    create table metadatum (
        id     integer primary key,
        file   integer references file(id),
        name   varchar(255) references metadata_source(name),
        value  text,
        unique (file,name)
    );

--- A L<repository|Data::Downloader::Repository> has many L<file|Data::Downloader::File>s.

    create table file (
       id                 integer primary key,
       repository         integer not null references repository(id),
       filename           text not null,
       md5                char(32),
       url                text,
       urn                text,
       size               integer,
       on_disk            integer, -- boolean
       disk               integer references disk(id),
       atime              datetime, -- stat(file)->atime
       unique(md5),
       unique(filename),
       unique(urn)
    );

--- A L<file|Data::Downloader::File> has many log entries
--- (if $ENV{DATA_DOWNLOADER_GATHER_STATS} is set).

    create table log_entry (
       id integer primary key,
       file integer not null references file(id),
       requested_at datetime not null,
       cache_hit integer, -- boolean
       completed_at datetime,
       prog text, -- $0
       pid integer, -- $$
       uid text, -- $<
       note text -- $ENV{DATA_DOWNLOADER_LOG_NOTE}, e.g. app info
    );

--- A L<file|Data::Downloader::File> has many expirations

    create table file_removal (
       id integer primary key,
       file integer not null references file(id),
       expired_on datetime,
       algorithm, -- the expiration algorithm
       prog text, -- $0
       pid integer, -- $$
       uid text, -- $<
       note text -- $ENV{DATA_DOWNLOADER_FILE_REMOVAL_NOTE},
    );


--- A L<file|Data::Downloader::File> has many symlinks.

    create table symlink (
       id                 integer primary key,
       linkname           text not null, -- linktree.path_template + file's metadata
       file               integer not null references file(id),
       linktree           integer not null references linktree(id),
       unique(linkname)
    );

/***\

=cut

*/

create index symlink_file on symlink(file);
create index symlink_file_linktree on symlink(file,linktree);
create index symlink_linktree on symlink(linktree);

/***\

=head1 AUTHOR

 Brian Duggan

 Phillip Durbin

 Stuart Pineo

 Arnold Martin

 Graham Ollis

 Michael Walters

=cut