#!/usr/bin/perl -w
use strict;

my $VERSION = '0.16';

#http://www.eurodns.com/search/index.php

#----------------------------------------------------------------------------

=head1 NAME

addresses.pl - helper script to map tester addresses to real people.

=head1 SYNOPSIS

  perl addresses.pl --config|c=<file> \
        [--address|a=<file>]  \
        [--mailrc|m=<file>]   \
        [--check=<file>]   \
        [--month=<string>] [--match] [--sort]

=head1 DESCRIPTION

Using the cpanstats database, the latest 01mailrc.txt file and the addresses
file, the script tries to match unmatched tester addresses to either a cpan
author or an already known tester.

For the remaining addresses, an attempt at pattern matching is made to try and
identify similar addresses in the hope they can be manually identified.

=cut

# -------------------------------------
# Library Modules

use lib qw(./lib ../lib);

use Config::IniFiles;
use CPAN::Testers::Common::DBUtils;
use DBI;
use IO::File;
use Getopt::ArgvFile default=>1;
use Getopt::Long;

# -------------------------------------
# Variables

my %defaults = (
    'address'   => 'data/addresses.txt',
    'mailrc'    => 'data/01mailrc.txt',
    'month'     => 199000,
    'match'     => 0,
    'sort'      => 0
);

my (%parsed_map,%cpan_map,%pause_map,%unparsed_map,%address_map,%domain_map,%target_map,%author_map);
my ($dbi,%result,%options);
my $parsed = 0;

my %phrasebook = (
    SRCHMNTH    => q{SELECT DISTINCT tester FROM cpanstats WHERE state IN ('pass','fail','na','unknown') AND type=2 AND postdate >= ?},
    SRCHALL     => q{SELECT DISTINCT tester FROM cpanstats WHERE state IN ('pass','fail','na','unknown') AND type=2},
    MBEMAIL     => q{SELECT fullname FROM metabase.testers_email WHERE email=?}
);

# -------------------------------------
# Program

##### INITIALISE #####

init_options();


##### MAIN #####

load_addresses();
if($options{check}) {
    check_addresses();
} else {
    match_addresses();
    print_addresses();
}

# -------------------------------------
# Subroutines

=head1 FUNCTIONS

=over 4

=item load_addresses

Loads all the data files with addresses against we can match, then load all
the addresses listed in the DB that we need to match against.

=cut

sub load_addresses {
    my $fh = IO::File->new($options{address})    or die "Cannot open address file [$options{address}]: $!";
    while(<$fh>) {
        s/\s+$//;
        next    if(/^$/);

        my ($source,$target) = (/(.*),(.*)/);
        next    unless($source && $target);
        $parsed_map{$source} = $target;
        my ($email) = $source =~ /([-+=\w]+\@(?:[-\w]+\.)+(?:com|net|org|info|biz|edu|museum|mil|gov|[a-z]{2,2}))/i;
        next    unless($email);
        $email = lc($email);
        my ($local,$domain) = split(/\@/,$email);
        $address_map{$email} = $target;
        $domain_map{$domain} = $target;
        $target_map{$target} = $email;
        my ($author) = ($target =~ /\(([A-Z0-9]+)\)/);
        $author_map{$author} = $email if($author);
#print STDERR "$source => $local => $domain\n"   unless($domain);

    }
    $fh->close;

    return if($options{check});

    if($options{verbose}) {
        print STDERR "parsed entries  = " . scalar(keys %parsed_map)  . "\n";
        print STDERR "address entries = " . scalar(keys %address_map) . "\n";
        print STDERR "domain entries  = " . scalar(keys %domain_map)  . "\n";
    }
#    use Data::Dumper;
#    print STDERR Dumper(\%domain_map);

    $fh = IO::File->new($options{mailrc})    or die "Cannot open mailrc file [$options{mailrc}]: $!";
    while(<$fh>) {
        s/\s+$//;
        next    if(/^$/);

        my ($alias,$name,$email) = (/alias\s+([A-Z]+)\s+"([^<]+) <([^>]+)>"/);
        next    unless($alias);
        $pause_map{lc($alias)} = "$name ($alias)";
        $cpan_map{lc($email)} = "$name ($alias)";
    }
    $fh->close;

    if($options{verbose}) {
        print STDERR "pause entries = " . scalar(keys %pause_map) . "\n";
        print STDERR "cpan entries  = " . scalar(keys %cpan_map)  . "\n";
    }

    # retrieve testers for reports
    my @rows;
    if($options{month}) {
        print STDERR "sql = $phrasebook{SRCHMNTH} [$options{month}]\n" if($options{verbose});
        @rows = $dbi->get_query('array',$phrasebook{SRCHMNTH},$options{month});
    } else {
        print STDERR "sql = $phrasebook{SRCHALL}\n" if($options{verbose});
        @rows = $dbi->get_query('array',$phrasebook{SRCHALL});
    }

    for my $row (@rows) {
        $parsed++;
        next    if($parsed_map{$row->[0]});
        $unparsed_map{$row->[0]} = "";
    }

    if($options{verbose}) {
        print STDERR "rows = " . scalar(@rows) . "\n";
        print STDERR "unparsed entries = " . scalar(keys %unparsed_map) . "\n";
    }
}

sub check_addresses {
    my ($match,$pause,$new) = (0,0,0);
    my $fh = IO::File->new($options{check})    or die "Cannot open check file [$options{check}]: $!";
    while(<$fh>) {
    #print STDERR "line=$_\n";
        s/\s+$//;
        next    if(/^$/);
        my ($email,$name) = split(/,/,$_,2);
        next	unless($email && $name);
        $name =~ s/ #.*$//;
    #print STDERR "name=[$name]\n";
	my ($author) = ($name =~ /\(([A-Z0-9]+)\)/);

        my ($type,$extra);
        if($target_map{$name}) {
            $type = 'MATCH';
            $match++;
            $extra = "[$target_map{$name},$name]";
        } elsif($author && $author_map{$author}) {
            $type = 'PAUSE';
            $pause++;
            $extra = "[$author_map{$author},$name]";
        } else {
            $type = 'NEW  ';
            $new++;
            $extra = '';
            $target_map{$name} = $email;
        }

        print "$type $email,$name $extra\n";
    }
    $fh->close;

    print "\nMATCH = $match\nPAUSE = $pause\nNEW   = $new\n";
}

sub match_addresses {
    for my $key (keys %unparsed_map) {
        my ($email) = $key =~ /([-+=\w]+\@(?:[-\w]+\.)+(?:com|net|org|info|biz|edu|museum|mil|gov|pro|xxx|name|mobi|tel|asia|[a-z]{2,2}))/i;
        unless($email) {
            push @{$result{NOEMAIL}}, $key;
            next;
        }
        $email = lc($email);
        my ($local,$domain) = split(/\@/,$email);
#print STDERR "email=[$email], local=[$local], domain=[$domain]\n"  if($email =~ /indiana/);
        next    if(map_pause(   $key,$local,$domain,$email));
        next    if(map_address( $key,$local,$domain,$email));
        next    if(map_cpan(    $key,$local,$domain,$email));
        next    if(map_metabase($key,$local,$domain,$email));

        my @parts = split(/\./,$domain);
        while(@parts > 1) {
            my $domain2 = join(".",@parts);
#print STDERR "domain2=[$domain2]\n"  if($email =~ /indiana/);
            last    if(map_domain($key,$local,$domain2,$email));
            shift @parts;
        }
    }
}

sub print_addresses {
    if($result{NOMAIL}) {
        print "ERRORS:\n";
        for my $email (sort @{$result{NOMAIL}}) {
            print "NOMAIL: $email\n";
        }
    }

    print "\nMATCH:\n";
    for my $key (sort {$unparsed_map{$a} cmp $unparsed_map{$b}} keys %unparsed_map) {
        if($unparsed_map{$key}) {
            print "$key,$unparsed_map{$key}\n";
            delete $unparsed_map{$key};
        }
    }

    print "\n";
    return  if($options{match});

    my @mails;
    print "PATTERNS:\n";
    for my $key (sort {$unparsed_map{$a} cmp $unparsed_map{$b}} keys %unparsed_map) {
        next    unless($key);

        my ($local,$domain) = $key =~ /([-+=\w]+)\@([^\s]+)/;
        if($domain) {
            my @parts = split(/\./,$domain);
            push @mails, [join(".",reverse @parts) . '@' . $local , $key];
        } else {
            print STDERR "FAIL: $key\n";
        }
    }
    for my $email (sort {$a->[0] cmp $b->[0]} @mails) {
        if($options{'sort'}) {
            print "$email->[1],\n";
        } else {
            print "$email->[0]\t$email->[1],\n";
        }
    }

    print "\nArticles parsed = $parsed\n\n";
}

sub map_metabase {
    my ($key,$local,$domain,$email) = @_;

    my @rows = $dbi->query_query('array',$phrasebook{MBEMAIL},$email);
    if(@rows) {
        $unparsed_map{$key} = $rows[0]->[0] . ' #[METABASE]';
        return 1;
    }
    return 0;
}

sub map_pause {
    my ($key,$local,$domain,$email) = @_;

    if($domain eq 'cpan.org') {
        $unparsed_map{$key} = ($pause_map{$local}||'UNKNOWN USER') . ' #[PAUSE]';
        return 1;
    }
    return 0;
}

sub map_address {
    my ($key,$local,$domain,$email) = @_;

    if($address_map{$email}) {
        $unparsed_map{$key} = $address_map{$email} . ' #[ADDRESS]';
        return 1;
    }
    return 0;
}

sub map_cpan {
    my ($key,$local,$domain,$email) = @_;

    if($cpan_map{$email}) {
        $unparsed_map{$key} = $cpan_map{$email} . ' #[CPAN]';
        return 1;
    }
    return 0;
}

sub map_domain {
    my ($key,$local,$domain,$email) = @_;

    return 0    if( $domain eq 'us.ibm.com'     ||
                    $domain eq 'shaw.ca'        ||
                    $domain eq 'ath.cx'         ||

                    $domain =~ /^(uklinux|eircom)\.net$/    ||
                    $domain =~ /^(rambler|mail)\.de$/       ||
                    $domain =~ /^(web|gmx)\.de$/            ||
                    $domain =~ /^(aacom|free)\.fr$/         ||
                    $domain =~ /^(xs4all|demon)\.nl$/       ||
                    $domain =~ /^(ne)\.jp$/                 ||

                    $domain =~ /^(nasa|nih)\.gov$/                                          ||
                    $domain =~ /^(ieee|no-ip|dyndns|cpan|perl|freebsd)\.org$/               ||
                    $domain =~ /^(verizon|gmx|comcast|earthlink|cox|usa)\.net$/             ||
                    $domain =~ /^(yahoo|google|gmail|googlemail|mac|pair|rr|sun|aol)\.com$/ ||
                    $domain =~ /^(pobox|hotmail|ibm|onlinehome-server)\.com$/               ||

                    $domain =~ /^mail\.(ru)$/                       ||
                    $domain =~ /^gov\.(au)$/                        ||
                    $domain =~ /^(net|org|com)\.(br|au|tw)$/        ||
                    $domain =~ /^(co|org)\.(uk|nz)$/                ||
                    $domain =~ /\b(edu|(ac|edu)\.(uk|jp|at|tw))$/             # education establishments
                );

#print STDERR "domain=[$domain]\n"   if($domain =~ /istic.org/);

    if($domain_map{$domain}) {
        $unparsed_map{$key} = $domain_map{$domain} . " #[DOMAIN] - $domain";
        return 1;
    }
    for my $map (keys %domain_map) {
        if($map =~ /\b$domain$/) {
            $unparsed_map{$key} = $domain_map{$map} . " #[DOMAIN] - $domain - $map";
            return 1;
        }
    }
    return 0;
}

=item init_options

Prepare command line options

=cut

sub init_options {
    GetOptions( \%options,
        'config=s',
        'address|a=s',
        'mailrc|m=s',
        'check=s',
        'month=s',
        'match',
        'sort',
        'verbose|v',
        'help|h'
    );

    _help(1) if($options{help});

    _help(1,"Must specify the configuration file")               unless(   $options{config});
    _help(1,"Configuration file [$options{config}] not found")   unless(-f $options{config});

    # load configuration
    my $cfg = Config::IniFiles->new( -file => $options{config} );

    # configure databases
    my $db = 'CPANSTATS';
    die "No configuration for $db database\n"   unless($cfg->SectionExists($db));
    my %opts = map {$_ => $cfg->val($db,$_);} qw(driver database dbfile dbhost dbport dbuser dbpass);
    $dbi = CPAN::Testers::Common::DBUtils->new(%opts);
    die "Cannot configure $db database\n" unless($dbi);

    # use defaults if none provided
    for my $opt (qw(address mailrc month match sort verbose)) {
        $options{$opt} ||= $cfg->val('MASTER',$opt) || $defaults{$opt};
    }

    for my $opt (qw(address mailrc)) {
        _help(1,"No $opt configuration setting given, see help below.")                 unless(   $options{$opt});
        _help(1,"Given $opt file [$options{$opt}] not a valid file, see help below.")   unless(-f $options{$opt});
    }

    return  unless($options{verbose});
    print STDERR "config: $_ = $options{$_}\n"  for(qw(address mailrc month match sort verbose));
}

sub _help {
    my ($full,$mess) = @_;

    print "\n$mess\n\n" if($mess);

    if($full) {
        print "\n";
        print "Usage:$0 [--help|h] [--version|v] \\\n";
        print "          --config|c=<file> \\\n";
        print "         [--address|a=<file>] \\\n";
        print "         [--mailrc|m=<file>] \\\n";
        print "         [--check=<file>] \\\n";
        print "         [--month=<string>] \\\n";
        print "         [--match] \n\n";

#              12345678901234567890123456789012345678901234567890123456789012345678901234567890
        print "This program checks for unknown tester addresses.\n";

        print "\nFunctional Options:\n";
        print "   --config=<file>           # path/file to configuration file\n";
        print "  [--address=<file>]         # path/file to addresses file\n";
        print "  [--mailrc=<file>]          # path/file to mailrc file\n";
        print "  [--check=<file>]           # path/file to check file\n";
        print "  [--month=<string>]         # YYYYMM string to match from\n";
        print "  [--match]                  # display matches only\n";

        print "\nOther Options:\n";
        print "  [--verbose]                # turn on verbose messages\n";
        print "  [--help]                   # this screen\n";

        print "\nFor further information type 'perldoc $0'\n";
    }

    print "$0 v$VERSION\n";
    exit(0);
}

__END__

=back

=head1 BUGS, PATCHES & FIXES

There are no known bugs at the time of this release. However, if you spot a
bug or are experiencing difficulties, that is not explained within the POD
documentation, please send bug reports and patches to the RT Queue (see below).

Fixes are dependent upon their severity and my availability. Should a fix not
be forthcoming, please feel free to (politely) remind me.

RT Queue -
http://rt.cpan.org/Public/Dist/Display.html?Name=CPAN-Testers-WWW-Statistics

=head1 SEE ALSO

L<CPAN::Testers::Data::Generator>,
L<CPAN::Testers::WWW::Reports>

F<http://www.cpantesters.org/>,
F<http://stats.cpantesters.org/>,
F<http://wiki.cpantesters.org/>

=head1 AUTHOR

  Barbie, <barbie@cpan.org>
  for Miss Barbell Productions <http://www.missbarbell.co.uk>.

=head1 COPYRIGHT AND LICENSE

  Copyright (C) 2005-2013 Barbie for Miss Barbell Productions.

  This module is free software; you can redistribute it and/or
  modify it under the same terms as Perl itself.

=cut