#!/usr/bin/env perl
# ABSTRACT: compute cosine similarity between two documents
# PODNAME: cosine_cmp
use autodie;
use strict;
use utf8;
use warnings;
our $VERSION = '0.008'; # VERSION
use Getopt::Long;
use Pod::Usage;
use Text::SpeedyFx;
GetOptions(
q(help) => \my $help,
q(length=i) => \my $length,
q(seed=i) => \my $seed,
q(bits=i) => \my $bits,
) or pod2usage(-verbose => 1);
pod2usage(-verbose => 2)
if $help or $#ARGV != 1;
$length = 10
unless $length;
$length *= 2 ** 10 << 3;
$seed = 0x4c53_4820
unless $seed;
$bits = 8
unless $bits;
my $sfx = Text::SpeedyFx->new($seed, $bits);
my @fv = map {
sub {
open my $fh, q(<:mmap), shift;
local $/ = undef;
my $data = <$fh>;
close $fh;
return $sfx->hash_fv($data, $length);
}->($_)
} @ARGV;
printf qq(similarity=%0.5f\n), cosine_similarity(@fv);
sub cosine_similarity {
my ($a, $b) = @_;
my $nbits_a = unpack(q(%32b*) => $a);
my $nbits_b = unpack(q(%32b*) => $b);
return $nbits_a * $nbits_b
? unpack(q(%32b*) => $a & $b) / sqrt $nbits_a * $nbits_b
: 0;
}
__END__
=pod
=encoding utf8
=head1 NAME
cosine_cmp - compute cosine similarity between two documents
=head1 VERSION
version 0.008
=head1 SYNOPSIS
cosine_cmp [options] FILE1 FILE2
=head1 DESCRIPTION
Cosine similarity is a measure of similarity between two vectors of an inner product space that measures the cosine of the angle between them.
The cosine of 0 is 1, and less than 1 for any other angle; the lowest value of the cosine is -1.
The cosine of the angle between two vectors thus determines whether two vectors are pointing in roughly the same direction.
This is often used to compare documents in text mining.
In addition, it is used to measure cohesion within clusters in the field of data mining.
(L