# ABSTRACT: Scrappy HTTP Request Flow-Control System # Dist::Zilla: +PodWeaver package Scrappy::Queue; BEGIN { $Scrappy::Queue::VERSION = '0.94112090'; } # load OO System use Moose; # load other libraries use Array::Unique; use URI; # queue and cursor variables for navigation our @_queue = (); tie @_queue, 'Array::Unique'; our $_cursor = -1; sub list { return @_queue; } sub add { my $self = shift; my @urls = @_; # validate and formulate proper URLs for (my $i = 0; $i < @urls; $i++) { my $u = URI->new($urls[$i]); if ('URI::' =~ ref $u) { $urls[$i] = $u->as_string; } else { unless ($urls[$i] =~ /\w{2,}\.\w{2,}/) { delete $urls[$i]; } } } push @_queue, @urls; return $self; } sub clear { my $self = shift; @_queue = (); $_cursor = -1; return $self; } sub reset { my $self = shift; $_cursor = -1; return $self; } sub current { my $self = shift; return $_queue[$_cursor]; } sub next { my $self = shift; return $_queue[++$_cursor]; } sub previous { my $self = shift; return $_queue[--$_cursor]; } sub first { my $self = shift; $_cursor = 0; return $_queue[$_cursor]; } sub last { my $self = shift; $_cursor = scalar(@_queue) - 1; return $_queue[$_cursor]; } sub index { my $self = shift; $_cursor = shift || 0; return $_queue[$_cursor]; } sub cursor { return $_cursor; } 1; __END__ =pod =head1 NAME Scrappy::Queue - Scrappy HTTP Request Flow-Control System =head1 VERSION version 0.94112090 =head1 SYNOPSIS #!/usr/bin/perl use Scrappy::Queue; my $queue = Scrappy::Queue->new; $queue->add($url); while (my $url = $queue->next) { ... $queue->add(...); } =head1 DESCRIPTION Scrappy::Queue provides a system for saving URLs to a recordset/queue and iterating of them using the L framework. =head1 METHODS =head2 list The list method return the list of URLs in the queue. This is returned in list context. my $queue = Scrappy::Queue->new; ... my @list = $queue->list; =head2 add The add method adds new URLs to the queue. Duplicate URLs will be ignored. my $queue = Scrappy::Queue->new; $queue->add($url); =head2 clear The clear method completely empties the queue and resets the cursor (loop position). my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); $queue->clear; =head2 reset The reset method resets the cursor (loop position). my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); while (my $url = $queue->next) { $queue->reset if ...; # beware the infinate loop } $queue->reset; =head2 current The current method returns the URL in the current loop position. my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); while (my $url = $queue->next) { last if ...; } print 'great' if $url eq $queue->current; =head2 next The next method moves the cursor to the next loop position and returns the URL. my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); while (my $url = $queue->next) { ... } =head2 previous The previous method moves the cursor to the previous loop position and returns the URL. my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); while (my $url = $queue->next) { ... } print $queue->previous; =head2 first The first method moves the cursor to the first loop position and returns the URL. my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); print $queue->first; =head2 last The last method moves the cursor to the last loop position and returns the URL. my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); print $queue->last; =head2 index The index method moves the cursor to the specified loop position and returns the URL. The loop position is a standard array index position. my $queue = Scrappy::Queue->new; $queue->add(...); $queue->add(...); $queue->add(...); print $queue->index(1); =head2 cursor The cursor method returns the current loop position. my $queue = Scrappy::Queue->new; print $queue->cursor; =head1 AUTHOR Al Newkirk =head1 COPYRIGHT AND LICENSE This software is copyright (c) 2010 by awncorp. This is free software; you can redistribute it and/or modify it under the same terms as the Perl 5 programming language system itself. =cut