package Perl6::MetaModel::Parser::Tokenizer;

use strict;
use warnings;

our $VERSION = '0.01';

use String::Tokenizer;

sub new {
    my $class = shift;
    my $self = bless {
        tokenizer => String::Tokenizer->new()
    } => $class;
    $self->{tokenizer}->setDelimiter("{}();,\n\t");
    $self->{tokenizer}->handleWhitespace(String::Tokenizer->RETAIN_WHITESPACE);
    return $self;
}

sub tokenize { 
    my ($self, $source) = @_;
    $self->{tokenizer}->tokenize($source);
    grep { $_ } $self->{tokenizer}->getTokens();
}

sub iterator { (shift)->{tokenizer}->iterator }

1;