AI::Categorizerで日本語

AI::Categorizerで日本語を読み込んでみたら全然動かなかったので30分Hack。以下をAI::Categorizer::Documentが読み込まれた後に読み込むと一応動く。３０分hackなのであまり信用しないように。

package AI::Categorizer::Document;

use strict;

use warnings;

use Text::MeCab;


sub tokenize_ascii {

  my $self = shift;

  my @tokens;

  while ($_[0] =~ /([-\w]+)/g) {

    my $word = lc $1;

    next unless $word =~ /[a-z]/;

    $word =~ s/^[^a-z]+//;  # Trim leading non-alpha characters (helps with ordinals)

    push @tokens, $word;

  }

  return \@tokens;

}

sub tokenize

{

    my $self = shift;

    if ($_[0] =~ /[\P{ASCII}]/) {

        my @ret;

        my $mecab = Text::MeCab->new();

        for(my $node = $mecab->parse($_[0]); $node; $node = $node->next) {

            push @ret, $node->surface;

        }

        return \@ret;

    } else {

        return $self->tokenize_ascii(@_);

    }

}

1;