[WIP] some hacks to allow working against ES 2.3 index

lib/MetaCPAN/Sitemap.pm

@@ -8,7 +8,8 @@ use warnings;
 use autodie;
 
 use Carp;
-use ElasticSearch;
+use Search::Elasticsearch;
+use ElasticSearch::SearchBuilder;
 use File::Spec;
 use MetaCPAN::Web::Types qw( HashRef Int Str );
 use Moose;
@@ -51,19 +52,21 @@ sub process {
  # actually exist and b) the directory itself is writeable.
 
  # Get started. Create the ES object and the scrolled search object.
- my $es = ElasticSearch->new(
- servers => 'api.metacpan.org',
- no_refresh => 1,
+ my $es = Search::Elasticsearch->new(
+ nodes => ['api.metacpan.org'],
+ cxn_pool => 'Static::NoPing',
+ send_get_body_as => 'POST',
  );
- defined $es or croak "Unable to create ElasticSearch: $!";
+
+ my $field_name = $self->field_name;
 
  # Start off with standard search parameters ..
 
  my %search_parameters = (
  index => 'v0',
  size => 5000,
  type => $self->object_type,
- fields => [ $self->field_name ],
+ fields => [$field_name],
  );
 
  # ..and augment them if necesary.
@@ -73,11 +76,12 @@ sub process {
  # Copy the filter over wholesale into the search parameters, and add
  # the filter fields to the field list.
 
- $search_parameters{'queryb'} = $self->filter;
+ $search_parameters{'body'}
+ = ElasticSearch::SearchBuilder->new->query( $self->filter );
  push @{ $search_parameters{'fields'} }, keys %{ $self->filter };
  }
 
- my $scrolled_search = $es->scrolled_search(%search_parameters);
+ my $scrolled_search = $es->scroll_helper(%search_parameters);
 
  # Open the output file, get ready to pump out the XML.
 
@@ -90,12 +94,11 @@ sub process {
  = 'https://metacpan.org/' . $self->cpan_directory . q{/};
  }
 
- do {
- my @hits = $scrolled_search->drain_buffer;
+ while ( $scrolled_search->refill_buffer ) {
  push @urls,
- map { $metacpan_url . $_->{'fields'}->{ $self->field_name } }
- @hits;
- } while ( $scrolled_search->next() );
+ map { $metacpan_url . $_->{'fields'}->{$field_name} }
+ $scrolled_search->drain_buffer;
+ }
 
  $_ = $_ . q{ } for @urls;
 
@@ -118,4 +121,6 @@ sub process {
  return;
 }
 
+__PACKAGE__->meta->make_immutable;
+
 1;

lib/MetaCPAN/Web/Controller/Activity.pm

@@ -38,10 +38,9 @@ sub index : Path {
  '/release/_search',
  {
  query => { match_all => {} },
- facets => {
+ aggregations => {
  histo => {
- date_histogram => { field => 'date', interval => $res },
- facet_filter => {
+ filter => {
  and => [
  {
  range => {
@@ -50,14 +49,19 @@ sub index : Path {
  },
  @$q
  ]
+ },
+ aggregations => {
+ entries => {
+ date_histogram => { field => 'date', interval => $res },
+ }
  }
  }
  },
  size => 0,
  }
  )->recv;
- my $entries = $data->{facets}->{histo}->{entries};
- $data = { map { $_->{time} => $_->{count} } @$entries };
+ my $entries = $data->{aggregations}->{histo}->{entries}->{buckets};
+ $data = { map { $_->{key} => $_->{doc_count} } @$entries };
  my $line = [
  map {
  $data->{ $start->clone->add( months => $_ )->epoch . '000' }

lib/MetaCPAN/Web/Controller/Author.pm

@@ -6,6 +6,7 @@ use List::Util ();
 use DateTime::Format::ISO8601 ();
 use namespace::autoclean;
 use Locale::Country ();
+use MetaCPAN::Web::Util qw( fix_structure );
 
 BEGIN { extends 'MetaCPAN::Web::Controller' }
 
@@ -50,15 +51,15 @@ sub index : Chained('root') PathPart('') Args(0) {
  my ( $author, $data ) = ( $author_cv->recv, $releases_cv->recv );
  $c->detach('/not_found') unless ( $author->{pauseid} );
 
- my $faves_cv = $c->model('API::Favorite')->by_user( $author->{user} );
+ my $faves_cv = $c->model('API::Favorite')->by_user( $author->{pauseid} );
 
  my $faves_data = $faves_cv->recv;
  my $faves = [
  sort { $b->{date} cmp $a->{date} }
  map { $_->{fields} } @{ $faves_data->{hits}{hits} }
  ];
 
- my $releases = [ map { $_->{fields} } @{ $data->{hits}->{hits} } ];
+ my $releases = [ map { fix_structure($_->{fields}) } @{ $data->{hits}->{hits} } ];
  my $date = List::Util::max
  map { DateTime::Format::ISO8601->parse_datetime( $_->{date} ) }
  @$releases;

lib/MetaCPAN/Web/Controller/Favorite.pm

@@ -43,7 +43,7 @@ sub index : Path('/favorite/leaderboard') {
  my ( $self, $c ) = @_;
 
  my $data = $c->model('API::Favorite')->leaderboard( $c->req->page )->recv;
- my @leaders = @{ $data->{facets}->{leaderboard}->{terms} }[ 0 .. 99 ];
+ my @leaders = @{ $data->{aggregations}->{leaderboard}->{terms} }[ 0 .. 99 ];
 
  $c->stash(
  {

lib/MetaCPAN/Web/Controller/Feed.pm

@@ -9,6 +9,7 @@ use HTML::Escape qw/escape_html/;
 use DateTime::Format::ISO8601;
 use Path::Tiny qw/path/;
 use Text::Markdown qw/markdown/;
+use MetaCPAN::Web::Util qw( fix_structure );
 
 sub index : PathPart('feed') : Chained('/') : CaptureArgs(0) {
 }
@@ -76,7 +77,7 @@ sub author : Chained('index') PathPart Args(1) {
  = [ map { $_->{fields} } @{ $releases_cv->recv->{hits}{hits} } ];
  my $author_info = $author_cv->recv;
  my $faves_cv
- = $c->model('API::Favorite')->by_user( $author_info->{user} );
+ = $c->model('API::Favorite')->by_user( $author_info->{pauseid} );
  my $faves_data
  = [ map { $_->{fields} } @{ $faves_cv->recv->{hits}{hits} } ];
 
@@ -101,6 +102,7 @@ sub distribution : Chained('index') PathPart Args(1) {
 
 sub build_entry {
  my ( $self, $entry ) = @_;
+ $entry = fix_structure($entry);
  my $e = XML::Feed::Entry->new('RSS');
  $e->title( $entry->{name} );
  $e->link(

lib/MetaCPAN/Web/Controller/Recent.pm

@@ -2,6 +2,7 @@ package MetaCPAN::Web::Controller::Recent;
 use strict;
 use warnings;
 use base 'MetaCPAN::Web::Controller';
+use MetaCPAN::Web::Util qw( fix_structure );
 
 sub index : Path {
  my ( $self, $c ) = @_;
@@ -12,8 +13,10 @@ sub index : Path {
  my ($data)
  = $c->model('API::Release')
  ->recent( $req->page, $page_size, $req->params->{f} || 'l' )->recv;
- my $latest = [ map { $_->{fields} } @{ $data->{hits}->{hits} } ];
- $c->res->last_modified( $latest->[0]->{date} ) if (@$latest);
+ my $latest = [ map { fix_structure($_->{fields}) } @{ $data->{hits}->{hits} } ];
+ if ( @$latest ) {
+ $c->res->last_modified( $latest->[0]->{date} );
+ }
  $c->stash(
  {
  recent => $latest,

lib/MetaCPAN/Web/Controller/Recent/TopUploaders.pm

@@ -27,8 +27,8 @@ sub topuploaders : Private {
  my ( $self, $c, $range ) = @_;
 
  my $data = $c->model('API::Release')->topuploaders($range)->recv;
- my $counts = { map { $_->{term} => $_->{count} }
- @{ $data->{facets}{author}{terms} } };
+ my $counts = { map { $_->{key} => $_->{doc_count} }
+ @{ $data->{aggregations}{author}{entries}{buckets} } };
  my $authors = $c->model('API::Author')->get( keys %$counts )->recv;
  $c->stash(
  {
@@ -41,7 +41,7 @@ sub topuploaders : Private {
  } @{ $authors->{hits}{hits} }
  ],
  took => $data->{took},
- total => $data->{facets}{author}{total},
+ total => $data->{aggregations}{author}{total},
  template => 'recent/topuploaders.html',
  range => $range,
  }

lib/MetaCPAN/Web/Controller/Search.pm

@@ -69,7 +69,7 @@ sub index : Path {
  # these would be nicer if we had variable-length lookbehinds...
  $query =~ s{(^|\s)author:([a-zA-Z]+)(?=\s|$)}{$1author:\U$2\E}g;
  $query
- =~ s/(^|\s)dist(ribution)?:([\w-]+)(?=\s|$)/$1file.distribution:$3/g;
+ =~ s/(^|\s)dist(ribution)?:([\w-]+)(?=\s|$)/$1distribution:$3/g;
  $query
  =~ s/(^|\s)module:(\w[\w:]*)(?=\s|$)/$1module.name.analyzed:$2/g;
 

lib/MetaCPAN/Web/Controller/Search/AutoComplete.pm

@@ -3,6 +3,7 @@ package MetaCPAN::Web::Controller::Search::AutoComplete;
 use Moose;
 use JSON::MaybeXS ();
 use namespace::autoclean;
+use MetaCPAN::Web::Util qw( fix_structure );
 
 BEGIN { extends 'MetaCPAN::Web::Controller' }
 
@@ -11,8 +12,9 @@ sub index : Path {
  my $model = $c->model('API::Module');
  my $query = join( q{ }, $c->req->param('q') );
  my $data = $model->autocomplete($query)->recv;
+ my @fixed_data = map { fix_structure($_) } @{ $data->{results} };
  $c->res->content_type('application/json');
- $c->res->body( JSON::MaybeXS::encode_json( $data->{results} ) );
+ $c->res->body( JSON::MaybeXS::encode_json( \@fixed_data ) );
 }
 
 1;

lib/MetaCPAN/Web/Model/API/Author.pm

@@ -50,19 +50,19 @@ sub search {
  bool => {
  should => [
  {
- text => {
+ match => {
  'author.name.analyzed' =>
  { query => $query, operator => 'and' }
  }
  },
  {
- text => {
+ match => {
  'author.asciiname.analyzed' =>
  { query => $query, operator => 'and' }
  }
  },
- { text => { 'author.pauseid' => uc($query) } },
- { text => { 'author.profile.id' => lc($query) } },
+ { match => { 'author.pauseid' => uc($query) } },
+ { match => { 'author.profile.id' => lc($query) } },
  ]
  }
  },

lib/MetaCPAN/Web/Model/API/Favorite.pm

@@ -10,6 +10,7 @@ sub get {
  my ( $self, $user, @distributions ) = @_;
  @distributions = uniq @distributions;
  my $cv = $self->cv;
+ @distributions or return $cv;
  $self->request(
  '/favorite/_search',
  {
@@ -26,7 +27,7 @@ sub get {
  }
  }
  },
- facets => {
+ aggregations => {
  favorites => {
  terms => {
  field => 'favorite.distribution',
@@ -36,28 +37,31 @@ sub get {
  $user
  ? (
  myfavorites => {
- terms => { field => 'favorite.distribution', },
- facet_filter =>
- { term => { 'favorite.user' => $user } }
+ filter => { term => { 'favorite.user' => $user } },
+ aggregations => {
+ enteries => {
+ terms => { field => 'favorite.distribution' }
+ }
+ }
  }
-  )
+ )
  : (),
  }
  }
-  )->cb(
+ )->cb(
  sub {
  my $data = shift->recv;
  $cv->send(
  {
  took => $data->{took},
  favorites => {
- map { $_->{term} => $_->{count} }
- @{ $data->{facets}->{favorites}->{terms} }
+ map { $_->{key} => $_->{doc_count} }
+ @{ $data->{aggregations}->{favorites}->{buckets} }
  },
  myfavorites => $user
  ? {
- map { $_->{term} => $_->{count} }
- @{ $data->{facets}->{myfavorites}->{terms} }
+ map { $_->{key} => $_->{doc_count} }
+ @{ $data->{aggregations}->{myfavorites}->{entries}->{buckets} }
  }
  : {},
  }
@@ -101,7 +105,7 @@ sub leaderboard {
  {
  size => 0,
  query => { match_all => {} },
- facets => {
+ aggregations => {
  leaderboard =>
  { terms => { field => 'distribution', size => 600 }, },
  },
@@ -122,8 +126,9 @@ sub find_plussers {
  my $total_plussers = @plusser_users;
 
  # find plussers by pause ids.
- my $authors
- = $self->plusser_by_id( \@plusser_users )->recv->{hits}->{hits};
+ my $authors = @plusser_users
+ ? $self->plusser_by_id( \@plusser_users )->recv->{hits}->{hits}
+ : [];
 
  my @plusser_details = map {
  {

lib/MetaCPAN/Web/Model/API/File.pm

@@ -22,12 +22,12 @@ sub dir {
  query => { match_all => {}, },
  filter => {
  and => [
- { term => { 'file.level' => scalar @path } },
- { term => { 'file.author' => $author } },
- { term => { 'file.release' => $release } },
+ { term => { 'level' => scalar @path } },
+ { term => { 'author' => $author } },
+ { term => { 'release' => $release } },
  {
  prefix => {
- 'file.path' => join( q{/}, @path, q{} )
+ 'path' => join( q{/}, @path, q{} )
  }
  },
  ]
@@ -36,7 +36,7 @@ sub dir {
  },
  size => 999,
  fields => [
- qw(name stat.mtime path file.stat.size file.directory slop documentation mime)
+ qw(name stat.mtime path stat.size directory slop documentation mime)
  ],
  }
  );