Bulk commit various permission indexing changes.

Author: oalders

.tidyallrc

@@ -57,3 +57,9 @@ ignore = lib/MetaCPAN/Server/View/JSON.pm
 ignore = lib/MetaCPAN/Server/View/Pod.pm
 ignore = lib/MetaCPAN/Util.pm
 ignore = lib/Plack/Session/Store/ElasticSearch.pm
+
+[SortLines]
+select = .gitignore
+
+[UniqueLines]
+select = .gitignore

.travis.yml

@@ -25,6 +25,9 @@ env:
     - USE_CPANFILE_SNAPSHOT=true
     - USE_CPANFILE_SNAPSHOT=false
 
+matrix:
+  allow_failures:
+    - env: USE_CPANFILE_SNAPSHOT=false
 
 before_install:
   - sudo service elasticsearch stop && curl -O -L https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.4.3.deb && sudo dpkg -i --force-confnew elasticsearch-2.4.3.deb && sudo service elasticsearch start

lib/MetaCPAN/Document/Permission.pm

@@ -1,29 +1,24 @@
 package MetaCPAN::Document::Permission;
 
-use strict;
-use warnings;
-
-use Moose;
+use MetaCPAN::Moose;
 
 use ElasticSearchX::Model::Document;
-use MooseX::StrictConstructor;
+use MetaCPAN::Types qw( ArrayRef Str );
 
-has module => (
+has module_name => (
     is       => 'ro',
-    isa      => 'Str',
+    isa      => Str,
     required => 1,
 );
 
 has owner => (
-    is       => 'ro',
-    isa      => 'Str',
-    required => 0,
+    is  => 'ro',
+    isa => Str,
 );
 
 has co_maintainers => (
-    is       => 'ro',
-    isa      => 'ArrayRef',
-    required => 0,
+    is  => 'ro',
+    isa => ArrayRef,
 );
 
 __PACKAGE__->meta->make_immutable;

lib/MetaCPAN/Model.pm

@@ -1,10 +1,8 @@
 package MetaCPAN::Model;
 
-use strict;
-use warnings;
-
 # load order important
 use Moose;
+
 use ElasticSearchX::Model;
 
 analyzer lowercase => (

lib/MetaCPAN/Role/Script.pm

@@ -201,6 +201,6 @@ __END__
 
 =head1 SYNOPSIS
 
-Roles which should be available to all modules
+Roles which should be available to all modules.
 
 =cut

lib/MetaCPAN/Script/Mapping.pm

@@ -1,25 +1,24 @@
 package MetaCPAN::Script::Mapping;
 
-use strict;
-use warnings;
+use Moose;
 
+use Cpanel::JSON::XS qw( decode_json );
+use DateTime ();
+use IO::Interactive qw( is_interactive );
+use IO::Prompt qw( prompt );
 use Log::Contextual qw( :log );
-use Moose;
+use MetaCPAN::Script::Mapping::CPAN::Author       ();
+use MetaCPAN::Script::Mapping::CPAN::Distribution ();
+use MetaCPAN::Script::Mapping::CPAN::Favorite     ();
+use MetaCPAN::Script::Mapping::CPAN::File         ();
+use MetaCPAN::Script::Mapping::CPAN::Mirror       ();
+use MetaCPAN::Script::Mapping::CPAN::Rating       ();
+use MetaCPAN::Script::Mapping::CPAN::Release      ();
+use MetaCPAN::Script::Mapping::DeployStatement    ();
+use MetaCPAN::Script::Mapping::User::Account      ();
+use MetaCPAN::Script::Mapping::User::Identity     ();
+use MetaCPAN::Script::Mapping::User::Session      ();
 use MetaCPAN::Types qw( Bool Str );
-use Cpanel::JSON::XS qw( decode_json );
-use DateTime;
-
-use MetaCPAN::Script::Mapping::DeployStatement;
-use MetaCPAN::Script::Mapping::CPAN::Author;
-use MetaCPAN::Script::Mapping::CPAN::Distribution;
-use MetaCPAN::Script::Mapping::CPAN::Favorite;
-use MetaCPAN::Script::Mapping::CPAN::File;
-use MetaCPAN::Script::Mapping::CPAN::Mirror;
-use MetaCPAN::Script::Mapping::CPAN::Rating;
-use MetaCPAN::Script::Mapping::CPAN::Release;
-use MetaCPAN::Script::Mapping::User::Account;
-use MetaCPAN::Script::Mapping::User::Identity;
-use MetaCPAN::Script::Mapping::User::Session;
 
 use constant {
     EXPECTED     => 1,

lib/MetaCPAN/Script/Permission.pm

@@ -1,15 +1,12 @@
 package MetaCPAN::Script::Permission;
 
-use strict;
-use warnings;
-
 use Moose;
-with 'MooseX::Getopt', 'MetaCPAN::Role::Script';
 
 use Log::Contextual qw( :log );
-use PAUSE::Permissions;
+use MetaCPAN::Document::Permission ();
+use PAUSE::Permissions             ();
 
-use MetaCPAN::Document::Permission;
+with 'MooseX::Getopt', 'MetaCPAN::Role::Script';
 
 =head1 SYNOPSIS
 
@@ -27,23 +24,42 @@ sub run {
 sub index_permissions {
     my $self = shift;
 
-    my $file_path = $self->cpan . '/modules/06perms.txt';
-    my $pp        = PAUSE::Permissions->new( path => $file_path );
-    my $type      = $self->index->type('permission');
-    my $bulk      = $self->model->bulk( size => 100 );
-
-    my $iterator = $pp->module_iterator();
-    while ( my $mp = $iterator->next_module ) {
-        my $put = { module => $mp->name };
-        $put->{owner}          = $mp->owner          if $mp->owner;
-        $put->{co_maintainers} = $mp->co_maintainers if $mp->co_maintainers;
-        $bulk->put( $type->new_document($put) );
+    my $file_path
+        = $self->cpan->subdir('modules')->file('06perms.txt')->absolute;
+    my $pp = PAUSE::Permissions->new( path => $file_path );
+
+    my $bulk_helper = $self->es->bulk_helper(
+        index => $self->index->name,
+        type  => 'permission',
+    );
+
+    my $iterator = $pp->module_iterator;
+    while ( my $perms = $iterator->next_module ) {
+
+        # This method does a "return sort @foo", so it can't be called in the
+        # ternary since it always returns false in that context.
+        # https://github.com/neilb/PAUSE-Permissions/pull/16
+
+        my @co_maints = $perms->co_maintainers;
+        my $doc       = {
+            @co_maints
+            ? ( co_maintainers => \@co_maints )
+            : (),
+            module_name => $perms->name,
+            owner       => $perms->owner,
+        };
+
+        $bulk_helper->update(
+            {
+                id            => $perms->name,
+                doc           => $doc,
+                doc_as_upsert => 1,
+            }
+        );
     }
 
-    $bulk->commit;
-
-    $self->index->refresh;
-    log_info {'done'};
+    $bulk_helper->flush;
+    log_info {'finished indexing 06perms'};
 }
 
 __PACKAGE__->meta->make_immutable;
@@ -61,6 +77,6 @@ Parse out CPAN author permissions.
 =head2 index_authors
 
 Adds/updates all ownership and maintenance permissions in the CPAN index to
-ElasticSearch.
+Elasticsearch.
 
 =cut

lib/MetaCPAN/Server/Controller/Permission.pm

@@ -1,10 +1,7 @@
 package MetaCPAN::Server::Controller::Permission;
 
-use strict;
-use warnings;
-use namespace::autoclean;
-
 use Moose;
+use namespace::autoclean;
 
 BEGIN { extends 'MetaCPAN::Server::Controller' }
 

t/00_setup.t

@@ -5,22 +5,20 @@ use lib 't/lib';
 
 use CPAN::Faker 0.010;
 use Devel::Confess;
-use File::Copy;
-use MetaCPAN::Script::Tickets;
+use File::Copy qw( copy );
+use MetaCPAN::Script::Tickets ();
 use MetaCPAN::Server::Test;
 use MetaCPAN::TestHelpers qw(
     fakecpan_configs_dir
     fakecpan_dir
     get_config
     tmp_dir
 );
-use MetaCPAN::TestServer;
+use MetaCPAN::TestServer ();
 use Module::Faker 0.015 ();    # Generates META.json.
-use Path::Class qw(dir);
 use Path::Class qw(dir file);
 use Test::More 0.96;
-use Test::More 0.96 ();
-use Test::Most;
+use URI::FromHash qw( uri );
 
 # Ensure we're starting fresh
 my $tmp_dir = tmp_dir();
@@ -55,7 +53,7 @@ my $cpan = CPAN::Faker->new(
 ok( $cpan->make_cpan, 'make fake cpan' );
 $fakecpan_dir->subdir('authors')->mkpath;
 
-# do some changes to 06perms.txt
+# make some changes to 06perms.txt
 {
     my $perms_file = $fakecpan_dir->subdir('modules')->file('06perms.txt');
     my $perms      = $perms_file->slurp;
@@ -79,6 +77,7 @@ copy( $src_dir->file('author-1.0.json'),
 
 copy( $src_dir->file('bugs.tsv'), $fakecpan_dir->file('bugs.tsv') );
 
+$server->index_permissions;
 $server->index_releases;
 $server->set_latest;
 $server->set_first;
@@ -89,12 +88,15 @@ $server->index_cpantesters;
 ok(
     MetaCPAN::Script::Tickets->new_with_options(
         {
-            %{$config},
-            rt_summary_url => 'file://'
-                . $fakecpan_dir->file('bugs.tsv')->absolute,
-            github_issues => 'file://'
-                . $fakecpan_dir->subdir('github')->absolute
-                . '/%s/%s.json?per_page=100',
+            rt_summary_url => uri(
+                scheme => 'file',
+                path => $fakecpan_dir->file('bugs.tsv')->absolute->stringify,
+            ),
+            github_issues => uri(
+                scheme => 'file',
+                path   => $fakecpan_dir->subdir('github')->absolute->stringify
+                    . '/%s/%s.json?per_page=100'
+            ),
         }
         )->run,
     'tickets'

t/lib/MetaCPAN/Server/Test.pm

@@ -4,9 +4,9 @@ use strict;
 use warnings;
 
 use HTTP::Request::Common qw(POST GET DELETE);
+use MetaCPAN::Server ();
 use Plack::Test;
 use Test::More;
-use MetaCPAN::Server;
 
 use base 'Exporter';
 our @EXPORT = qw(

t/lib/MetaCPAN/TestServer.pm

@@ -2,15 +2,15 @@ package MetaCPAN::TestServer;
 
 use MetaCPAN::Moose;
 
-use CPAN::Repository::Perms;
-use MetaCPAN::Script::Author;
+use MetaCPAN::Script::Author      ();
 use MetaCPAN::Script::CPANTesters ();
-use MetaCPAN::Script::Latest;
-use MetaCPAN::Script::First;
-use MetaCPAN::Script::Mapping;
-use MetaCPAN::Script::Release;
-use MetaCPAN::Server ();
-use MetaCPAN::TestHelpers qw( get_config fakecpan_dir );
+use MetaCPAN::Script::First       ();
+use MetaCPAN::Script::Latest      ();
+use MetaCPAN::Script::Mapping     ();
+use MetaCPAN::Script::Permission  ();
+use MetaCPAN::Script::Release     ();
+use MetaCPAN::Server              ();
+use MetaCPAN::TestHelpers qw( fakecpan_dir );
 use MetaCPAN::Types qw( Dir HashRef Str );
 use Search::Elasticsearch;
 use Search::Elasticsearch::TestServer;
@@ -214,6 +214,15 @@ sub index_cpantesters {
     );
 }
 
+sub index_permissions {
+    my $self = shift;
+
+    ok(
+        MetaCPAN::Script::Permission->new_with_options( $self->_config )->run,
+        'index permissions'
+    );
+}
+
 sub prepare_user_test_data {
     my $self = shift;
     ok(

t/server/controller/permission.t

@@ -0,0 +1,31 @@
+use strict;
+use warnings;
+
+use Cpanel::JSON::XS qw( decode_json );
+use MetaCPAN::Server::Test;
+use MetaCPAN::TestServer;
+use Test::More;
+
+my $server = MetaCPAN::TestServer->new;
+$server->index_permissions;
+
+test_psgi app, sub {
+    my $cb = shift;
+
+    my $module_name = 'CPAN::Test::Dummy::Perl5::VersionBump::Undef';
+    ok( my $res = $cb->( GET "/permission/$module_name" ),
+        "GET $module_name" );
+    is( $res->code, 200, '200 OK' );
+
+    is_deeply(
+        decode_json( $res->content ),
+        {
+            co_maintainers => ['FOOBAR'],
+            module_name    => $module_name,
+            owner          => 'MIYAGAWA',
+        },
+        'Owned by MIYAGAWA'
+    );
+};
+
+done_testing;