Draft: Added setup test

Dockerfile

@@ -26,6 +26,10 @@ COPY lib lib
 COPY conf conf
 COPY *.conf .
 
+# move the following to test only
+COPY t t
+COPY test_data test_data
+
 ENV PERL5LIB="/app/local/lib/perl5:/app/lib" PATH="/app/local/bin:${PATH}"
 
 USER metacpan

bin/author.pl

@@ -51,15 +51,14 @@
 };
 
 # args
-my ( $mode, $pauseid, $whois_file );
+my ( $pauseid, $whois_file );
 GetOptions(
-    "mode=s"       => \$mode,
     "pauseid=s"    => \$pauseid,
     "whois_file=s" => \$whois_file,
 );
 
 # setup
-my $es = MetaCPAN::ES->new( index => "author", ( $mode ? ( mode => $mode ) : () ) );
+my $es = MetaCPAN::ES->new( index => "author" );
 
 log_info {'Reading 00whois'};
 my $authors_data = read_00whois( $whois_file );

bin/backpan.pl

@@ -18,8 +18,8 @@
 
 # setup
 my $cpan_file_map = cpan_file_map();
-my $es_release    = MetaCPAN::ES->new( type => "release" );
-my $es_file       = MetaCPAN::ES->new( type => "file" );
+my $es_release    = MetaCPAN::ES->new( index => "release" );
+my $es_file       = MetaCPAN::ES->new( index => "file" );
 
 my %bulk;
 my %release_status;
@@ -169,6 +169,6 @@ =head1 DESCRIPTION
 Sets "backpan" status on all BackPAN releases.
 
 --undo will set distributions' status back as 'cpan'
---file_only will only fix the 'file' type
+--file_only will only fix the 'file' index
 
 =cut

bin/backup.pl

@@ -21,19 +21,15 @@
 my $size       = 1000;
 my $index      = "cpan";
 
-my ( $dry_run, $mode, $purge, $restore );
+my ( $dry_run, $purge, $restore );
 GetOptions(
     "batch_size=i" => \$batch_size,
     "dry_run"      => \$dry_run,
     "index=s"      => \$index,
-    "mode=s"       => \$mode,
     "purge"        => \$purge,
     "restore=s"    => \$restore,
     "size=i"       => \$size,
 );
-# TODO: find a better way
-my @es_mode = ( $mode ? mode => $mode : () );
-$mode eq 'test' and Log::Log4perl::init('log4perl_test.conf');
 
 # setup
 my $home = home();
@@ -76,7 +72,7 @@ ()
         # create a new bulk helper for each
         my $key = $raw->{_index};
 
-        $es_store{$key} ||= MetaCPAN::ES->new( index => $key, @es_mode );
+        $es_store{$key} ||= MetaCPAN::ES->new( index => $key );
         my $es = $es_store{$key};
 
         $bulk_store{$key} ||= $es->bulk( max_count => $batch_size );
@@ -164,7 +160,7 @@ sub run_backup {
     $file->parent->mkpath unless ( -e $file->parent );
     my $fh = IO::Zlib->new( "$file", 'wb4' );
 
-    my $es = MetaCPAN::ES->new( index => $index, @es_mode );
+    my $es = MetaCPAN::ES->new( index => $index );
     my $scroll = $es->scroll(
         scroll => '1m',
         body   => {

bin/check.pl

@@ -9,19 +9,18 @@
 use MetaCPAN::Ingest qw< read_02packages_fh >;
 
 # args
-my ( $module, $max_errors, $errors_only );
-
+my (  $errors_only, $max_errors, $module );
 GetOptions(
-    "module=s"    => \$module,
-    "max_errors"  => \$max_errors,
     "errors_only" => \$errors_only,
+    "max_errors"  => \$max_errors,
+    "module=s"    => \$module,
 );
 
 # setup
 my $error_count = 0;
 my $packages_fh = read_02packages_fh();
-my $es_file     = MetaCPAN::ES->new( type => "file" );
-my $es_release  = MetaCPAN::ES->new( type => "release" );
+my $es_file     = MetaCPAN::ES->new( index => "file" );
+my $es_release  = MetaCPAN::ES->new( index => "release" );
 
 my $modules_start = 0;
 while ( my $line = <$packages_fh> ) {

bin/checksum.pl

@@ -11,14 +11,14 @@
 
 # args
 my $limit = 1000;
-my $dry_run;
+my ( $dry_run );
 GetOptions(
-    "limit=i" => \$limit,
     "dry_run" => \$dry_run,
+    "limit=i" => \$limit,
 );
 
 # setup
-my $es = MetaCPAN::ES->new( type => "release" );
+my $es = MetaCPAN::ES->new( index => "release" );
 my $bulk;
 $bulk = $es->bulk() unless $dry_run;
 

bin/contributor.pl

@@ -17,16 +17,15 @@
     "all"            => \$all,
     "distribution=s" => \$distribution,
     "release=s"      => \$release,
-
 );
 
 # setup
 my $author_mapping = {};
 my $email_mapping  = {};
 
-my $es_author      = MetaCPAN::ES->new( type => 'author' );
-my $es_release     = MetaCPAN::ES->new( type => "release" );
-my $es_contributor = MetaCPAN::ES->new( type => "contributor" );
+my $es_author      = MetaCPAN::ES->new( index => 'author' );
+my $es_release     = MetaCPAN::ES->new( index => "release" );
+my $es_contributor = MetaCPAN::ES->new( index => "contributor" );
 
 run();
 
@@ -296,6 +295,6 @@ =head1 SYNOPSIS
 =head1 DESCRIPTION
 
 Update the list of contributors (CPAN authors only) of all/matching
-releases in the 'contributor' type (index).
+releases in the 'contributor' index.
 
 =cut

bin/cover.pl

@@ -15,7 +15,7 @@
 my ( $json_file, $test );
 GetOptions(
     "json=s" => \$json_file,
-    "test"   => \$test
+    "test"   => \$test,
 );
 my $cover_url     //= 'http://cpancover.com/latest/cpancover.json';
 my $cover_dev_url //= 'http://cpancover.com/latest/cpancover_dev.json';
@@ -24,8 +24,9 @@
 my %valid_keys
     = map { $_ => 1 } qw< branch condition statement subroutine total >;
 
-my $es   = MetaCPAN::ES->new( index => "cover", type => "cover" );
-my $bulk = $es->bulk();
+my $es_release = MetaCPAN::ES->new( index => "release" );
+my $es_cover = MetaCPAN::ES->new( index => "cover" );
+my $bulk_cover = $es_cover->bulk();
 
 my $data = retrieve_cover_data();
 
@@ -34,9 +35,7 @@
 for my $dist ( sort keys %{$data} ) {
     for my $version ( keys %{ $data->{$dist} } ) {
         my $release   = $dist . '-' . $version;
-        my $rel_check = $es->search(
-            index => 'cpan',
-            type  => 'release',
+        my $rel_check = $es_release->search(
             size  => 0,
             body  => {
                 query => { term => { name => $release } },
@@ -56,7 +55,7 @@
             delete $doc_data{$k} unless exists $valid_keys{$k};
         }
 
-        $bulk->update( {
+        $bulk_cover->update( {
             id  => $release,
             doc => {
                 distribution => $dist,
@@ -69,7 +68,7 @@
     }
 }
 
-$bulk->flush;
+$bulk_cover->flush;
 
 ###
 

bin/cpan_testers.pl

@@ -36,7 +36,7 @@
 
 my $ua = ua();
 
-my $es   = MetaCPAN::ES->new( type => "release" );
+my $es   = MetaCPAN::ES->new( index => "release" );
 my $bulk = $es->bulk();
 
 log_info { 'Mirroring ' . $db };

bin/cpan_testers_api.pl

@@ -3,7 +3,6 @@
 use v5.36;
 
 use Cpanel::JSON::XS qw< decode_json >;
-use Getopt::Long;
 
 use MetaCPAN::Logger qw< :log :dlog >;
 
@@ -13,10 +12,6 @@
     ua
 >;
 
-# args
-my ();
-GetOptions();
-
 # setup
 my $home = home();
 
@@ -27,7 +22,7 @@
 
 my $ua = ua();
 
-my $es   = MetaCPAN::ES->new( type => "release" );
+my $es   = MetaCPAN::ES->new( index => "release" );
 my $bulk = $es->bulk();
 
 log_info { 'Fetching ' . $url };

bin/cve.pl

@@ -38,12 +38,14 @@
     "json_file=s"   => \$json_file,
     "test"          => \$test,
 );
+
+# setup
 $cve_url     //= 'https://hackeriet.github.io/cpansa-feed/cpansa.json';
 $cve_dev_url //= 'https://hackeriet.github.io/cpansa-feed/cpansa_dev.json';
 
-# setup
-my $es   = MetaCPAN::ES->new( index => "cve", type => "cve" );
-my $bulk = $es->bulk();
+my $es_release = MetaCPAN::ES->new( index => "release" );
+my $es_cve   = MetaCPAN::ES->new( index => "cve" );
+my $bulk_cve = $es_cve->bulk();
 
 my $data = retrieve_cve_data();
 
@@ -140,9 +142,7 @@
                 }
             };
 
-            my $releases = $es->search(
-                index => 'cpan',
-                type  => 'release',
+            my $releases = $es_release->search(
                 body  => {
                     query   => $query,
                     _source => [qw< version name author >],
@@ -183,15 +183,15 @@
             delete $doc_data->{$k} unless exists $valid_keys{$k};
         }
 
-        $bulk->update( {
+        $bulk_cve->update( {
             id            => $cpansa->{cpansa_id},
             doc           => $doc_data,
             doc_as_upsert => 1,
         } );
     }
 }
 
-$bulk->flush;
+$bulk_cve->flush;
 
 sub retrieve_cve_data {
     return decode_json( path($json_file)->slurp ) if $json_file;

bin/external.pl

@@ -11,11 +11,6 @@
 use MetaCPAN::External::Cygwin qw< run_cygwin >;
 use MetaCPAN::External::Debian qw< run_debian >;
 
-# with(
-#     'MetaCPAN::Script::Role::External::Cygwin',
-#     'MetaCPAN::Script::Role::External::Debian',
-# );
-
 # args
 my ( $email_to, $external_source );
 GetOptions(
@@ -28,7 +23,18 @@
     and grep { $_ eq $external_source } qw< cygwin debian >;
 
 # setup
-my $es = MetaCPAN::ES->new( type => "author" );
+my $es = MetaCPAN::ES->new( index => "distribution" );
+my $bulk = $es->bulk();
+my $scroll = $es->scroll(
+    scroll => '10m',
+    body   => {
+        query => {
+            exists => { field => "external_package." . $external_source }
+        }
+    },
+);
+
+
 
 my $ret;
 
@@ -54,16 +60,6 @@
     log_debug {$email_body};
 }
 
-my $scroll = $es->scroll(
-    type   => 'distribution',
-    scroll => '10m',
-    body   => {
-        query => {
-            exists => { field => "external_package." . $external_source }
-        }
-    },
-);
-
 my @to_remove;
 
 while ( my $s = $scroll->next ) {
@@ -80,8 +76,6 @@
     }
 }
 
-my $bulk = $es->bulk( type => 'distribution' );
-
 for my $d ( keys %{$dist} ) {
     log_debug {"[$external_source] adding $d"};
     $bulk->update( {

bin/favorite.pl

@@ -51,7 +51,7 @@ ()
         $query = { term => { distribution => $distribution } };
     }
     elsif ($age) {
-        my $es   = MetaCPAN::ES->new( type => "favorite" );
+        my $es   = MetaCPAN::ES->new( index => "favorite" );
         my $favs = $es->scroll(
             scroll => '5m',
             body   => {
@@ -84,7 +84,7 @@ ()
         $dist_fav_count{$distribution} = $count;
     }
     else {
-        my $es   = MetaCPAN::ES->new( type => "favorite" );
+        my $es   = MetaCPAN::ES->new( index => "favorite" );
         my $favs = $es->scroll(
             scroll => '30s',
             body   => {
@@ -113,7 +113,7 @@ ()
             @age_filter = ( must => [$age_filter] );
         }
 
-        my $es    = MetaCPAN::ES->new( type => "file" );
+        my $es    = MetaCPAN::ES->new( index => "file" );
         my $files = $es->scroll(
             scroll => '15m',
             body   => {
@@ -184,7 +184,7 @@ ()
             );
         }
         else {
-            my $es    = MetaCPAN::ES->new( type => "file" );
+            my $es    = MetaCPAN::ES->new( index => "file" );
             my $bulk  = $es->bulk( timeout => '120m' );
             my $files = $es->scroll(
                 scroll => '15s',