Merge pull request #804 from alphagov/temp-up-max-deletions-fix-duplicates

Up the max deletions to fix duplicate datasets

Author: kentsanggds

.rubocop.yml

@@ -4,6 +4,7 @@ AllCops:
     - bin/**/*
     - config.ru
     - db/schema.rb
+    - db/migrate/*
     - vendor/**/*
     - lib/ckan/v26/depaginator.rb
 

app/controllers/application_controller.rb

@@ -17,7 +17,7 @@ class ApplicationController < ActionController::Base
 private
 
   def record_not_found
-    render plain: '404 Not Found', status: 404
+    render plain: '404 Not Found', status: :not_found
   end
 
   def set_raven_context

app/controllers/datasets/licences_controller.rb

@@ -20,7 +20,7 @@ def create
 
   def update
     @dataset = current_dataset
-    @dataset.update_attributes(params.require(:dataset).permit(:licence_code))
+    @dataset.update(params.require(:dataset).permit(:licence_code))
 
     if @dataset.save(context: :dataset_form)
       redirect_to dataset_path(@dataset.uuid, @dataset.name)

app/controllers/datasets/locations_controller.rb

@@ -10,7 +10,7 @@ def edit
   def create
     @dataset = current_dataset
     location_params = params.require(:dataset).permit(:location1, :location2, :location3)
-    @dataset.update_attributes(location_params)
+    @dataset.update(location_params)
 
     if @dataset.save
       redirect_to new_dataset_frequency_path(@dataset.uuid, @dataset.name)
@@ -22,7 +22,7 @@ def create
   def update
     @dataset = current_dataset
     location_params = params.require(:dataset).permit(:location1, :location2, :location3)
-    @dataset.update_attributes(location_params)
+    @dataset.update(location_params)
 
     if @dataset.save
       redirect_to dataset_path(@dataset.uuid, @dataset.name)

app/helpers/application_helper.rb

@@ -1,10 +1,4 @@
 module ApplicationHelper
-  def url_contains(action)
-    url = request.path
-    url.gsub(@dataset.title, '') if @dataset.title
-    url.include?(action)
-  end
-
   def sortable(column, title = nil)
     title ||= column.titleize
     direction = column == sort_column && sort_direction == "asc" ? "desc" : "asc"

app/models/healthcheck/recurring_jobs/ckan_org_sync.rb

@@ -13,9 +13,9 @@ def name
       end
 
       def status
-        if Time.parse(when_last_run) <= critical_latency
+        if Time.zone.parse(when_last_run) <= critical_latency
           :critical
-        elsif Time.parse(when_last_run) <= warning_latency
+        elsif Time.zone.parse(when_last_run) <= warning_latency
           :warning
         else
           :ok

app/models/healthcheck/recurring_jobs/package_sync.rb

@@ -13,9 +13,9 @@ def name
       end
 
       def status
-        if Time.parse(when_last_run) <= critical_latency
+        if Time.zone.parse(when_last_run) <= critical_latency
           :critical
-        elsif Time.parse(when_last_run) <= warning_latency
+        elsif Time.zone.parse(when_last_run) <= warning_latency
           :warning
         else
           :ok

app/models/topic.rb

@@ -1,3 +1,3 @@
 class Topic < ApplicationRecord
-  has_many :datasets
+  has_many :datasets, dependent: :restrict_with_exception
 end

app/services/index_deletion_service.rb

@@ -25,7 +25,7 @@ def run
   def select_indexes_for_deletion(indexes)
     ordered_indexes = indexes
                         .select { |index_name| index_name.include? "#{index_alias}_" }
-                        .sort_by { |index_name| Time.parse(index_name.gsub(/"#{index_alias}_"/, '')) }
+                        .sort_by { |index_name| Time.zone.parse(index_name.gsub(/"#{index_alias}_"/, '')) }
                         .reverse
 
     # Ensure that the three most recent indexes are not deleted

config/application.rb

@@ -40,9 +40,9 @@ class Application < Rails::Application
       html_tag
     }
 
-    config.autoload_paths += [Rails.root.join("app/workers")]
-    config.autoload_paths += [Rails.root.join("lib/validators")]
-    config.autoload_paths += [Rails.root.join("lib/ckan")]
+    config.autoload_paths += [Rails.root.join("app", "workers")]
+    config.autoload_paths += [Rails.root.join("lib", "validators")]
+    config.autoload_paths += [Rails.root.join("lib", "ckan")]
 
     config.elasticsearch = config_for(:elasticsearch)
   end

config/environments/development.rb

@@ -13,7 +13,7 @@
   config.consider_all_requests_local = true
 
   # Enable/disable caching. By default caching is disabled.
-  if Rails.root.join('tmp/caching-dev.txt').exist?
+  if Rails.root.join('tmp', 'caching-dev.txt').exist?
     config.action_controller.perform_caching = true
 
     config.cache_store = :memory_store

config/initializers/elasticsearch.rb

@@ -6,7 +6,7 @@ def es_config_from_vcap
     Rails.logger.fatal "Failed to extract ES creds from VCAP_SERVICES. Exiting"
     Rails.logger.fatal Rails.configuration.elasticsearch['vcap_services']
     Rails.logger.fatal e
-    exit
+    return
   end
 
   es_config_from_host(es_server)

lib/ckan/v26/depaginator.rb

@@ -5,7 +5,7 @@ module V26
     class Depaginator
       include CKAN::Modules::URLBuilder
 
-      MAX_DELETIONS = 100
+      MAX_DELETIONS = 1500
 
       def self.depaginate(*args)
         self.new(*args).depaginate

lib/tasks/search.rake

@@ -4,7 +4,7 @@ namespace :search do
     batch_size = args.batch_size.nil? ? 50 : args.batch_size.to_i
     logger = Rails.logger
     client = Dataset.__elasticsearch__.client
-    date = Time.now.strftime('%Y%m%d%H%M%S')
+    date = Time.zone.now.strftime('%Y%m%d%H%M%S')
     index_alias = ENV['ES_INDEX'] || "datasets-#{Rails.env}"
     legacy_index = index_alias
     new_index_name = "#{Dataset.index_name}_#{date}"

spec/features/ckan_v26_package_sync_spec.rb

@@ -21,7 +21,7 @@
       legacy_name: "dataset_to_reimport",
       uuid: search_dataset_p1["results"][3]["id"],
       status: "draft",
-      updated_at: Time.parse(search_dataset_p1["results"][3]["metadata_modified"]),
+      updated_at: Time.zone.parse(search_dataset_p1["results"][3]["metadata_modified"]),
     )
   }
 
@@ -34,7 +34,7 @@
   let!(:dataset_not_to_update) do
     create :dataset, legacy_name: "dataset_not_to_update",
                      uuid: dataset_not_to_update_id,
-                     updated_at: Time.now
+                     updated_at: Time.zone.now
   end
 
   before do
@@ -70,14 +70,14 @@
   it 'updates existing datasets when they change in ckan' do
     expect { subject.perform }
       .to change { dataset_to_update.reload.updated_at }
-      .to(Time.parse(package_show_update["result"]["metadata_modified"]))
+      .to(Time.zone.parse(package_show_update["result"]["metadata_modified"]))
   end
 
   it "updates existing datasets when they're draft but appear in CKAN" do
     subject.perform
     dataset_to_reimport.reload
 
-    expect(dataset_to_reimport.updated_at).to eq(Time.parse(package_show_update["result"]["metadata_modified"]))
+    expect(dataset_to_reimport.updated_at).to eq(Time.zone.parse(package_show_update["result"]["metadata_modified"]))
     expect(dataset_to_reimport.status).to eq("published")
   end
 

spec/features/dataset_create_spec.rb

@@ -361,28 +361,28 @@ def pick_quarter(quarter)
       fill_in 'datafile[url]', with: 'https://localhost/doc'
       fill_in 'datafile[name]', with: 'my test doc'
       choose option: quarter.to_s
-      fill_in "datafile[year]", with: Date.today.year
+      fill_in "datafile[year]", with: Time.zone.today.year
       click_button "Save and continue"
     end
 
     it "calculates correct dates for Q1" do
       pick_quarter(1)
-      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Date.today.year, 6).end_of_month)
+      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Time.zone.today.year, 6).end_of_month)
     end
 
     it "calculates correct dates for Q2" do
       pick_quarter(2)
-      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Date.today.year, 9).end_of_month)
+      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Time.zone.today.year, 9).end_of_month)
     end
 
     it "calculates correct dates for Q3" do
       pick_quarter(3)
-      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Date.today.year, 12).end_of_month)
+      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Time.zone.today.year, 12).end_of_month)
     end
 
     it "calculates correct dates for Q4" do
       pick_quarter(4)
-      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Date.today.year, 3).end_of_month + 1.year)
+      expect(Dataset.last.datafiles.last.end_date).to eq(Date.new(Time.zone.today.year, 3).end_of_month + 1.year)
     end
   end
 

spec/rails_helper.rb

@@ -23,7 +23,7 @@
 # directory. Alternatively, in the individual `*_spec.rb` files, manually
 # require only the support files necessary.
 #
-Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f }
+Dir[Rails.root.join('spec', 'support', '**', '*.rb')].each { |f| require f }
 
 # Checks for pending migration and applies them before tests are run.
 # If you are not using ActiveRecord, you can remove this line.