diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000000..ae10a5cce3b26 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +# editorconfig.org +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/Gemfile b/Gemfile index 478f81d27a60c..f84fe3c658271 100644 --- a/Gemfile +++ b/Gemfile @@ -78,6 +78,7 @@ gem "redcarpet", "~> 3.4" gem "reverse_markdown", "~> 1.1" gem "rolify", "~> 5.2" gem "rouge", "~> 3.2" +gem "rubyzip", "~> 1.2", ">= 1.2.2" gem "s3_direct_upload", "~> 0.1" gem "sass-rails", "~> 5.0" gem "sdoc", "~> 0.4", group: :doc diff --git a/Gemfile.lock b/Gemfile.lock index e4d1b4ed80614..f145cc7c9534f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -997,6 +997,7 @@ DEPENDENCIES rubocop (~> 0.59) rubocop-rspec (~> 1.29) ruby-prof (~> 0.17) + rubyzip (~> 1.2, >= 1.2.2) s3_direct_upload (~> 0.1) sass-rails (~> 5.0) sdoc (~> 0.4) diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index 31370a0add625..6d41530464e52 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -25,6 +25,12 @@ def update if @user.update(permitted_attributes(@user)) RssReader.new.delay.fetch_user(@user) if @user.feed_url.present? notice = "Your profile was successfully updated." + + if @user.export_requested? + notice = notice + " The export will be emailed to you shortly." + Exporter::Service.new(@user).delay.export(send_email: true) + end + follow_hiring_tag(@user) redirect_to "/settings/#{@tab}", notice: notice else diff --git a/app/mailers/notify_mailer.rb b/app/mailers/notify_mailer.rb index 039c58f9ddff4..bd91472504b8b 100644 --- a/app/mailers/notify_mailer.rb +++ b/app/mailers/notify_mailer.rb @@ -94,4 +94,11 @@ def mentor_email(mentor, mentee) subject = "You have been matched with a new DEV mentee!" mail(to: @mentor.email, subject: subject, from: "Liana (from dev.to) ") end + + def export_email(user, attachment) + @user = user + export_filename = "devto-export-#{Date.current.iso8601}.zip" + attachments[export_filename] = attachment + mail(to: @user.email, subject: "The export of your data is ready") + end end diff --git a/app/policies/user_policy.rb b/app/policies/user_policy.rb index 54dbfde7963d8..b7d72f89b1f37 100644 --- a/app/policies/user_policy.rb +++ b/app/policies/user_policy.rb @@ -103,7 +103,8 @@ def permitted_attributes summary text_color_hex username - website_url] + website_url + export_requested] end private diff --git a/app/services/exporter/articles.rb b/app/services/exporter/articles.rb new file mode 100644 index 0000000000000..eedb46fc34b7e --- /dev/null +++ b/app/services/exporter/articles.rb @@ -0,0 +1,67 @@ +module Exporter + class Articles + attr_reader :name + attr_reader :user + + def initialize(user) + @name = :articles + @user = user + end + + def export(slug: nil) + articles = user.articles + articles = articles.where(slug: slug) if slug.present? + json_articles = jsonify_articles(articles) + + { "articles.json" => json_articles } + end + + private + + def whitelisted_attributes + %i[ + body_markdown + cached_tag_list + cached_user_name + cached_user_username + canonical_url + comments_count + created_at + crossposted_at + description + edited_at + feed_source_url + language + last_comment_at + main_image + main_image_background_hex_color + path + positive_reactions_count + processed_html + published + published_at + published_from_feed + reactions_count + show_comments + slug + social_image + title + video + video_closed_caption_track_url + video_code + video_source_url + video_thumbnail_url + ] + end + + def jsonify_articles(articles) + articles_to_jsonify = [] + # load articles in batches, we don't want to hog the DB + # if a user has lots and lots of articles + articles.find_each do |article| + articles_to_jsonify << article + end + articles_to_jsonify.to_json(only: whitelisted_attributes) + end + end +end diff --git a/app/services/exporter/service.rb b/app/services/exporter/service.rb new file mode 100644 index 0000000000000..127df1f42c015 --- /dev/null +++ b/app/services/exporter/service.rb @@ -0,0 +1,64 @@ +require "zip" + +module Exporter + class Service + attr_reader :user + + EXPORTERS = [ + Articles, + ].freeze + + def initialize(user) + @user = user + end + + def export(send_email: false, config: {}) + exports = {} + + # export content with filenames + EXPORTERS.each do |exporter| + files = exporter.new(user).export(config.fetch(exporter.name.to_sym, {})) + files.each do |name, content| + exports[name] = content + end + end + + zipped_exports = zip_exports(exports) + + send_exports_by_email(zipped_exports) if send_email + + update_user_export_fields + + zipped_exports.rewind + zipped_exports + end + + private + + def zip_exports(exports) + buffer = StringIO.new + Zip::OutputStream.write_buffer(buffer) do |stream| + exports.each do |name, content| + stream.put_next_entry( + name, + nil, # comment + nil, # extra + Zip::Entry::DEFLATED, + Zlib::BEST_COMPRESSION, + ) + stream.write content + end + end + buffer + end + + def send_exports_by_email(zipped_exports) + zipped_exports.rewind + NotifyMailer.export_email(user, zipped_exports.read).deliver + end + + def update_user_export_fields + user.update!(export_requested: false, exported_at: Time.current) + end + end +end diff --git a/app/views/mailers/notify_mailer/export_email.html.erb b/app/views/mailers/notify_mailer/export_email.html.erb new file mode 100644 index 0000000000000..82385c73a68b5 --- /dev/null +++ b/app/views/mailers/notify_mailer/export_email.html.erb @@ -0,0 +1,4 @@ +

Your data has been exported.

+

+Please check the attached file. +

diff --git a/app/views/mailers/notify_mailer/export_email.text.erb b/app/views/mailers/notify_mailer/export_email.text.erb new file mode 100644 index 0000000000000..0ee052c38c2f4 --- /dev/null +++ b/app/views/mailers/notify_mailer/export_email.text.erb @@ -0,0 +1,3 @@ +Your data has been exported. + +Please check the attached file. diff --git a/app/views/users/_misc.html.erb b/app/views/users/_misc.html.erb index e7927b7c74696..f78f519ba8ce4 100644 --- a/app/views/users/_misc.html.erb +++ b/app/views/users/_misc.html.erb @@ -50,7 +50,7 @@ <%= f.check_box :prefer_language_it %> <%= f.label :prefer_language_it, "Italian" %> - +
<%= f.hidden_field :tab, value: @tab %> @@ -85,3 +85,32 @@ <%= f.submit "SUBMIT", class: "cta" %>
<% end %> + +

Export posts

+ +<% if @user.export_requested? %> +

+ You have recently requested an export of your data. + Please check your email. +

+<% else %> +

+ You can request an export of all your data. + Currently we only support the export of posts. + It will be emailed to your inbox. +

+ + <%= form_for(@user) do |f| %> +
+
+ <%= f.check_box :export_requested %> + <%= f.label :export_requested, "Request an export of your data" %> +
+
+
+ + <%= f.hidden_field :tab, value: @tab %> + <%= f.submit "SUBMIT", class: "cta" %> +
+ <% end %> +<% end %> diff --git a/db/migrate/20181026112019_add_export_fields_to_users.rb b/db/migrate/20181026112019_add_export_fields_to_users.rb new file mode 100644 index 0000000000000..f270c31383c9e --- /dev/null +++ b/db/migrate/20181026112019_add_export_fields_to_users.rb @@ -0,0 +1,6 @@ +class AddExportFieldsToUsers < ActiveRecord::Migration[5.1] + def change + add_column :users, :export_requested, :boolean, default: false + add_column :users, :exported_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index e26b42b0e118b..08fe2e2c3e784 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -711,6 +711,8 @@ t.string "employment_title" t.string "encrypted_password", default: "", null: false t.integer "experience_level" + t.boolean "export_requested", default: false + t.datetime "exported_at" t.string "facebook_url" t.boolean "feed_admin_publish_permission", default: true t.boolean "feed_mark_canonical", default: false diff --git a/spec/mailers/notify_mailer_spec.rb b/spec/mailers/notify_mailer_spec.rb index 482adc017df00..0358e8b033e5d 100644 --- a/spec/mailers/notify_mailer_spec.rb +++ b/spec/mailers/notify_mailer_spec.rb @@ -120,6 +120,7 @@ def params(user_email, feedback_message_id) mentee_email = described_class.mentee_email(mentee, mentor) expect(mentee_email.subject).to eq "You have been matched with a DEV mentor!" end + it "renders proper from" do mentee_email = described_class.mentee_email(mentee, mentor) expect(mentee_email.from).to include "liana@dev.to" @@ -134,10 +135,34 @@ def params(user_email, feedback_message_id) mentor_email = described_class.mentor_email(mentor, mentee) expect(mentor_email.subject).to eq "You have been matched with a new DEV mentee!" end - it "renders proper subject" do + + it "renders proper from" do mentor_email = described_class.mentor_email(mentor, mentee) expect(mentor_email.from).to include "liana@dev.to" end end + + describe "#export_email" do + it "renders proper subject" do + export_email = described_class.export_email(user, "attachment") + expect(export_email.subject).to include("export of your data is ready") + end + + it "renders proper receiver" do + export_email = described_class.export_email(user, "attachment") + expect(export_email.to).to eq([user.email]) + end + + it "attaches a zip file" do + export_email = described_class.export_email(user, "attachment") + expect(export_email.attachments[0].content_type).to include("application/zip") + end + + it "adds the correct filename" do + export_email = described_class.export_email(user, "attachment") + expected_filename = "devto-export-#{Date.current.iso8601}.zip" + expect(export_email.attachments[0].filename).to eq(expected_filename) + end + end end end diff --git a/spec/requests/user_settings_spec.rb b/spec/requests/user_settings_spec.rb index 6a2e8d8e282c4..9ada4ccc45887 100644 --- a/spec/requests/user_settings_spec.rb +++ b/spec/requests/user_settings_spec.rb @@ -53,6 +53,10 @@ describe "PUT /update/:id" do before { login_as user } + after do + Delayed::Worker.delay_jobs = false + end + it "updates summary" do put "/users/#{user.id}", params: { user: { tab: "profile", summary: "Hello new summary" } } expect(user.summary).to eq("Hello new summary") @@ -62,6 +66,50 @@ put "/users/#{user.id}", params: { user: { tab: "profile", username: "h" } } expect(response.body).to include("Username is too short") end + + context "when requesting an export of the articles" do + def send_request(flag = true) + Delayed::Worker.delay_jobs = true + put "/users/#{user.id}", params: { + user: { tab: "misc", export_requested: flag } + } + end + + it "updates export_requested flag" do + send_request + expect(user.reload.export_requested).to be(true) + end + + it "displays a flash with a reminder for the user to expect an email" do + send_request + expect(flash[:notice]).to include("The export will be emailed to you shortly.") + end + + it "hides the checkbox" do + send_request + follow_redirect! + expect(response.body).not_to include("Request an export of your posts") + end + + it "tells the user they recently requested an export" do + send_request + follow_redirect! + expect(response.body).to include("You have recently requested an export") + end + + it "sends an email" do + expect do + send_request + Delayed::Worker.new.work_off + # Delayed::Worker.delay_jobs = false + end.to change { ActionMailer::Base.deliveries.count }.by(1) + end + + it "does not send an email if there was no request" do + Delayed::Worker.delay_jobs = false + expect { send_request(false) }.not_to(change { ActionMailer::Base.deliveries.count }) + end + end end describe "DELETE /users/remove_association" do diff --git a/spec/services/exporter/articles_spec.rb b/spec/services/exporter/articles_spec.rb new file mode 100644 index 0000000000000..1c73e342cf620 --- /dev/null +++ b/spec/services/exporter/articles_spec.rb @@ -0,0 +1,115 @@ +require "rails_helper" + +RSpec.describe Exporter::Articles do + let(:user) { create(:user) } + let(:article) { create(:article, user: user) } + let(:other_user) { create(:user) } + let(:other_user_article) { create(:article, user: other_user) } + + def valid_instance(user) + described_class.new(user) + end + + def expected_fields + %w[ + body_markdown + cached_tag_list + cached_user_name + cached_user_username + canonical_url + comments_count + created_at + crossposted_at + description + edited_at + feed_source_url + language + last_comment_at + main_image + main_image_background_hex_color + path + positive_reactions_count + processed_html + published + published_at + published_from_feed + reactions_count + show_comments + slug + social_image + title + video + video_closed_caption_track_url + video_code + video_source_url + video_thumbnail_url + ] + end + + def load_articles(data) + JSON.parse(data["articles.json"]) + end + + describe "#initialize" do + it "accepts a user" do + exporter = valid_instance(user) + expect(exporter.user).to be(user) + end + + it "names itself articles" do + exporter = valid_instance(user) + expect(exporter.name).to eq(:articles) + end + end + + describe "#export" do + context "when slug is unknown" do + it "returns no articles if the slug is not found" do + exporter = valid_instance(user) + result = exporter.export(slug: "not found") + articles = load_articles(result) + expect(articles).to be_empty + end + + it "no articles if slug belongs to another user" do + exporter = valid_instance(user) + result = exporter.export(slug: other_user_article.slug) + articles = load_articles(result) + expect(articles).to be_empty + end + end + + context "when slug is known" do + it "returns the article" do + exporter = valid_instance(user) + result = exporter.export(slug: article.slug) + articles = load_articles(result) + expect(articles.length).to eq(1) + end + + it "returns only expected fields for the article" do + exporter = valid_instance(user) + result = exporter.export(slug: article.slug) + articles = load_articles(result) + expect(articles.first.keys).to eq(expected_fields) + end + end + + context "when all articles are requested" do + it "returns all the articles as json" do + exporter = valid_instance(article.user) + result = exporter.export + articles = load_articles(result) + user.reload + expect(articles.length).to eq(user.articles.size) + end + + it "returns only expected fields for the article" do + exporter = valid_instance(article.user) + result = exporter.export + articles = load_articles(result) + expect(articles.first.keys).to eq(expected_fields) + end + end + end +end diff --git a/spec/services/exporter/service_spec.rb b/spec/services/exporter/service_spec.rb new file mode 100644 index 0000000000000..587b6aec0ff7b --- /dev/null +++ b/spec/services/exporter/service_spec.rb @@ -0,0 +1,132 @@ +require "rails_helper" +require "zip" + +RSpec.describe Exporter::Service do + let(:user) { create(:user) } + let(:article) { create(:article, user: user) } + let(:other_user) { create(:user) } + let(:other_user_article) { create(:article, user: other_user) } + + before do + ActionMailer::Base.deliveries.clear + end + + def valid_instance(user) + described_class.new(user) + end + + def extract_zipped_exports(buffer) + exports = {} + + buffer.rewind + Zip::InputStream.open(buffer) do |stream| + loop do + entry = stream.get_next_entry + break if entry.blank? + exports[entry.name] = stream.read + end + end + + exports + end + + def expected_fields + [ + "body_html", + "body_markdown", + "cached_tag_list", + "cached_user_name", + "cached_user_username", + "canonical_url", + "comments_count", + "created_at", + "crossposted_at", + "description", + "edited_at", + "feed_source_url", + "language", + "last_comment_at", + "lat", + "long", + "main_image", + "main_image_background_hex_color", + "path", + "positive_reactions_count", + "processed_html", + "published", + "published_at", + "published_from_feed", + "reactions_count", + "show_comments", + "slug", + "social_image", + "title", + "updated_at", + "video", + "video_closed_caption_track_url", + "video_code", + "video_source_url", + "video_thumbnail_url", + ] + end + + describe "EXPORTERS" do + it "is a list of supported exporters" do + expect(described_class::EXPORTERS).to eq([Exporter::Articles]) + end + end + + describe "#initialize" do + it "accepts a user" do + service = valid_instance(user) + expect(service.user).to be(user) + end + end + + describe "#export" do + it "exports a zip file with files" do + service = valid_instance(article.user) + zipped_exports = service.export + exports = extract_zipped_exports(zipped_exports) + expect(exports.keys).to eq(["articles.json"]) + end + + it "passes configuration to an exporter" do + service = valid_instance(article.user) + zipped_exports = service.export(config: { articles: { slug: article.slug } }) + exports = extract_zipped_exports(zipped_exports) + expect(exports.length).to eq(1) + end + + context "when emailing the user" do + it "delivers one email" do + service = valid_instance(article.user) + service.export(send_email: true) + expect(ActionMailer::Base.deliveries.count).to eq(1) + end + + it "delivers an email with the export" do + service = valid_instance(article.user) + zipped_export = service.export(send_email: true) + attachment = ActionMailer::Base.deliveries.last.attachments[0].decoded + + exports = extract_zipped_exports(zipped_export) + expect(exports).to eq(extract_zipped_exports(StringIO.new(attachment))) + end + end + + it "sets the requested flag as false" do + service = valid_instance(article.user) + service.export + expect(user.export_requested).to be(false) + end + + it "sets the exported at datetime as the current one" do + Timecop.freeze(Time.current) do + service = valid_instance(article.user) + service.export + expect(user.exported_at).to eq(Time.current) + end + end + end +end