From 88a1d0cdb4df422594c3ffb335ad794df4e55ea7 Mon Sep 17 00:00:00 2001 From: Sam Schlinkert Date: Mon, 28 Jan 2019 16:57:42 -0500 Subject: [PATCH 01/42] Bumps copyright year in README.md to 2019 (#9939) This is so incredibly small, but assuming this is a needed change. Might want to check year in other files. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ad6894ca7332a..03c8472b9b17ab 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ You can open issues for bugs you've found or features you think are missing. You ## License -Copyright (C) 2016-2018 Eugen Rochko & other Mastodon contributors (see [AUTHORS.md](AUTHORS.md)) +Copyright (C) 2016-2019 Eugen Rochko & other Mastodon contributors (see [AUTHORS.md](AUTHORS.md)) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. From e31970b924a7eaa1279708919b2743a15fb099f0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 1 Feb 2019 00:15:38 +0100 Subject: [PATCH 02/42] Fix link color in high-contrast theme, add underlines (#9949) Improve sorting of default themes in the dropdown --- app/javascript/styles/contrast/diff.scss | 51 ++++++++++++++++++++++++ config/locales/en.yml | 6 +-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/app/javascript/styles/contrast/diff.scss b/app/javascript/styles/contrast/diff.scss index eee9ecc3ef7f27..7d8993a50802a1 100644 --- a/app/javascript/styles/contrast/diff.scss +++ b/app/javascript/styles/contrast/diff.scss @@ -12,3 +12,54 @@ } } } + +.status__content a, +.reply-indicator__content a { + color: lighten($ui-highlight-color, 12%); + text-decoration: underline; + + &.mention { + text-decoration: none; + } + + &.mention span { + text-decoration: underline; + + &:hover, + &:focus, + &:active { + text-decoration: none; + } + } + + &:hover, + &:focus, + &:active { + text-decoration: none; + } + + &.status__content__spoiler-link { + color: $secondary-text-color; + text-decoration: none; + } +} + +.status__content__read-more-button { + text-decoration: underline; + + &:hover, + &:focus, + &:active { + text-decoration: none; + } +} + +.getting-started__footer a { + text-decoration: underline; + + &:hover, + &:focus, + &:active { + text-decoration: none; + } +} diff --git a/config/locales/en.yml b/config/locales/en.yml index 10749b21b4d870..ba413f8fc2e6de 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -929,9 +929,9 @@ en:

Originally adapted from the Discourse privacy policy.

title: "%{instance} Terms of Service and Privacy Policy" themes: - contrast: High contrast - default: Mastodon - mastodon-light: Mastodon (light) + contrast: Mastodon (High contrast) + default: Mastodon (Dark) + mastodon-light: Mastodon (Light) time: formats: default: "%b %d, %Y, %H:%M" From 687a0cbcb036255d02aa1fab5132c51da2adc888 Mon Sep 17 00:00:00 2001 From: Clar Charr Date: Thu, 31 Jan 2019 07:45:15 -0500 Subject: [PATCH 03/42] Replace unlock-alt icon with unlock (#9952) --- app/helpers/stream_entries_helper.rb | 2 +- app/javascript/mastodon/components/account.js | 2 +- app/javascript/mastodon/components/domain.js | 2 +- app/javascript/mastodon/features/account/components/header.js | 2 +- .../mastodon/features/compose/components/privacy_dropdown.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb index 033d435c497809..7a74c0b7d058f1 100644 --- a/app/helpers/stream_entries_helper.rb +++ b/app/helpers/stream_entries_helper.rb @@ -170,7 +170,7 @@ def fa_visibility_icon(status) when 'public' fa_icon 'globe fw' when 'unlisted' - fa_icon 'unlock-alt fw' + fa_icon 'unlock fw' when 'private' fa_icon 'lock fw' when 'direct' diff --git a/app/javascript/mastodon/components/account.js b/app/javascript/mastodon/components/account.js index 206030c006bb34..2705a6001341bf 100644 --- a/app/javascript/mastodon/components/account.js +++ b/app/javascript/mastodon/components/account.js @@ -88,7 +88,7 @@ class Account extends ImmutablePureComponent { if (requested) { buttons = ; } else if (blocking) { - buttons = ; + buttons = ; } else if (muting) { let hidingNotificationsButton; if (account.getIn(['relationship', 'muting_notifications'])) { diff --git a/app/javascript/mastodon/components/domain.js b/app/javascript/mastodon/components/domain.js index 24f80e7888a61c..85729ca9438ea2 100644 --- a/app/javascript/mastodon/components/domain.js +++ b/app/javascript/mastodon/components/domain.js @@ -32,7 +32,7 @@ class Account extends ImmutablePureComponent {
- +
diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js index 2ab25cde44777c..4f6d0712d11054 100644 --- a/app/javascript/mastodon/features/account/components/header.js +++ b/app/javascript/mastodon/features/account/components/header.js @@ -132,7 +132,7 @@ class Header extends ImmutablePureComponent { } else if (account.getIn(['relationship', 'blocking'])) { actionBtn = (
- +
); } diff --git a/app/javascript/mastodon/features/compose/components/privacy_dropdown.js b/app/javascript/mastodon/features/compose/components/privacy_dropdown.js index 5698765d94f2b5..e8f57a4661b201 100644 --- a/app/javascript/mastodon/features/compose/components/privacy_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/privacy_dropdown.js @@ -214,7 +214,7 @@ class PrivacyDropdown extends React.PureComponent { this.options = [ { icon: 'globe', value: 'public', text: formatMessage(messages.public_short), meta: formatMessage(messages.public_long) }, - { icon: 'unlock-alt', value: 'unlisted', text: formatMessage(messages.unlisted_short), meta: formatMessage(messages.unlisted_long) }, + { icon: 'unlock', value: 'unlisted', text: formatMessage(messages.unlisted_short), meta: formatMessage(messages.unlisted_long) }, { icon: 'lock', value: 'private', text: formatMessage(messages.private_short), meta: formatMessage(messages.private_long) }, { icon: 'envelope', value: 'direct', text: formatMessage(messages.direct_short), meta: formatMessage(messages.direct_long) }, ]; From fdf819b83e820576164074b6726cb6ffdb4a47f6 Mon Sep 17 00:00:00 2001 From: Jakub Mendyk Date: Sat, 2 Feb 2019 19:01:18 +0100 Subject: [PATCH 04/42] Allow most kinds of characters in URL query (fixes #8408) (#8447) * Allow unicode characters in URL query strings Fixes #8408 * Alternative approach to unicode support in urls Adds PoC/idea to approch this problem. --- app/lib/formatter.rb | 39 +++++++++++++++++++++++++++++++++++++- spec/lib/formatter_spec.rb | 32 ++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 05fd9eeb1261ab..2e358716968a4b 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -99,7 +99,7 @@ def encode(html) end def encode_and_link_urls(html, accounts = nil, options = {}) - entities = Extractor.extract_entities_with_indices(html, extract_url_without_protocol: false) + entities = utf8_friendly_extractor(html, extract_url_without_protocol: false) if accounts.is_a?(Hash) options = accounts @@ -199,6 +199,43 @@ def rewrite(text, entities) result.flatten.join end + def utf8_friendly_extractor(text, options = {}) + old_to_new_index = [0] + + escaped = text.chars.map do |c| + output = c.ord.to_s(16).length > 2 ? CGI.escape(c) : c + old_to_new_index << old_to_new_index.last + output.length + output + end.join + + # Note: I couldn't obtain list_slug with @user/list-name format + # for mention so this requires additional check + special = Extractor.extract_entities_with_indices(escaped, options).map do |extract| + # exactly one of :url, :hashtag, :screen_name, :cashtag keys is present + key = (extract.keys & [:url, :hashtag, :screen_name, :cashtag]).first + + new_indices = [ + old_to_new_index.find_index(extract[:indices].first), + old_to_new_index.find_index(extract[:indices].last), + ] + + has_prefix_char = [:hashtag, :screen_name, :cashtag].include?(key) + value_indices = [ + new_indices.first + (has_prefix_char ? 1 : 0), # account for #, @ or $ + new_indices.last - 1, + ] + + next extract.merge( + :indices => new_indices, + key => text[value_indices.first..value_indices.last] + ) + end + + standard = Extractor.extract_entities_with_indices(text, options) + + Extractor.remove_overlapping_entities(special + standard) + end + def link_to_url(entity, options = {}) url = Addressable::URI.parse(entity[:url]) html_attrs = { target: '_blank', rel: 'nofollow noopener' } diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 0c1efe7c3cc763..9872d375679dc0 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -74,10 +74,36 @@ end context 'given a URL with a query string' do - let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink' } + context 'with escaped unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink' } - it 'matches the full URL' do - is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink"' + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink"' + end + end + + context 'with unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓&q=autolink' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓&q=autolink"' + end + end + + context 'with unicode character at the end' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓"' + end + end + + context 'with escaped and not escaped unicode characters' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink' } + + it 'preserves escaped unicode characters' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink"' + end end end From a742a09530b6bdca78713e14dbab51bc3a56d222 Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Tue, 5 Feb 2019 06:25:42 +0900 Subject: [PATCH 05/42] Fix authorized applications list page design (#9969) --- app/controllers/oauth/authorized_applications_controller.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb index 0c28d194bc82c8..f3d2353669464c 100644 --- a/app/controllers/oauth/authorized_applications_controller.rb +++ b/app/controllers/oauth/authorized_applications_controller.rb @@ -5,6 +5,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio before_action :store_current_location before_action :authenticate_resource_owner! + before_action :set_body_classes include Localized @@ -15,6 +16,10 @@ def destroy private + def set_body_classes + @body_classes = 'admin' + end + def store_current_location store_location_for(:user, request.url) end From 5e7c75cfd328af81a557985e1b5b2c70e6c68645 Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Tue, 5 Feb 2019 07:14:57 +0900 Subject: [PATCH 06/42] Fix not showing custom emojis in share page emoji picker (#9970) --- app/javascript/mastodon/containers/compose_container.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/javascript/mastodon/containers/compose_container.js b/app/javascript/mastodon/containers/compose_container.js index 5ee1d2f141ff6f..7bc7bbaa4dc29d 100644 --- a/app/javascript/mastodon/containers/compose_container.js +++ b/app/javascript/mastodon/containers/compose_container.js @@ -7,6 +7,7 @@ import { IntlProvider, addLocaleData } from 'react-intl'; import { getLocale } from '../locales'; import Compose from '../features/standalone/compose'; import initialState from '../initial_state'; +import { fetchCustomEmojis } from '../actions/custom_emojis'; const { localeData, messages } = getLocale(); addLocaleData(localeData); @@ -17,6 +18,8 @@ if (initialState) { store.dispatch(hydrateStore(initialState)); } +store.dispatch(fetchCustomEmojis()); + export default class TimelineContainer extends React.PureComponent { static propTypes = { From cd36ff43fd214b6b1447e88457b5a3f56461c1dd Mon Sep 17 00:00:00 2001 From: trwnh Date: Mon, 4 Feb 2019 21:46:18 -0600 Subject: [PATCH 07/42] [UI] Fix whitespace being applied to div instead of p (#9968) * fix large line breaks * fix ascii art posts --- app/javascript/styles/mastodon/components.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 10e094648197e6..32fd773859e758 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -638,7 +638,6 @@ font-weight: 400; overflow: hidden; text-overflow: ellipsis; - white-space: pre-wrap; padding-top: 2px; color: $primary-text-color; @@ -662,6 +661,7 @@ p { margin-bottom: 20px; + white-space: pre-wrap; &:last-child { margin-bottom: 0; From edde07f5ab235d6ceca95db66bb53161d372f830 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 5 Feb 2019 15:11:35 +0100 Subject: [PATCH 08/42] =?UTF-8?q?Hide=20misleading=20=E2=80=9CYou=20will?= =?UTF-8?q?=20be=20sent=20a=20confirmation=20e-mail=E2=80=9D=20hint=20from?= =?UTF-8?q?=20admin=20view=20(#9973)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks @wryk for noticing this issue. --- app/views/admin/change_emails/show.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/admin/change_emails/show.html.haml b/app/views/admin/change_emails/show.html.haml index 6febef9b1a811a..6ff0d785ed21c9 100644 --- a/app/views/admin/change_emails/show.html.haml +++ b/app/views/admin/change_emails/show.html.haml @@ -3,7 +3,7 @@ = simple_form_for @user, url: admin_account_change_email_path(@account.id) do |f| .fields-group - = f.input :email, wrapper: :with_label, disabled: true, label: t('admin.accounts.change_email.current_email') + = f.input :email, wrapper: :with_label, hint: false, disabled: true, label: t('admin.accounts.change_email.current_email') .fields-group = f.input :unconfirmed_email, wrapper: :with_label, label: t('admin.accounts.change_email.new_email') From e2afe5fdfba352b35731acf028bad54f32223282 Mon Sep 17 00:00:00 2001 From: abcang Date: Wed, 6 Feb 2019 10:50:52 +0900 Subject: [PATCH 09/42] Fix Tombstone.delete_all ArgumentError (#9978) --- app/services/activitypub/process_account_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 487456f3afcec2..5e33084282d93b 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -212,7 +212,7 @@ def key_changed? end def clear_tombstones! - Tombstone.delete_all(account_id: @account.id) + Tombstone.where(account_id: @account.id).delete_all end def protocol_changed? From 2a7c091eae68b06ae4ad7c566878a04f9926ac92 Mon Sep 17 00:00:00 2001 From: Hinaloe Date: Sat, 9 Feb 2019 11:39:38 +0900 Subject: [PATCH 10/42] Only URLs extract with pre-escaped text (#9991) * [test] add japanese hashtag testcase * Only URLs extract with pre-escaped text ( https://github.com/tootsuite/mastodon/issues/9989 ) --- app/lib/formatter.rb | 2 +- spec/lib/formatter_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 2e358716968a4b..6603b8df178c91 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -210,7 +210,7 @@ def utf8_friendly_extractor(text, options = {}) # Note: I couldn't obtain list_slug with @user/list-name format # for mention so this requires additional check - special = Extractor.extract_entities_with_indices(escaped, options).map do |extract| + special = Extractor.extract_urls_with_indices(escaped, options).map do |extract| # exactly one of :url, :hashtag, :screen_name, :cashtag keys is present key = (extract.keys & [:url, :hashtag, :screen_name, :cashtag]).first diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 9872d375679dc0..8fb6695a9bf12c 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -194,6 +194,14 @@ is_expected.to include '/tags/hashtag" class="mention hashtag" rel="tag">#hashtag' end end + + context 'given text containing a hashtag with Unicode chars' do + let(:text) { '#hashtagタグ' } + + it 'creates a hashtag link' do + is_expected.to include '/tags/hashtag%E3%82%BF%E3%82%B0" class="mention hashtag" rel="tag">#hashtagタグ' + end + end end describe '#format_spoiler' do From 6ea4cd5b86b0f96bfa9cfc904ee2c1adb48079a4 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 9 Feb 2019 20:13:11 +0100 Subject: [PATCH 11/42] Fix URL linkifier grabbing full-width spaces and quotations (#9997) Fix #9993 Fix #5654 --- app/lib/formatter.rb | 12 +++++++- config/initializers/twitter_regex.rb | 4 +-- spec/lib/formatter_spec.rb | 44 ++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 6603b8df178c91..0653214f53d693 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -199,12 +199,22 @@ def rewrite(text, entities) result.flatten.join end + UNICODE_ESCAPE_BLACKLIST_RE = /\p{Z}|\p{P}/ + def utf8_friendly_extractor(text, options = {}) old_to_new_index = [0] escaped = text.chars.map do |c| - output = c.ord.to_s(16).length > 2 ? CGI.escape(c) : c + output = begin + if c.ord.to_s(16).length > 2 && UNICODE_ESCAPE_BLACKLIST_RE.match(c).nil? + CGI.escape(c) + else + c + end + end + old_to_new_index << old_to_new_index.last + output.length + output end.join diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb index 0e8f5bfeb47353..0ddbbee9828bfd 100644 --- a/config/initializers/twitter_regex.rb +++ b/config/initializers/twitter_regex.rb @@ -1,7 +1,7 @@ module Twitter class Regex - REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}\(\)\?]/iou - REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*';:=\,\.\$%\[\]~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou + REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}<>\(\)\?]/iou + REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*"'「」<>;:=\,\.\$%\[\]~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou REGEXEN[:valid_url_balanced_parens] = / \( (?: diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 8fb6695a9bf12c..96d2fc7e06c247 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -115,6 +115,22 @@ end end + context 'given a URL in quotation marks' do + let(:text) { '"https://example.com/"' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in angle brackets' do + let(:text) { '' } + + it 'does not match the angle brackets' do + is_expected.to include 'href="https://example.com/"' + end + end + context 'given a URL with Japanese path string' do let(:text) { 'https://ja.wikipedia.org/wiki/日本' } @@ -131,6 +147,22 @@ end end + context 'given a URL with a full-width space' do + let(:text) { 'https://example.com/ abc123' } + + it 'does not match the full-width space' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in Japanese quotation marks' do + let(:text) { '「[https://example.org/」' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.org/"' + end + end + context 'given a URL with Simplified Chinese path string' do let(:text) { 'https://baike.baidu.com/item/中华人民共和国' } @@ -150,7 +182,11 @@ context 'given a URL containing unsafe code (XSS attack, visible part)' do let(:text) { %q{http://example.com/bb} } - it 'escapes the HTML in the URL' do + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/b"' + end + + it 'escapes the HTML' do is_expected.to include '<del>b</del>' end end @@ -158,7 +194,11 @@ context 'given a URL containing unsafe code (XSS attack, invisible part)' do let(:text) { %q{http://example.com/blahblahblahblah/a} } - it 'escapes the HTML in the URL' do + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/blahblahblahblah/a"' + end + + it 'escapes the HTML' do is_expected.to include '<script>alert("Hello")</script>' end end From d9f0c7fb841bebfa942ddc5b7aae2857eb1381e3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 6 Feb 2019 23:36:43 +0100 Subject: [PATCH 12/42] Fix IntersectionObserverArticle not hiding some out-of-view items (#9982) IntersectionObserverArticle is made to save on RAM by avoiding fully rendering items that are far out of view. However, it did not work for items spawned outside the intersection observer. --- .../mastodon/components/intersection_observer_article.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/components/intersection_observer_article.js b/app/javascript/mastodon/components/intersection_observer_article.js index de2203a4bc4c69..e453730ba4fccd 100644 --- a/app/javascript/mastodon/components/intersection_observer_article.js +++ b/app/javascript/mastodon/components/intersection_observer_article.js @@ -65,7 +65,7 @@ export default class IntersectionObserverArticle extends React.Component { } updateStateAfterIntersection = (prevState) => { - if (prevState.isIntersecting && !this.entry.isIntersecting) { + if (prevState.isIntersecting !== false && !this.entry.isIntersecting) { scheduleIdleTask(this.hideIfNotIntersecting); } return { From e1dbdf7377f20ace894ee92cf681542cc4b5eddb Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 11 Feb 2019 13:19:59 +0100 Subject: [PATCH 13/42] Fix timeline jumps (#10001) * Avoid two-step rendering of statuses as much as possible Cache width shared by Video player, MediaGallery and Cards at the ScrollableList level, pass it down through StatusList and Notifications. * Adjust scroll when new preview cards appear * Adjust scroll when statuses above the current scroll position are deleted --- .../mastodon/components/media_gallery.js | 10 ++- .../mastodon/components/scrollable_list.js | 28 +++++++- app/javascript/mastodon/components/status.js | 67 +++++++++++++++++-- .../notifications/components/notification.js | 32 ++++++++- .../features/status/components/card.js | 5 +- .../mastodon/features/video/index.js | 4 +- 6 files changed, 134 insertions(+), 12 deletions(-) diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js index c507920d0ee34a..a2bc9525564b4c 100644 --- a/app/javascript/mastodon/components/media_gallery.js +++ b/app/javascript/mastodon/components/media_gallery.js @@ -194,6 +194,8 @@ class MediaGallery extends React.PureComponent { height: PropTypes.number.isRequired, onOpenMedia: PropTypes.func.isRequired, intl: PropTypes.object.isRequired, + defaultWidth: PropTypes.number, + cacheWidth: PropTypes.func, }; static defaultProps = { @@ -202,6 +204,7 @@ class MediaGallery extends React.PureComponent { state = { visible: displayMedia !== 'hide_all' && !this.props.sensitive || displayMedia === 'show_all', + width: this.props.defaultWidth, }; componentWillReceiveProps (nextProps) { @@ -221,6 +224,7 @@ class MediaGallery extends React.PureComponent { handleRef = (node) => { if (node /*&& this.isStandaloneEligible()*/) { // offsetWidth triggers a layout, so only calculate when we need to + if (this.props.cacheWidth) this.props.cacheWidth(node.offsetWidth); this.setState({ width: node.offsetWidth, }); @@ -233,8 +237,10 @@ class MediaGallery extends React.PureComponent { } render () { - const { media, intl, sensitive, height } = this.props; - const { width, visible } = this.state; + const { media, intl, sensitive, height, defaultWidth } = this.props; + const { visible } = this.state; + + const width = this.state.width || defaultWidth; let children; diff --git a/app/javascript/mastodon/components/scrollable_list.js b/app/javascript/mastodon/components/scrollable_list.js index fec06e26373357..0376cf85accef9 100644 --- a/app/javascript/mastodon/components/scrollable_list.js +++ b/app/javascript/mastodon/components/scrollable_list.js @@ -40,6 +40,7 @@ export default class ScrollableList extends PureComponent { state = { fullscreen: null, + cachedMediaWidth: 250, // Default media/card width using default Mastodon theme }; intersectionObserverWrapper = new IntersectionObserverWrapper(); @@ -130,6 +131,20 @@ export default class ScrollableList extends PureComponent { this.handleScroll(); } + getScrollPosition = () => { + if (this.node && (this.node.scrollTop > 0 || this.mouseMovedRecently)) { + return { height: this.node.scrollHeight, top: this.node.scrollTop }; + } else { + return null; + } + } + + updateScrollBottom = (snapshot) => { + const newScrollTop = this.node.scrollHeight - snapshot; + + this.setScrollTop(newScrollTop); + } + getSnapshotBeforeUpdate (prevProps) { const someItemInserted = React.Children.count(prevProps.children) > 0 && React.Children.count(prevProps.children) < React.Children.count(this.props.children) && @@ -150,6 +165,12 @@ export default class ScrollableList extends PureComponent { } } + cacheMediaWidth = (width) => { + if (width && this.state.cachedMediaWidth !== width) { + this.setState({ cachedMediaWidth: width }); + } + } + componentWillUnmount () { this.clearMouseIdleTimer(); this.detachScrollListener(); @@ -239,7 +260,12 @@ export default class ScrollableList extends PureComponent { intersectionObserverWrapper={this.intersectionObserverWrapper} saveHeightKey={trackScroll ? `${this.context.router.route.location.key}:${scrollKey}` : null} > - {child} + {React.cloneElement(child, { + getScrollPosition: this.getScrollPosition, + updateScrollBottom: this.updateScrollBottom, + cachedMediaWidth: this.state.cachedMediaWidth, + cacheMediaWidth: this.cacheMediaWidth, + })} ))} diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index 20d838500967f3..fde8e60a423cbc 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -68,6 +68,10 @@ class Status extends ImmutablePureComponent { onMoveUp: PropTypes.func, onMoveDown: PropTypes.func, showThread: PropTypes.bool, + getScrollPosition: PropTypes.func, + updateScrollBottom: PropTypes.func, + cacheMediaWidth: PropTypes.func, + cachedMediaWidth: PropTypes.number, }; // Avoid checking props that are functions (and whose equality will always @@ -79,6 +83,43 @@ class Status extends ImmutablePureComponent { 'hidden', ]; + // Track height changes we know about to compensate scrolling + componentDidMount () { + this.didShowCard = !this.props.muted && !this.props.hidden && this.props.status.get('card'); + } + + getSnapshotBeforeUpdate () { + if (this.props.getScrollPosition) { + return this.props.getScrollPosition(); + } else { + return null; + } + } + + // Compensate height changes + componentDidUpdate (prevProps, prevState, snapshot) { + const doShowCard = !this.props.muted && !this.props.hidden && this.props.status.get('card'); + if (doShowCard && !this.didShowCard) { + this.didShowCard = true; + if (snapshot !== null && this.props.updateScrollBottom) { + if (this.node && this.node.offsetTop < snapshot.top) { + this.props.updateScrollBottom(snapshot.height - snapshot.top); + } + } + } + } + + componentWillUnmount() { + if (this.node && this.props.getScrollPosition) { + const position = this.props.getScrollPosition(); + if (position !== null && this.node.offsetTop < position.top) { + requestAnimationFrame(() => { + this.props.updateScrollBottom(position.height - position.top); + }); + } + } + } + handleClick = () => { if (this.props.onClick) { this.props.onClick(); @@ -165,6 +206,10 @@ class Status extends ImmutablePureComponent { } } + handleRef = c => { + this.node = c; + } + render () { let media = null; let statusAvatar, prepend, rebloggedByText; @@ -179,7 +224,7 @@ class Status extends ImmutablePureComponent { if (hidden) { return ( -
+
{status.getIn(['account', 'display_name']) || status.getIn(['account', 'username'])} {status.get('content')}
@@ -194,7 +239,7 @@ class Status extends ImmutablePureComponent { return ( -
+
@@ -242,11 +287,12 @@ class Status extends ImmutablePureComponent { preview={video.get('preview_url')} src={video.get('url')} alt={video.get('description')} - width={239} + width={this.props.cachedMediaWidth} height={110} inline sensitive={status.get('sensitive')} onOpenVideo={this.handleOpenVideo} + cacheWidth={this.props.cacheMediaWidth} /> )} @@ -254,7 +300,16 @@ class Status extends ImmutablePureComponent { } else { media = ( - {Component => } + {Component => ( + + )} ); } @@ -264,6 +319,8 @@ class Status extends ImmutablePureComponent { onOpenMedia={this.props.onOpenMedia} card={status.get('card')} compact + cacheWidth={this.props.cacheMediaWidth} + defaultWidth={this.props.cachedMediaWidth} /> ); } @@ -290,7 +347,7 @@ class Status extends ImmutablePureComponent { return ( -
+
{prepend}
diff --git a/app/javascript/mastodon/features/notifications/components/notification.js b/app/javascript/mastodon/features/notifications/components/notification.js index 97efff69c85208..24843a2b603f0c 100644 --- a/app/javascript/mastodon/features/notifications/components/notification.js +++ b/app/javascript/mastodon/features/notifications/components/notification.js @@ -34,6 +34,10 @@ class Notification extends ImmutablePureComponent { onToggleHidden: PropTypes.func.isRequired, status: PropTypes.option, intl: PropTypes.object.isRequired, + getScrollPosition: PropTypes.func, + updateScrollBottom: PropTypes.func, + cacheMediaWidth: PropTypes.func, + cachedMediaWidth: PropTypes.number, }; handleMoveUp = () => { @@ -128,6 +132,10 @@ class Notification extends ImmutablePureComponent { onMoveDown={this.handleMoveDown} onMoveUp={this.handleMoveUp} contextType='notifications' + getScrollPosition={this.props.getScrollPosition} + updateScrollBottom={this.props.updateScrollBottom} + cachedMediaWidth={this.props.cachedMediaWidth} + cacheMediaWidth={this.props.cacheMediaWidth} /> ); } @@ -148,7 +156,17 @@ class Notification extends ImmutablePureComponent {
-
); @@ -170,7 +188,17 @@ class Notification extends ImmutablePureComponent {
-
); diff --git a/app/javascript/mastodon/features/status/components/card.js b/app/javascript/mastodon/features/status/components/card.js index 8491299ef49c68..30ed3fc2103839 100644 --- a/app/javascript/mastodon/features/status/components/card.js +++ b/app/javascript/mastodon/features/status/components/card.js @@ -60,6 +60,8 @@ export default class Card extends React.PureComponent { maxDescription: PropTypes.number, onOpenMedia: PropTypes.func.isRequired, compact: PropTypes.bool, + defaultWidth: PropTypes.number, + cacheWidth: PropTypes.func, }; static defaultProps = { @@ -68,7 +70,7 @@ export default class Card extends React.PureComponent { }; state = { - width: 280, + width: this.props.defaultWidth || 280, embedded: false, }; @@ -111,6 +113,7 @@ export default class Card extends React.PureComponent { setRef = c => { if (c) { + if (this.props.cacheWidth) this.props.cacheWidth(c.offsetWidth); this.setState({ width: c.offsetWidth }); } } diff --git a/app/javascript/mastodon/features/video/index.js b/app/javascript/mastodon/features/video/index.js index 0d0c24d7194ee8..823bc25ab710ab 100644 --- a/app/javascript/mastodon/features/video/index.js +++ b/app/javascript/mastodon/features/video/index.js @@ -99,6 +99,7 @@ class Video extends React.PureComponent { onCloseVideo: PropTypes.func, detailed: PropTypes.bool, inline: PropTypes.bool, + cacheWidth: PropTypes.func, intl: PropTypes.object.isRequired, }; @@ -108,7 +109,7 @@ class Video extends React.PureComponent { volume: 0.5, paused: true, dragging: false, - containerWidth: false, + containerWidth: this.props.width, fullscreen: false, hovered: false, muted: false, @@ -128,6 +129,7 @@ class Video extends React.PureComponent { this.player = c; if (c) { + if (this.props.cacheWidth) this.props.cacheWidth(this.player.offsetWidth); this.setState({ containerWidth: c.offsetWidth, }); From 41ecf80645d465d67dfee54d3cbb14a825ed8953 Mon Sep 17 00:00:00 2001 From: Hinaloe Date: Tue, 12 Feb 2019 13:10:31 +0900 Subject: [PATCH 14/42] Don't focus spiler input when disabled spoiler (#10017) --- .../mastodon/features/compose/components/compose_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/features/compose/components/compose_form.js b/app/javascript/mastodon/features/compose/components/compose_form.js index ac458fd2592a79..294dcb5c18b610 100644 --- a/app/javascript/mastodon/features/compose/components/compose_form.js +++ b/app/javascript/mastodon/features/compose/components/compose_form.js @@ -179,7 +179,7 @@ class ComposeForm extends ImmutablePureComponent {
From d66267508ac316ffc2756a6677fcbae5a44605c2 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 12 Feb 2019 05:10:43 +0100 Subject: [PATCH 15/42] =?UTF-8?q?Move=20sending=20account=20Delete=20to=20?= =?UTF-8?q?anyone=20but=20the=20account's=20followers=20to=20the=20pull?= =?UTF-8?q?=CC=80=20queue=20(#10016)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/suspend_account_service.rb | 10 +++++++++- .../activitypub/low_priority_delivery_worker.rb | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 app/workers/activitypub/low_priority_delivery_worker.rb diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 1bc2314de7156e..fc3bc03a532db3 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -102,6 +102,10 @@ def distribute_delete_actor! ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end + + ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes) do |inbox_url| + [delete_actor_json, @account.id, inbox_url] + end end def delete_actor_json @@ -117,7 +121,11 @@ def delete_actor_json end def delivery_inboxes - Account.inboxes + Relay.enabled.pluck(:inbox_url) + @delivery_inboxes ||= @account.followers.inboxes + Relay.enabled.pluck(:inbox_url) + end + + def low_priority_delivery_inboxes + Account.inboxes - delivery_inboxes end def associations_for_destruction diff --git a/app/workers/activitypub/low_priority_delivery_worker.rb b/app/workers/activitypub/low_priority_delivery_worker.rb new file mode 100644 index 00000000000000..a141b8f7809da1 --- /dev/null +++ b/app/workers/activitypub/low_priority_delivery_worker.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class ActivityPub::LowPriorityDeliveryWorker < ActivityPub::DeliveryWorker + sidekiq_options queue: 'pull', retry: 8, dead: false +end From 27310a84a43623da8f08832e30e078ce936f6a1d Mon Sep 17 00:00:00 2001 From: Franck Zoccolo Date: Tue, 12 Feb 2019 14:48:04 +0100 Subject: [PATCH 16/42] Add support for IPv6 only MXes in Email validation (#10009) * Add support for IPv6 only MXes * Fixed email validator tests --- app/validators/email_mx_validator.rb | 1 + spec/validators/email_mx_validator_spec.rb | 38 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/app/validators/email_mx_validator.rb b/app/validators/email_mx_validator.rb index 5b4c684b2c4b44..96fbedcfcf9b81 100644 --- a/app/validators/email_mx_validator.rb +++ b/app/validators/email_mx_validator.rb @@ -24,6 +24,7 @@ def invalid_mx?(value) ([domain] + hostnames).uniq.each do |hostname| ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s }) + ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s }) end end diff --git a/spec/validators/email_mx_validator_spec.rb b/spec/validators/email_mx_validator_spec.rb index bc68f63cfbc16f..48e17a4f1086ef 100644 --- a/spec/validators/email_mx_validator_spec.rb +++ b/spec/validators/email_mx_validator_spec.rb @@ -11,6 +11,7 @@ allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -23,7 +24,9 @@ allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -37,6 +40,21 @@ allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '1.2.3.4')]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:timeouts=).and_return(nil) + allow(Resolv::DNS).to receive(:open).and_yield(resolver) + + subject.validate(user) + expect(user.errors).to have_received(:add) + end + + it 'adds an error if the AAAA record is blacklisted' do + EmailDomainBlock.create!(domain: 'fd00::1') + resolver = double + + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::1')]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -50,7 +68,25 @@ allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '2.3.4.5')]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:timeouts=).and_return(nil) + allow(Resolv::DNS).to receive(:open).and_yield(resolver) + + subject.validate(user) + expect(user.errors).to have_received(:add) + end + + it 'adds an error if the MX IPv6 record is blacklisted' do + EmailDomainBlock.create!(domain: 'fd00::2') + resolver = double + + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::2')]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -64,7 +100,9 @@ allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '2.3.4.5')]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::2')]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) From 1186b9abebc0b92acecebcd020d2f9c17de5d0b3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 12 Feb 2019 22:24:14 +0100 Subject: [PATCH 17/42] Save IP address used for sign-up, not only sign-in (#10026) Fixes #9995 --- app/controllers/auth/registrations_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index f2a832542fe548..ad7b1859f68247 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -28,6 +28,7 @@ def build_resource(hash = nil) resource.invite_code = params[:invite_code] if resource.invite_code.blank? resource.agreement = true + resource.current_sign_in_ip = request.remote_ip if resource.current_sign_in_ip.nil? resource.build_account if resource.account.nil? end From f0f657e77c65923e6d77b5f62f7ee8544b4d9e00 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 05:30:49 +0100 Subject: [PATCH 18/42] Fix color of static page links in high contrast theme (#10028) --- app/javascript/styles/contrast/diff.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/javascript/styles/contrast/diff.scss b/app/javascript/styles/contrast/diff.scss index 7d8993a50802a1..8429103b8aff1e 100644 --- a/app/javascript/styles/contrast/diff.scss +++ b/app/javascript/styles/contrast/diff.scss @@ -13,6 +13,10 @@ } } +.rich-formatting a, +.rich-formatting p a, +.rich-formatting li a, +.landing-page__short-description p a, .status__content a, .reply-indicator__content a { color: lighten($ui-highlight-color, 12%); From a46487e895fbba23922888820c6b88ad07ebe56b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 18:34:58 +0100 Subject: [PATCH 19/42] Fix hashtags select styling in default and high contrast themes (#10029) --- .../components/column_settings.js | 55 ++++++----- app/javascript/styles/mastodon/_mixins.scss | 31 ++++++ .../styles/mastodon/components.scss | 95 +++++++++++++------ 3 files changed, 128 insertions(+), 53 deletions(-) diff --git a/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js b/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js index 9c9f62d8211ebc..cdc138c8bf3a44 100644 --- a/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js +++ b/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js @@ -1,10 +1,15 @@ import React from 'react'; import PropTypes from 'prop-types'; import ImmutablePropTypes from 'react-immutable-proptypes'; -import { injectIntl, FormattedMessage } from 'react-intl'; +import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; import Toggle from 'react-toggle'; import AsyncSelect from 'react-select/lib/Async'; +const messages = defineMessages({ + placeholder: { id: 'hashtag.column_settings.select.placeholder', defaultMessage: 'Enter hashtags…' }, + noOptions: { id: 'hashtag.column_settings.select.no_options_message', defaultMessage: 'No suggestions found' }, +}); + export default @injectIntl class ColumnSettings extends React.PureComponent { @@ -25,6 +30,7 @@ class ColumnSettings extends React.PureComponent { tags (mode) { let tags = this.props.settings.getIn(['tags', mode]) || []; + if (tags.toJSON) { return tags.toJSON(); } else { @@ -32,33 +38,36 @@ class ColumnSettings extends React.PureComponent { } }; - onSelect = (mode) => { - return (value) => { - this.props.onChange(['tags', mode], value); - }; - }; + onSelect = mode => value => this.props.onChange(['tags', mode], value); onToggle = () => { if (this.state.open && this.hasTags()) { this.props.onChange('tags', {}); } + this.setState({ open: !this.state.open }); }; + noOptionsMessage = () => this.props.intl.formatMessage(messages.noOptions); + modeSelect (mode) { return ( -
- {this.modeLabel(mode)} +
+ + {this.modeLabel(mode)} + +
); @@ -66,11 +75,15 @@ class ColumnSettings extends React.PureComponent { modeLabel (mode) { switch(mode) { - case 'any': return ; - case 'all': return ; - case 'none': return ; + case 'any': + return ; + case 'all': + return ; + case 'none': + return ; + default: + return ''; } - return ''; }; render () { @@ -78,23 +91,21 @@ class ColumnSettings extends React.PureComponent {
- + +
- {this.state.open && + + {this.state.open && (
{this.modeSelect('any')} {this.modeSelect('all')} {this.modeSelect('none')}
- } + )}
); } diff --git a/app/javascript/styles/mastodon/_mixins.scss b/app/javascript/styles/mastodon/_mixins.scss index d5bafe6b6a2e7a..08806599e8f498 100644 --- a/app/javascript/styles/mastodon/_mixins.scss +++ b/app/javascript/styles/mastodon/_mixins.scss @@ -41,3 +41,34 @@ font-size: 16px; } } + +@mixin search-popout() { + background: $simple-background-color; + border-radius: 4px; + padding: 10px 14px; + padding-bottom: 14px; + margin-top: 10px; + color: $light-text-color; + box-shadow: 2px 4px 15px rgba($base-shadow-color, 0.4); + + h4 { + text-transform: uppercase; + color: $light-text-color; + font-size: 13px; + font-weight: 500; + margin-bottom: 10px; + } + + li { + padding: 4px 0; + } + + ul { + margin-bottom: 10px; + } + + em { + font-weight: 500; + color: $inverted-text-color; + } +} diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 32fd773859e758..02bbd345a7b87c 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -3056,14 +3056,41 @@ a.status-card.compact:hover { display: block; font-weight: 500; margin-bottom: 10px; +} + +.column-settings__hashtags { + .column-settings__row { + margin-bottom: 15px; + } - .column-settings__hashtag-select { + .column-select { &__control { @include search-input(); } + &__placeholder { + color: $dark-text-color; + padding-left: 2px; + font-size: 12px; + } + + &__value-container { + padding-left: 6px; + } + &__multi-value { background: lighten($ui-base-color, 8%); + + &__remove { + cursor: pointer; + + &:hover, + &:active, + &:focus { + background: lighten($ui-base-color, 12%); + color: lighten($darker-text-color, 4%); + } + } } &__multi-value__label, @@ -3071,9 +3098,42 @@ a.status-card.compact:hover { color: $darker-text-color; } - &__indicator-separator, + &__clear-indicator, &__dropdown-indicator { - display: none; + cursor: pointer; + transition: none; + color: $dark-text-color; + + &:hover, + &:active, + &:focus { + color: lighten($dark-text-color, 4%); + } + } + + &__indicator-separator { + background-color: lighten($ui-base-color, 8%); + } + + &__menu { + @include search-popout(); + padding: 0; + background: $ui-secondary-color; + } + + &__menu-list { + padding: 6px; + } + + &__option { + color: $inverted-text-color; + border-radius: 4px; + font-size: 14px; + + &--is-focused, + &--is-selected { + background: darken($ui-secondary-color, 10%); + } } } } @@ -4867,34 +4927,7 @@ a.status-card.compact:hover { } .search-popout { - background: $simple-background-color; - border-radius: 4px; - padding: 10px 14px; - padding-bottom: 14px; - margin-top: 10px; - color: $light-text-color; - box-shadow: 2px 4px 15px rgba($base-shadow-color, 0.4); - - h4 { - text-transform: uppercase; - color: $light-text-color; - font-size: 13px; - font-weight: 500; - margin-bottom: 10px; - } - - li { - padding: 4px 0; - } - - ul { - margin-bottom: 10px; - } - - em { - font-weight: 500; - color: $inverted-text-color; - } + @include search-popout(); } noscript { From 2a1adab7d7824df9fa148a9431e942d6677c1d71 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 18:36:40 +0100 Subject: [PATCH 20/42] Fix style regressions on landing page (#10030) --- app/javascript/styles/mastodon/about.scss | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/app/javascript/styles/mastodon/about.scss b/app/javascript/styles/mastodon/about.scss index b6c92a09e7dac6..b078d4d24d5def 100644 --- a/app/javascript/styles/mastodon/about.scss +++ b/app/javascript/styles/mastodon/about.scss @@ -49,15 +49,9 @@ $small-breakpoint: 960px; } } + strong, em { - display: inline; - margin: 0; - padding: 0; font-weight: 700; - background: transparent; - font-family: inherit; - font-size: inherit; - line-height: inherit; color: lighten($darker-text-color, 10%); } @@ -796,7 +790,7 @@ $small-breakpoint: 960px; width: 100%; display: flex; flex-direction: row-reverse; - flex-wrap: wrap; + flex-wrap: nowrap; justify-content: space-between; align-items: center; } @@ -846,14 +840,7 @@ $small-breakpoint: 960px; } strong { - display: inline; - margin: 0; - padding: 0; - font-weight: 700; - background: transparent; - font-family: inherit; - font-size: inherit; - line-height: inherit; + font-weight: 500; color: lighten($darker-text-color, 10%); } From 5a04861c7f2e98c80f315a19d7eadade044d8aae Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 14 Feb 2019 06:27:54 +0100 Subject: [PATCH 21/42] Add tight rate-limit for API deletions (#10042) Deletions take a lot of resources to execute and cause a lot of federation traffic, so it makes sense to decrease the number someone can queue up through the API. 30 per 30 minutes --- config/initializers/rack_attack.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 35302e37b1be76..28201cc64c803d 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -46,14 +46,14 @@ def web_request? end throttle('throttle_authenticated_api', limit: 300, period: 5.minutes) do |req| - req.api_request? && req.authenticated_user_id + req.authenticated_user_id if req.api_request? end throttle('throttle_unauthenticated_api', limit: 7_500, period: 5.minutes) do |req| req.ip if req.api_request? end - throttle('throttle_media', limit: 30, period: 30.minutes) do |req| + throttle('throttle_api_media', limit: 30, period: 30.minutes) do |req| req.authenticated_user_id if req.post? && req.path.start_with?('/api/v1/media') end @@ -61,6 +61,13 @@ def web_request? req.ip if req.post? && req.path == '/api/v1/accounts' end + API_DELETE_REBLOG_REGEX = /\A\/api\/v1\/statuses\/[\d]+\/unreblog/.freeze + API_DELETE_STATUS_REGEX = /\A\/api\/v1\/statuses\/[\d]+/.freeze + + throttle('throttle_api_delete', limit: 30, period: 30.minutes) do |req| + req.authenticated_user_id if (req.post? && req.path =~ API_DELETE_REBLOG_REGEX) || (req.delete? && req.path =~ API_DELETE_STATUS_REGEX) + end + throttle('protected_paths', limit: 25, period: 5.minutes) do |req| req.ip if req.post? && req.path =~ PROTECTED_PATHS_REGEX end From 17a41e1f779874a270f531086f5576affcdb6cb1 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 14 Feb 2019 15:46:42 +0100 Subject: [PATCH 22/42] Fix hashtag column not subscribing to stream on mount (#10040) Fix #9895 --- .../mastodon/features/hashtag_timeline/index.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/javascript/mastodon/features/hashtag_timeline/index.js b/app/javascript/mastodon/features/hashtag_timeline/index.js index c2e026d13611d9..0d3c97a6489cc9 100644 --- a/app/javascript/mastodon/features/hashtag_timeline/index.js +++ b/app/javascript/mastodon/features/hashtag_timeline/index.js @@ -41,15 +41,19 @@ class HashtagTimeline extends React.PureComponent { title = () => { let title = [this.props.params.id]; + if (this.additionalFor('any')) { - title.push(' ', ); + title.push(' ', ); } + if (this.additionalFor('all')) { - title.push(' ', ); + title.push(' ', ); } + if (this.additionalFor('none')) { - title.push(' ', ); + title.push(' ', ); } + return title; } @@ -77,9 +81,10 @@ class HashtagTimeline extends React.PureComponent { let all = (tags.all || []).map(tag => tag.value); let none = (tags.none || []).map(tag => tag.value); - [id, ...any].map((tag) => { - this.disconnects.push(dispatch(connectHashtagStream(id, tag, (status) => { + [id, ...any].map(tag => { + this.disconnects.push(dispatch(connectHashtagStream(id, tag, status => { let tags = status.tags.map(tag => tag.name); + return all.filter(tag => tags.includes(tag)).length === all.length && none.filter(tag => tags.includes(tag)).length === 0; }))); @@ -95,12 +100,14 @@ class HashtagTimeline extends React.PureComponent { const { dispatch } = this.props; const { id, tags } = this.props.params; + this._subscribe(dispatch, id, tags); dispatch(expandHashtagTimeline(id, { tags })); } componentWillReceiveProps (nextProps) { const { dispatch, params } = this.props; const { id, tags } = nextProps.params; + if (id !== params.id || !isEqual(tags, params.tags)) { this._unsubscribe(); this._subscribe(dispatch, id, tags); From 737ac4b59df77e43bed71abc6de80bc89f893de1 Mon Sep 17 00:00:00 2001 From: ysksn Date: Sun, 3 Feb 2019 03:11:38 +0900 Subject: [PATCH 23/42] Create Redisable#redis (#9633) * Create Redisable * Use #redis instead of Redis.current --- app/lib/activity_tracker.rb | 6 ++---- app/lib/activitypub/activity.rb | 5 +---- app/lib/feed_manager.rb | 9 +++------ app/lib/ostatus/activity/base.rb | 6 ++---- app/lib/potential_friendship_tracker.rb | 8 ++------ app/models/concerns/redisable.rb | 11 +++++++++++ app/models/feed.rb | 6 ++---- app/models/trending_tags.rb | 6 ++---- app/services/batched_remove_status_service.rb | 5 +---- app/services/follow_service.rb | 6 ++---- app/services/post_status_service.rb | 6 ++---- app/services/remove_status_service.rb | 19 ++++++++----------- .../scheduler/feed_cleanup_scheduler.rb | 5 +---- 13 files changed, 39 insertions(+), 59 deletions(-) create mode 100644 app/models/concerns/redisable.rb diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb index 5b497267477aff..ae3c11b6af5c99 100644 --- a/app/lib/activity_tracker.rb +++ b/app/lib/activity_tracker.rb @@ -4,6 +4,8 @@ class ActivityTracker EXPIRE_AFTER = 90.days.seconds class << self + include Redisable + def increment(prefix) key = [prefix, current_week].join(':') @@ -20,10 +22,6 @@ def record(prefix, value) private - def redis - Redis.current - end - def current_week Time.zone.today.cweek end diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 87318fb1c5c33b..919678618a3168 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -2,6 +2,7 @@ class ActivityPub::Activity include JsonLdHelper + include Redisable def initialize(json, account, **options) @json = json @@ -70,10 +71,6 @@ def object_uri @object_uri ||= value_or_id(@object) end - def redis - Redis.current - end - def distribute(status) crawl_links(status) diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index f99df33e54f0d2..d77cdb3a3cc206 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -4,6 +4,7 @@ class FeedManager include Singleton + include Redisable MAX_ITEMS = 400 @@ -35,7 +36,7 @@ def push_to_home(account, status) def unpush_from_home(account, status) return false unless remove_from_feed(:home, account.id, status) - Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end @@ -53,7 +54,7 @@ def push_to_list(list, status) def unpush_from_list(list, status) return false unless remove_from_feed(:list, list.id, status) - Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end @@ -142,10 +143,6 @@ def populate_feed(account) private - def redis - Redis.current - end - def push_update_required?(timeline_id) redis.exists("subscribed:#{timeline_id}") end diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb index c5933f3adfecfe..db70f19980cd58 100644 --- a/app/lib/ostatus/activity/base.rb +++ b/app/lib/ostatus/activity/base.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class OStatus::Activity::Base + include Redisable + def initialize(xml, account = nil, **options) @xml = xml @account = account @@ -66,8 +68,4 @@ def find_activitypub_status(uri, href) Status.find_by(uri: uri) end end - - def redis - Redis.current - end end diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb index 017a9748d5f979..188aa4a275657e 100644 --- a/app/lib/potential_friendship_tracker.rb +++ b/app/lib/potential_friendship_tracker.rb @@ -11,6 +11,8 @@ class PotentialFriendshipTracker }.freeze class << self + include Redisable + def record(account_id, target_account_id, action) return if account_id == target_account_id @@ -31,11 +33,5 @@ def get(account_id, limit: 20, offset: 0) return [] if account_ids.empty? Account.searchable.where(id: account_ids) end - - private - - def redis - Redis.current - end end end diff --git a/app/models/concerns/redisable.rb b/app/models/concerns/redisable.rb new file mode 100644 index 00000000000000..c6cf9735960c1c --- /dev/null +++ b/app/models/concerns/redisable.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Redisable + extend ActiveSupport::Concern + + private + + def redis + Redis.current + end +end diff --git a/app/models/feed.rb b/app/models/feed.rb index 5bce88f25522df..0e8943ff863bde 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Feed + include Redisable + def initialize(type, id) @type = type @id = id @@ -27,8 +29,4 @@ def from_redis(limit, max_id, since_id, min_id) def key FeedManager.instance.key(@type, @id) end - - def redis - Redis.current - end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index 3a8be21649b936..148535c2121e0a 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -7,6 +7,8 @@ class TrendingTags THRESHOLD = 5 class << self + include Redisable + def record_use!(tag, account, at_time = Time.now.utc) return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot? @@ -59,9 +61,5 @@ def disallowed_hashtags @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String @disallowed_hashtags = @disallowed_hashtags.map(&:downcase) end - - def redis - Redis.current - end end end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 2e61904fc770f6..cd3b14e08e34ef 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -2,6 +2,7 @@ class BatchedRemoveStatusService < BaseService include StreamEntryRenderer + include Redisable # Delete given statuses and reblogs of them # Dispatch PuSH updates of the deleted statuses, but only local ones @@ -109,10 +110,6 @@ def batch_salmon_slaps(status) end end - def redis - Redis.current - end - def build_xml(stream_entry) return @activity_xml[stream_entry.id] if @activity_xml.key?(stream_entry.id) diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index 9d36a1449dff37..92d8c864a7de1e 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class FollowService < BaseService + include Redisable + # Follow a remote user, notify remote user about the follow # @param [Account] source_account From which to follow # @param [String, Account] uri User URI to follow in the form of username@domain (or account record) @@ -67,10 +69,6 @@ def direct_follow(source_account, target_account, reblogs: true) follow end - def redis - Redis.current - end - def build_follow_request_xml(follow_request) OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_request_salmon(follow_request)) end diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 9959bb1fbf7d02..686b10c5817c3b 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class PostStatusService < BaseService + include Redisable + MIN_SCHEDULE_OFFSET = 5.minutes.freeze # Post a text status update, fetch and notify remote users mentioned @@ -110,10 +112,6 @@ def process_hashtags_service ProcessHashtagsService.new end - def redis - Redis.current - end - def scheduled? @scheduled_at.present? end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 11d28e783d04c8..28c5224b01184f 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -2,6 +2,7 @@ class RemoveStatusService < BaseService include StreamEntryRenderer + include Redisable def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @@ -55,7 +56,7 @@ def remove_from_lists def remove_from_affected @mentions.map(&:account).select(&:local?).each do |account| - Redis.current.publish("timeline:#{account.id}", @payload) + redis.publish("timeline:#{account.id}", @payload) end end @@ -133,26 +134,22 @@ def remove_from_hashtags return unless @status.public_visibility? @tags.each do |hashtag| - Redis.current.publish("timeline:hashtag:#{hashtag}", @payload) - Redis.current.publish("timeline:hashtag:#{hashtag}:local", @payload) if @status.local? + redis.publish("timeline:hashtag:#{hashtag}", @payload) + redis.publish("timeline:hashtag:#{hashtag}:local", @payload) if @status.local? end end def remove_from_public return unless @status.public_visibility? - Redis.current.publish('timeline:public', @payload) - Redis.current.publish('timeline:public:local', @payload) if @status.local? + redis.publish('timeline:public', @payload) + redis.publish('timeline:public:local', @payload) if @status.local? end def remove_from_media return unless @status.public_visibility? - Redis.current.publish('timeline:public:media', @payload) - Redis.current.publish('timeline:public:local:media', @payload) if @status.local? - end - - def redis - Redis.current + redis.publish('timeline:public:media', @payload) + redis.publish('timeline:public:local:media', @payload) if @status.local? end end diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb index cd22734180572c..bf5e20757007d8 100644 --- a/app/workers/scheduler/feed_cleanup_scheduler.rb +++ b/app/workers/scheduler/feed_cleanup_scheduler.rb @@ -2,6 +2,7 @@ class Scheduler::FeedCleanupScheduler include Sidekiq::Worker + include Redisable sidekiq_options unique: :until_executed, retry: 0 @@ -57,8 +58,4 @@ def inactive_list_ids def feed_manager FeedManager.instance end - - def redis - Redis.current - end end From 6c11f0f8cf91aa4d0b83a0a9c5a92cfa99dcbfdd Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 13 Feb 2019 18:36:23 +0100 Subject: [PATCH 24/42] Alternative handling of private self-boosts (#9998) * When self-boosting, embed original toot into Announce serialization * Process unknown self-boosts from Announce object if it is more than an URI * Add some self-boost specs * Only serialize private toots in self-Announces --- app/lib/activitypub/activity.rb | 32 +++++++++++ app/lib/activitypub/activity/announce.rb | 4 +- app/lib/activitypub/activity/create.rb | 15 ------ .../activitypub/activity_serializer.rb | 8 ++- .../lib/activitypub/activity/announce_spec.rb | 53 ++++++++++++++++--- 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 919678618a3168..7e4e195313a655 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -4,6 +4,9 @@ class ActivityPub::Activity include JsonLdHelper include Redisable + SUPPORTED_TYPES = %w(Note).freeze + CONVERTED_TYPES = %w(Image Video Article Page).freeze + def initialize(json, account, **options) @json = json @account = account @@ -71,6 +74,18 @@ def object_uri @object_uri ||= value_or_id(@object) end + def unsupported_object_type? + @object.is_a?(String) || !(supported_object_type? || converted_object_type?) + end + + def supported_object_type? + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) + end + + def converted_object_type? + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) + end + def distribute(status) crawl_links(status) @@ -120,6 +135,23 @@ def delete_later!(uri) redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri) end + def status_from_object + # If the status is already known, return it + status = status_from_uri(object_uri) + return status unless status.nil? + + # If the boosted toot is embedded and it is a self-boost, handle it like a Create + unless unsupported_object_type? + actor_id = value_or_id(first_of_value(@object['attributedTo'])) || @account.uri + if actor_id == @account.uri + return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform + end + end + + # If the status is not from the actor, try to fetch it + return fetch_remote_original_status if value_or_id(first_of_value(@json['attributedTo'])) == @account.uri + end + def fetch_remote_original_status if object_uri.start_with?('http') return if ActivityPub::TagManager.instance.local_uri?(object_uri) diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 34d1b7cbd0fe1f..04afeea202c34f 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -2,9 +2,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform - original_status = status_from_uri(object_uri) - original_status ||= fetch_remote_original_status - + original_status = status_from_object return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) status = Status.find_by(account: @account, reblog: original_status) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index b49657d4b14625..9a3db51dd9e6bd 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -1,9 +1,6 @@ # frozen_string_literal: true class ActivityPub::Activity::Create < ActivityPub::Activity - SUPPORTED_TYPES = %w(Note).freeze - CONVERTED_TYPES = %w(Image Video Article Page).freeze - def perform return if unsupported_object_type? || invalid_origin?(@object['id']) return if Tombstone.exists?(uri: @object['id']) @@ -318,22 +315,10 @@ def name_language_map? @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty? end - def unsupported_object_type? - @object.is_a?(String) || !(supported_object_type? || converted_object_type?) - end - def unsupported_media_type?(mime_type) mime_type.present? && !(MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES).include?(mime_type) end - def supported_object_type? - equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) - end - - def converted_object_type? - equals_or_includes_any?(@object['type'], CONVERTED_TYPES) - end - def skip_download? return @skip_download if defined?(@skip_download) @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media? diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb index 50c4f6a049ac43..b51e8c54429bb6 100644 --- a/app/serializers/activitypub/activity_serializer.rb +++ b/app/serializers/activitypub/activity_serializer.rb @@ -3,8 +3,8 @@ class ActivityPub::ActivitySerializer < ActiveModel::Serializer attributes :id, :type, :actor, :published, :to, :cc - has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, unless: :announce? - attribute :proper_uri, key: :object, if: :announce? + has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, unless: :owned_announce? + attribute :proper_uri, key: :object, if: :owned_announce? attribute :atom_uri, if: :announce? def id @@ -42,4 +42,8 @@ def atom_uri def announce? object.reblog? end + + def owned_announce? + announce? && object.account == object.proper.account && object.proper.private_visibility? + end end diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 54dd52a60490a1..1725c2843bb7ea 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Announce do - let(:sender) { Fabricate(:account) } + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers') } let(:recipient) { Fabricate(:account) } let(:status) { Fabricate(:status, account: recipient) } @@ -11,19 +11,60 @@ id: 'foo', type: 'Announce', actor: ActivityPub::TagManager.instance.uri_for(sender), - object: ActivityPub::TagManager.instance.uri_for(status), + object: object_json, }.with_indifferent_access end - describe '#perform' do - subject { described_class.new(json, sender) } + subject { described_class.new(json, sender) } + + before do + sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) + end + describe '#perform' do before do subject.perform end - it 'creates a reblog by sender of status' do - expect(sender.reblogged?(status)).to be true + context 'a known status' do + let(:object_json) do + ActivityPub::TagManager.instance.uri_for(status) + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(status)).to be true + end + end + + context 'self-boost of a previously unknown status with missing attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end + + context 'self-boost of a previously unknown status with correct attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end end end end From ef45411c537d37f06b23914135460544d348bfd6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 18:42:47 +0100 Subject: [PATCH 25/42] Filter incoming Create activities by relation to local activity (#10005) Reject those from accounts with no local followers, from relays that are not enabled, which do not address local accounts and are not replies to accounts that do have local followers --- app/lib/activitypub/activity/create.rb | 34 +++++++++++++++++-- .../activitypub/process_collection_service.rb | 1 + app/workers/activitypub/processing_worker.rb | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 9a3db51dd9e6bd..1b31768d964a61 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -2,8 +2,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def perform - return if unsupported_object_type? || invalid_origin?(@object['id']) - return if Tombstone.exists?(uri: @object['id']) + return if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? RedisLock.acquire(lock_options) do |lock| if lock.acquired? @@ -337,6 +336,37 @@ def reply_to_local? !replied_to_status.nil? && replied_to_status.account.local? end + def related_to_local_activity? + fetch? || followed_by_local_accounts? || requested_through_relay? || + responds_to_followed_account? || addresses_local_accounts? + end + + def fetch? + !@options[:delivery] + end + + def followed_by_local_accounts? + @account.passive_relationships.exists? + end + + def requested_through_relay? + @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? + end + + def responds_to_followed_account? + !replied_to_status.nil? && (replied_to_status.account.local? || replied_to_status.account.passive_relationships.exists?) + end + + def addresses_local_accounts? + return true if @options[:delivered_to_account_id] + + local_usernames = (as_array(@object['to']) + as_array(@object['cc'])).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } + + return false if local_usernames.empty? + + Account.local.where(username: local_usernames).exists? + end + def forward_for_reply return unless @json['signature'].present? && reply_to_local? ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb index 5c54aad89f1840..881df478bf53c7 100644 --- a/app/services/activitypub/process_collection_service.rb +++ b/app/services/activitypub/process_collection_service.rb @@ -44,6 +44,7 @@ def process_item(item) end def verify_account! + @options[:relayed_through_account] = @account @account = ActivityPub::LinkedDataSignature.new(@json).verify_account! rescue JSON::LD::JsonLdError => e Rails.logger.debug "Could not verify LD-Signature for #{value_or_id(@json['actor'])}: #{e.message}" diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb index a8a3ebf0f56987..a3abe72cf66eed 100644 --- a/app/workers/activitypub/processing_worker.rb +++ b/app/workers/activitypub/processing_worker.rb @@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker sidekiq_options backtrace: true def perform(account_id, body, delivered_to_account_id = nil) - ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true, delivered_to_account_id: delivered_to_account_id) + ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true, delivered_to_account_id: delivered_to_account_id, delivery: true) end end From e84c7618192b1ba1538e3c0af79acdf604aea5fc Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 Feb 2019 18:19:45 +0100 Subject: [PATCH 26/42] Filter incoming Announce activities by relation to local activity (#10041) * Filter incoming Announce activities by relation to local activity Reject if announcer is not followed by local accounts, and is not from an enabled relay, and the object is not a local status Follow-up to #10005 * Fix tests --- app/lib/activitypub/activity.rb | 14 ++++++++++++++ app/lib/activitypub/activity/announce.rb | 11 ++++++++++- app/lib/activitypub/activity/create.rb | 12 ------------ spec/lib/activitypub/activity/announce_spec.rb | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 7e4e195313a655..3cf38764a985cc 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -138,11 +138,13 @@ def delete_later!(uri) def status_from_object # If the status is already known, return it status = status_from_uri(object_uri) + return status unless status.nil? # If the boosted toot is embedded and it is a self-boost, handle it like a Create unless unsupported_object_type? actor_id = value_or_id(first_of_value(@object['attributedTo'])) || @account.uri + if actor_id == @account.uri return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform end @@ -166,4 +168,16 @@ def lock_or_return(key, expire_after = 7.days.seconds) ensure redis.del(key) end + + def fetch? + !@options[:delivery] + end + + def followed_by_local_accounts? + @account.passive_relationships.exists? + end + + def requested_through_relay? + @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? + end end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 04afeea202c34f..28a1cda024da6a 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -3,7 +3,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform original_status = status_from_object - return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) + + return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) || !related_to_local_activity? status = Status.find_by(account: @account, reblog: original_status) @@ -39,4 +40,12 @@ def visibility_from_audience def announceable?(status) status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility? end + + def related_to_local_activity? + followed_by_local_accounts? || requested_through_relay? || reblog_of_local_status? + end + + def reblog_of_local_status? + status_from_uri(object_uri)&.account&.local? + end end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 1b31768d964a61..4fc37fb4b5ee7b 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -341,18 +341,6 @@ def related_to_local_activity? responds_to_followed_account? || addresses_local_accounts? end - def fetch? - !@options[:delivery] - end - - def followed_by_local_accounts? - @account.passive_relationships.exists? - end - - def requested_through_relay? - @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? - end - def responds_to_followed_account? !replied_to_status.nil? && (replied_to_status.account.local? || replied_to_status.account.passive_relationships.exists?) end diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 1725c2843bb7ea..5e6f008ec0b4c7 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -18,6 +18,7 @@ subject { described_class.new(json, sender) } before do + Fabricate(:account).follow!(sender) sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) end From 71b831601db2fdc41aaf4ddbe9fd60db109a3153 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 17 Feb 2019 03:38:25 +0100 Subject: [PATCH 27/42] Add logging for rejected ActivityPub payloads and add tests (#10062) --- app/lib/activitypub/activity.rb | 5 + app/lib/activitypub/activity/announce.rb | 4 +- app/lib/activitypub/activity/create.rb | 2 +- .../lib/activitypub/activity/announce_spec.rb | 117 ++- spec/lib/activitypub/activity/create_spec.rb | 710 ++++++++++-------- 5 files changed, 525 insertions(+), 313 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 3cf38764a985cc..8265810a0011a5 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -180,4 +180,9 @@ def followed_by_local_accounts? def requested_through_relay? @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? end + + def reject_payload! + Rails.logger.info("Rejected #{@json['type']} activity #{@json['id']} from #{@account.uri}#{@options[:relayed_through_account] && "via #{@options[:relayed_through_account].uri}"}") + nil + end end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 28a1cda024da6a..9f8ffd9fb77f7a 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -2,9 +2,11 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform + return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity? + original_status = status_from_object - return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) || !related_to_local_activity? + return reject_payload! if original_status.nil? || !announceable?(original_status) status = Status.find_by(account: @account, reblog: original_status) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 4fc37fb4b5ee7b..d7bd65c80668b6 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -2,7 +2,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def perform - return if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? + return reject_payload! if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? RedisLock.acquire(lock_options) do |lock| if lock.acquired? diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 5e6f008ec0b4c7..94b9d348d01abb 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -18,16 +18,63 @@ subject { described_class.new(json, sender) } before do - Fabricate(:account).follow!(sender) sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) end describe '#perform' do - before do - subject.perform + context 'when sender is followed by a local account' do + before do + Fabricate(:account).follow!(sender) + subject.perform + end + + context 'a known status' do + let(:object_json) do + ActivityPub::TagManager.instance.uri_for(status) + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(status)).to be true + end + end + + context 'self-boost of a previously unknown status with missing attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end + + context 'self-boost of a previously unknown status with correct attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end end - context 'a known status' do + context 'when the status belongs to a local user' do + before do + subject.perform + end + let(:object_json) do ActivityPub::TagManager.instance.uri_for(status) end @@ -37,34 +84,68 @@ end end - context 'self-boost of a previously unknown status with missing attributedTo' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: 'http://example.com/followers', - } + context 'when the sender is relayed' do + let!(:relay_account) { Fabricate(:account, inbox_url: 'https://relay.example.com/inbox') } + let!(:relay) { Fabricate(:relay, inbox_url: 'https://relay.example.com/inbox') } + + subject { described_class.new(json, sender, relayed_through_account: relay_account) } + + context 'and the relay is enabled' do + before do + relay.update(state: :accepted) + subject.perform + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.statuses.count).to eq 2 + end end - it 'creates a reblog by sender of status' do - expect(sender.reblogged?(sender.statuses.first)).to be true + context 'and the relay is disabled' do + before do + subject.perform + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'does not create anything' do + expect(sender.statuses.count).to eq 0 + end end end - context 'self-boost of a previously unknown status with correct attributedTo' do + context 'when the sender has no relevance to local activity' do + before do + subject.perform + end + let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - attributedTo: ActivityPub::TagManager.instance.uri_for(sender), to: 'http://example.com/followers', } end - it 'creates a reblog by sender of status' do - expect(sender.reblogged?(sender.statuses.first)).to be true + it 'does not create anything' do + expect(sender.statuses.count).to eq 0 end end end diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index cd20b7c7cb0069..26cb84871c33dd 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -13,8 +13,6 @@ }.with_indifferent_access end - subject { described_class.new(json, sender) } - before do sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) @@ -23,59 +21,407 @@ end describe '#perform' do - before do - subject.perform - end - - context 'standalone' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.text).to eq 'Lorem ipsum' - end - - it 'missing to/cc defaults to direct privacy' do - status = sender.statuses.first + context 'when fetching' do + subject { described_class.new(json, sender) } + + before do + subject.perform + end + + context 'standalone' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.text).to eq 'Lorem ipsum' + end + + it 'missing to/cc defaults to direct privacy' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'direct' + end + end + + context 'public' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'https://www.w3.org/ns/activitystreams#Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'public' + end + end + + context 'unlisted' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + cc: 'https://www.w3.org/ns/activitystreams#Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'unlisted' + end + end + + context 'private' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'private' + end + end + + context 'limited' do + let(:recipient) { Fabricate(:account) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: ActivityPub::TagManager.instance.uri_for(recipient), + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'limited' + end + + it 'creates silent mention' do + status = sender.statuses.first + expect(status.mentions.first).to be_silent + end + end + + context 'direct' do + let(:recipient) { Fabricate(:account) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: ActivityPub::TagManager.instance.uri_for(recipient), + tag: { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(recipient), + }, + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'direct' + end + end + + context 'as a reply' do + let(:original_status) { Fabricate(:status) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status), + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.thread).to eq original_status + expect(status.reply?).to be true + expect(status.in_reply_to_account).to eq original_status.account + expect(status.conversation).to eq original_status.conversation + end + end + + context 'with mentions' do + let(:recipient) { Fabricate(:account) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(recipient), + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.mentions.map(&:account)).to include(recipient) + end + end + + context 'with mentions missing href' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Mention', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with media attachments' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + url: 'http://example.com/attachment.png', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.media_attachments.map(&:remote_url)).to include('http://example.com/attachment.png') + end + end + + context 'with media attachments with focal points' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + url: 'http://example.com/attachment.png', + focalPoint: [0.5, -0.7], + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.media_attachments.map(&:focus)).to include('0.5,-0.7') + end + end + + context 'with media attachments missing url' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with hashtags' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Hashtag', + href: 'http://example.com/blah', + name: '#test', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.tags.map(&:name)).to include('test') + end + end + + context 'with hashtags missing name' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Hashtag', + href: 'http://example.com/blah', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with emojis' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + icon: { + url: 'http://example.com/emoji.png', + }, + name: 'tinking', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.emojis.map(&:shortcode)).to include('tinking') + end + end + + context 'with emojis missing name' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + icon: { + url: 'http://example.com/emoji.png', + }, + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with emojis missing icon' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + name: 'tinking', + }, + ], + } + end - expect(status).to_not be_nil - expect(status.visibility).to eq 'direct' + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end end end - context 'public' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: 'https://www.w3.org/ns/activitystreams#Public', - } - end - - it 'creates status' do - status = sender.statuses.first + context 'when sender is followed by local users' do + subject { described_class.new(json, sender, delivery: true) } - expect(status).to_not be_nil - expect(status.visibility).to eq 'public' + before do + Fabricate(:account).follow!(sender) + subject.perform end - end - context 'unlisted' do let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - cc: 'https://www.w3.org/ns/activitystreams#Public', } end @@ -83,66 +429,25 @@ status = sender.statuses.first expect(status).to_not be_nil - expect(status.visibility).to eq 'unlisted' - end - end - - context 'private' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: 'http://example.com/followers', - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.visibility).to eq 'private' + expect(status.text).to eq 'Lorem ipsum' end end - context 'limited' do - let(:recipient) { Fabricate(:account) } + context 'when sender replies to local status' do + let!(:local_status) { Fabricate(:status) } - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: ActivityPub::TagManager.instance.uri_for(recipient), - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.visibility).to eq 'limited' - end + subject { described_class.new(json, sender, delivery: true) } - it 'creates silent mention' do - status = sender.statuses.first - expect(status.mentions.first).to be_silent + before do + subject.perform end - end - - context 'direct' do - let(:recipient) { Fabricate(:account) } let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - to: ActivityPub::TagManager.instance.uri_for(recipient), - tag: { - type: 'Mention', - href: ActivityPub::TagManager.instance.uri_for(recipient), - }, + inReplyTo: ActivityPub::TagManager.instance.uri_for(local_status), } end @@ -150,47 +455,25 @@ status = sender.statuses.first expect(status).to_not be_nil - expect(status.visibility).to eq 'direct' + expect(status.text).to eq 'Lorem ipsum' end end - context 'as a reply' do - let(:original_status) { Fabricate(:status) } + context 'when sender targets a local user' do + let!(:local_account) { Fabricate(:account) } - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status), - } - end - - it 'creates status' do - status = sender.statuses.first + subject { described_class.new(json, sender, delivery: true) } - expect(status).to_not be_nil - expect(status.thread).to eq original_status - expect(status.reply?).to be true - expect(status.in_reply_to_account).to eq original_status.account - expect(status.conversation).to eq original_status.conversation + before do + subject.perform end - end - - context 'with mentions' do - let(:recipient) { Fabricate(:account) } let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - tag: [ - { - type: 'Mention', - href: ActivityPub::TagManager.instance.uri_for(recipient), - }, - ], + to: ActivityPub::TagManager.instance.uri_for(local_account), } end @@ -198,68 +481,25 @@ status = sender.statuses.first expect(status).to_not be_nil - expect(status.mentions.map(&:account)).to include(recipient) - end - end - - context 'with mentions missing href' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - tag: [ - { - type: 'Mention', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil + expect(status.text).to eq 'Lorem ipsum' end end - context 'with media attachments' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - attachment: [ - { - type: 'Document', - mediaType: 'image/png', - url: 'http://example.com/attachment.png', - }, - ], - } - end + context 'when sender cc\'s a local user' do + let!(:local_account) { Fabricate(:account) } - it 'creates status' do - status = sender.statuses.first + subject { described_class.new(json, sender, delivery: true) } - expect(status).to_not be_nil - expect(status.media_attachments.map(&:remote_url)).to include('http://example.com/attachment.png') + before do + subject.perform end - end - context 'with media attachments with focal points' do let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - attachment: [ - { - type: 'Document', - mediaType: 'image/png', - url: 'http://example.com/attachment.png', - focalPoint: [0.5, -0.7], - }, - ], + cc: ActivityPub::TagManager.instance.uri_for(local_account), } end @@ -267,143 +507,27 @@ status = sender.statuses.first expect(status).to_not be_nil - expect(status.media_attachments.map(&:focus)).to include('0.5,-0.7') + expect(status.text).to eq 'Lorem ipsum' end end - context 'with media attachments missing url' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - attachment: [ - { - type: 'Document', - mediaType: 'image/png', - }, - ], - } - end + context 'when the sender has no relevance to local activity' do + subject { described_class.new(json, sender, delivery: true) } - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil + before do + subject.perform end - end - context 'with hashtags' do let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - tag: [ - { - type: 'Hashtag', - href: 'http://example.com/blah', - name: '#test', - }, - ], } end - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.tags.map(&:name)).to include('test') - end - end - - context 'with hashtags missing name' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - tag: [ - { - type: 'Hashtag', - href: 'http://example.com/blah', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil - end - end - - context 'with emojis' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum :tinking:', - tag: [ - { - type: 'Emoji', - icon: { - url: 'http://example.com/emoji.png', - }, - name: 'tinking', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.emojis.map(&:shortcode)).to include('tinking') - end - end - - context 'with emojis missing name' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum :tinking:', - tag: [ - { - type: 'Emoji', - icon: { - url: 'http://example.com/emoji.png', - }, - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil - end - end - - context 'with emojis missing icon' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum :tinking:', - tag: [ - { - type: 'Emoji', - name: 'tinking', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil + it 'does not create anything' do + expect(sender.statuses.count).to eq 0 end end end From b163368c3e3e3a22c4ef98c0d0cd6c07a2ad13e6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 17 Feb 2019 15:16:36 +0100 Subject: [PATCH 28/42] Fix Announce activities of unknown statuses not fetching those statuses (#10065) Regression from #9998 --- app/lib/activitypub/activity.rb | 3 +- .../lib/activitypub/activity/announce_spec.rb | 43 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 8265810a0011a5..11fa3363aff5e6 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -150,8 +150,7 @@ def status_from_object end end - # If the status is not from the actor, try to fetch it - return fetch_remote_original_status if value_or_id(first_of_value(@json['attributedTo'])) == @account.uri + fetch_remote_original_status end def fetch_remote_original_status diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 94b9d348d01abb..aa58d9e2348848 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Announce do - let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers') } + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', uri: 'https://example.com/actor') } let(:recipient) { Fabricate(:account) } let(:status) { Fabricate(:status, account: recipient) } @@ -10,21 +10,29 @@ '@context': 'https://www.w3.org/ns/activitystreams', id: 'foo', type: 'Announce', - actor: ActivityPub::TagManager.instance.uri_for(sender), + actor: 'https://example.com/actor', object: object_json, }.with_indifferent_access end - subject { described_class.new(json, sender) } - - before do - sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) + let(:unknown_object_json) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: 'https://example.com/actor/hello-world', + type: 'Note', + attributedTo: 'https://example.com/actor', + content: 'Hello world', + to: 'http://example.com/followers', + } end + subject { described_class.new(json, sender) } + describe '#perform' do context 'when sender is followed by a local account' do before do Fabricate(:account).follow!(sender) + stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json)) subject.perform end @@ -38,10 +46,21 @@ end end + context 'an unknown status' do + let(:object_json) { 'https://example.com/actor/hello-world' } + + it 'creates a reblog by sender of status' do + reblog = sender.statuses.first + + expect(reblog).to_not be_nil + expect(reblog.reblog.text).to eq 'Hello world' + end + end + context 'self-boost of a previously unknown status with missing attributedTo' do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', @@ -56,10 +75,10 @@ context 'self-boost of a previously unknown status with correct attributedTo' do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', - attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + attributedTo: 'https://example.com/actor', to: 'http://example.com/followers', } end @@ -98,7 +117,7 @@ let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', @@ -117,7 +136,7 @@ let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', @@ -137,7 +156,7 @@ let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', From 8ad75eea62117f8635de8897c74297a5a8b3bd5c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 Feb 2019 16:08:59 +0100 Subject: [PATCH 29/42] Fix relay enabling/disabling not resetting inbox availability status (#10048) Fix #10033 --- app/models/relay.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/relay.rb b/app/models/relay.rb index 7478c110d45575..6934a5c628760f 100644 --- a/app/models/relay.rb +++ b/app/models/relay.rb @@ -29,6 +29,7 @@ def enable! payload = Oj.dump(follow_activity(activity_id)) update!(state: :pending, follow_activity_id: activity_id) + DeliveryFailureTracker.new(inbox_url).track_success! ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url) end @@ -37,6 +38,7 @@ def disable! payload = Oj.dump(unfollow_activity(activity_id)) update!(state: :idle, follow_activity_id: nil) + DeliveryFailureTracker.new(inbox_url).track_success! ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url) end From 637f0007b9535ed988cbe94c9b9b40a8f8e8a24f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 21:28:18 +0100 Subject: [PATCH 30/42] Change robots.txt to exclude some URLs (#10037) - Exclude static assets - Exclude uploaded files - Exclude alternate versions of the profile page - Exclude media proxy URLs --- public/robots.txt | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 3c9c7c01f30b90..36afc85eff582b 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,5 +1,13 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / +User-Agent: * +Disallow: /users/*/followers +Disallow: /users/*/following +Disallow: /@*/media +Disallow: /@*/with_replies +Disallow: /@*/tagged/* +Disallow: /media_proxy/* +Disallow: /emoji/* +Disallow: /packs/* +Disallow: /sounds/* +Disallow: /system/* +Disallow: /avatars/* +Disallow: /headers/* From 45b2bb464b5ff5b5b5805004a5dc856b495dfc54 Mon Sep 17 00:00:00 2001 From: nightpool Date: Wed, 13 Feb 2019 21:11:47 -0500 Subject: [PATCH 31/42] Change robots.txt to exclude only media proxy URLs (#10038) * Revert "Change robots.txt to exclude some URLs (#10037)" This reverts commit 80161f43510ad9316c60c9b50dd5c09c2dae4d54. * Let's block media_proxy /media_proxy/ is a dynamic route used for requesting uncached media, so it's probably bad to let crawlers use it * misleading comment --- public/robots.txt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 36afc85eff582b..d93648beee28c9 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,13 +1,4 @@ -User-Agent: * -Disallow: /users/*/followers -Disallow: /users/*/following -Disallow: /@*/media -Disallow: /@*/with_replies -Disallow: /@*/tagged/* -Disallow: /media_proxy/* -Disallow: /emoji/* -Disallow: /packs/* -Disallow: /sounds/* -Disallow: /system/* -Disallow: /avatars/* -Disallow: /headers/* +# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file + +User-agent: * +Disallow: /media_proxy/ From 1ad0d232b3ed1c2005d64f1746ecd2d476379852 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Wed, 13 Feb 2019 18:04:43 -0600 Subject: [PATCH 32/42] Improve image description user experience (#10036) * Add image descriptions to searchable post content. * Allow multi-line image descriptions. * Request image descriptions in the same query as posts when creating the search index. (see https://github.com/tootsuite/mastodon/pull/10036#discussion_r256551624) --- app/chewy/statuses_index.rb | 4 ++-- app/javascript/mastodon/features/compose/components/upload.js | 3 +-- app/javascript/styles/mastodon/components.scss | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index d3104172c485d1..eafc1818b8f02f 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -31,7 +31,7 @@ class StatusesIndex < Chewy::Index }, } - define_type ::Status.unscoped.without_reblogs do + define_type ::Status.unscoped.without_reblogs.includes(:media_attachments) do crutch :mentions do |collection| data = ::Mention.where(status_id: collection.map(&:id)).pluck(:status_id, :account_id) data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) } @@ -50,7 +50,7 @@ class StatusesIndex < Chewy::Index root date_detection: false do field :account_id, type: 'long' - field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].join("\n\n") } do + field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).join("\n\n") } do field :stemmed, type: 'text', analyzer: 'content' end diff --git a/app/javascript/mastodon/features/compose/components/upload.js b/app/javascript/mastodon/features/compose/components/upload.js index a1e99dcbbdbca3..947c3acc9ae60c 100644 --- a/app/javascript/mastodon/features/compose/components/upload.js +++ b/app/javascript/mastodon/features/compose/components/upload.js @@ -107,9 +107,8 @@ class Upload extends ImmutablePureComponent {