Skip to content

Commit 93948d7

Browse files
author
Daniel Yrovas
committed
squash keep rss & web content
1 parent fb8737e commit 93948d7

File tree

13 files changed

+151
-60
lines changed

13 files changed

+151
-60
lines changed

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ miniflux
33
*.rpm
44
*.deb
55
.idea
6-
.vscode
6+
.vscode

‎internal/api/entry.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,11 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) {
279279
return
280280
}
281281

282-
json.OK(w, r, map[string]string{"content": entry.Content})
282+
if err := h.store.UpdateEntryContent(entry); err != nil {
283+
json.ServerError(w, r, err)
284+
}
285+
286+
json.OK(w, r, map[string]string{"content": entry.Content, "web_content": entry.WebContent})
283287
}
284288

285289
func configureFilters(builder *storage.EntryQueryBuilder, r *http.Request) {

‎internal/database/migrations.go

+7
Original file line numberDiff line numberDiff line change
@@ -733,4 +733,11 @@ var migrations = []func(tx *sql.Tx) error{
733733
_, err = tx.Exec(sql)
734734
return err
735735
},
736+
func(tx *sql.Tx) (err error) {
737+
sql := `
738+
ALTER TABLE entries ADD COLUMN web_content text default '';
739+
`
740+
_, err = tx.Exec(sql)
741+
return err
742+
},
736743
}

‎internal/locale/translations/en_US.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Save this entry",
6868
"entry.save.completed": "Done!",
6969
"entry.save.toast.completed": "Entry saved",
70+
"entry.scraper.label.rss": "Show RSS Content",
71+
"entry.scraper.title.rss": "Fetch RSS content",
7072
"entry.scraper.label": "Download",
7173
"entry.scraper.title": "Fetch original content",
7274
"entry.scraper.completed": "Done!",

‎internal/model/entry.go

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type Entry struct {
3030
CreatedAt time.Time `json:"created_at"`
3131
ChangedAt time.Time `json:"changed_at"`
3232
Content string `json:"content"`
33+
WebContent string `json:"web_content,omitempty"`
3334
Author string `json:"author"`
3435
ShareCode string `json:"share_code"`
3536
Starred bool `json:"starred"`

‎internal/reader/processor/processor.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,15 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
8181
logger.Error(`[Processor] Unable to crawl this entry: %q => %v`, entry.URL, scraperErr)
8282
} else if content != "" {
8383
// We replace the entry content only if the scraper doesn't return any error.
84-
entry.Content = content
84+
// TODO: document change
85+
entry.WebContent = content
8586
}
8687
}
8788

8889
rewrite.Rewriter(url, entry, feed.RewriteRules)
8990

9091
// The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered.
91-
entry.Content = sanitizer.Sanitize(url, entry.Content)
92+
entry.WebContent = sanitizer.Sanitize(url, entry.WebContent)
9293

9394
if entryIsNew {
9495
intg, err := store.Integration(feed.UserID)
@@ -169,18 +170,18 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
169170
}
170171

171172
if content != "" {
172-
entry.Content = content
173+
entry.WebContent = content
173174
entry.ReadingTime = calculateReadingTime(content, user)
174175
}
175176

176177
rewrite.Rewriter(url, entry, entry.Feed.RewriteRules)
177-
entry.Content = sanitizer.Sanitize(url, entry.Content)
178+
entry.WebContent = sanitizer.Sanitize(url, entry.WebContent)
178179

179180
return nil
180181
}
181182

182183
func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
183-
var url = entry.URL
184+
url := entry.URL
184185
if feed.UrlRewriteRules != "" {
185186
parts := customReplaceRuleRegex.FindStringSubmatch(feed.UrlRewriteRules)
186187

‎internal/reader/rewrite/rewriter.go

+19-19
Original file line numberDiff line numberDiff line change
@@ -61,55 +61,55 @@ func parseRules(rulesText string) (rules []rule) {
6161
func applyRule(entryURL string, entry *model.Entry, rule rule) {
6262
switch rule.name {
6363
case "add_image_title":
64-
entry.Content = addImageTitle(entryURL, entry.Content)
64+
entry.WebContent = addImageTitle(entryURL, entry.WebContent)
6565
case "add_mailto_subject":
66-
entry.Content = addMailtoSubject(entryURL, entry.Content)
66+
entry.WebContent = addMailtoSubject(entryURL, entry.WebContent)
6767
case "add_dynamic_image":
68-
entry.Content = addDynamicImage(entryURL, entry.Content)
68+
entry.WebContent = addDynamicImage(entryURL, entry.WebContent)
6969
case "add_youtube_video":
70-
entry.Content = addYoutubeVideo(entryURL, entry.Content)
70+
entry.WebContent = addYoutubeVideo(entryURL, entry.WebContent)
7171
case "add_invidious_video":
72-
entry.Content = addInvidiousVideo(entryURL, entry.Content)
72+
entry.WebContent = addInvidiousVideo(entryURL, entry.WebContent)
7373
case "add_youtube_video_using_invidious_player":
74-
entry.Content = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.Content)
74+
entry.WebContent = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.WebContent)
7575
case "add_youtube_video_from_id":
76-
entry.Content = addYoutubeVideoFromId(entry.Content)
76+
entry.WebContent = addYoutubeVideoFromId(entry.WebContent)
7777
case "add_pdf_download_link":
78-
entry.Content = addPDFLink(entryURL, entry.Content)
78+
entry.WebContent = addPDFLink(entryURL, entry.WebContent)
7979
case "nl2br":
80-
entry.Content = replaceLineFeeds(entry.Content)
80+
entry.WebContent = replaceLineFeeds(entry.WebContent)
8181
case "convert_text_link", "convert_text_links":
82-
entry.Content = replaceTextLinks(entry.Content)
82+
entry.WebContent = replaceTextLinks(entry.WebContent)
8383
case "fix_medium_images":
84-
entry.Content = fixMediumImages(entryURL, entry.Content)
84+
entry.WebContent = fixMediumImages(entryURL, entry.WebContent)
8585
case "use_noscript_figure_images":
86-
entry.Content = useNoScriptImages(entryURL, entry.Content)
86+
entry.WebContent = useNoScriptImages(entryURL, entry.WebContent)
8787
case "replace":
8888
// Format: replace("search-term"|"replace-term")
8989
if len(rule.args) >= 2 {
90-
entry.Content = replaceCustom(entry.Content, rule.args[0], rule.args[1])
90+
entry.WebContent = replaceCustom(entry.WebContent, rule.args[0], rule.args[1])
9191
} else {
9292
logger.Debug("[Rewrite] Cannot find search and replace terms for replace rule %s", rule)
9393
}
9494
case "remove":
9595
// Format: remove("#selector > .element, .another")
9696
if len(rule.args) >= 1 {
97-
entry.Content = removeCustom(entry.Content, rule.args[0])
97+
entry.WebContent = removeCustom(entry.WebContent, rule.args[0])
9898
} else {
9999
logger.Debug("[Rewrite] Cannot find selector for remove rule %s", rule)
100100
}
101101
case "add_castopod_episode":
102-
entry.Content = addCastopodEpisode(entryURL, entry.Content)
102+
entry.WebContent = addCastopodEpisode(entryURL, entry.WebContent)
103103
case "base64_decode":
104104
if len(rule.args) >= 1 {
105-
entry.Content = applyFuncOnTextContent(entry.Content, rule.args[0], decodeBase64Content)
105+
entry.WebContent = applyFuncOnTextContent(entry.WebContent, rule.args[0], decodeBase64Content)
106106
} else {
107-
entry.Content = applyFuncOnTextContent(entry.Content, "body", decodeBase64Content)
107+
entry.WebContent = applyFuncOnTextContent(entry.WebContent, "body", decodeBase64Content)
108108
}
109109
case "parse_markdown":
110-
entry.Content = parseMarkdown(entry.Content)
110+
entry.WebContent = parseMarkdown(entry.WebContent)
111111
case "remove_tables":
112-
entry.Content = removeTables(entry.Content)
112+
entry.WebContent = removeTables(entry.WebContent)
113113
case "remove_clickbait":
114114
entry.Title = removeClickbait(entry.Title)
115115
}

‎internal/storage/entry.go

+16-14
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,11 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
7575
UPDATE
7676
entries
7777
SET
78-
content=$1, reading_time=$2
78+
content=$1, web_content=$2, reading_time=$3
7979
WHERE
80-
id=$3 AND user_id=$4
80+
id=$4 AND user_id=$5
8181
`
82-
_, err = tx.Exec(query, entry.Content, entry.ReadingTime, entry.ID, entry.UserID)
82+
_, err = tx.Exec(query, entry.Content, entry.WebContent, entry.ReadingTime, entry.ID, entry.UserID)
8383
if err != nil {
8484
tx.Rollback()
8585
return fmt.Errorf(`store: unable to update content of entry #%d: %v`, entry.ID, err)
@@ -89,7 +89,7 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
8989
UPDATE
9090
entries
9191
SET
92-
document_vectors = setweight(to_tsvector(left(coalesce(title, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce(content, ''), 500000)), 'B')
92+
document_vectors = setweight(to_tsvector(left(coalesce(title, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce(content, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce(web_content, ''), 500000)), 'C')
9393
WHERE
9494
id=$1 AND user_id=$2
9595
`
@@ -98,7 +98,6 @@ func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
9898
tx.Rollback()
9999
return fmt.Errorf(`store: unable to update content of entry #%d: %v`, entry.ID, err)
100100
}
101-
102101
return tx.Commit()
103102
}
104103

@@ -113,6 +112,7 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
113112
comments_url,
114113
published_at,
115114
content,
115+
web_content,
116116
author,
117117
user_id,
118118
feed_id,
@@ -133,9 +133,10 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
133133
$8,
134134
$9,
135135
$10,
136+
$11,
136137
now(),
137-
setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($6, ''), 500000)), 'B'),
138-
$11
138+
setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($6, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($7, ''), 500000)), 'C'),
139+
$12
139140
)
140141
RETURNING
141142
id, status
@@ -148,13 +149,13 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error {
148149
entry.CommentsURL,
149150
entry.Date,
150151
entry.Content,
152+
entry.WebContent,
151153
entry.Author,
152154
entry.UserID,
153155
entry.FeedID,
154156
entry.ReadingTime,
155157
pq.Array(removeDuplicates(entry.Tags)),
156158
).Scan(&entry.ID, &entry.Status)
157-
158159
if err != nil {
159160
return fmt.Errorf(`store: unable to create entry %q (feed #%d): %v`, entry.URL, entry.FeedID, err)
160161
}
@@ -183,12 +184,13 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
183184
url=$2,
184185
comments_url=$3,
185186
content=$4,
186-
author=$5,
187-
reading_time=$6,
188-
document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($4, ''), 500000)), 'B'),
189-
tags=$10
187+
web_content=$5,
188+
author=$6,
189+
reading_time=$7,
190+
document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($4, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($5, ''), 500000)), 'C'),
191+
tags=$11
190192
WHERE
191-
user_id=$7 AND feed_id=$8 AND hash=$9
193+
user_id=$8 AND feed_id=$9 AND hash=$10
192194
RETURNING
193195
id
194196
`
@@ -198,14 +200,14 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error {
198200
entry.URL,
199201
entry.CommentsURL,
200202
entry.Content,
203+
entry.WebContent,
201204
entry.Author,
202205
entry.ReadingTime,
203206
entry.UserID,
204207
entry.FeedID,
205208
entry.Hash,
206209
pq.Array(removeDuplicates(entry.Tags)),
207210
).Scan(&entry.ID)
208-
209211
if err != nil {
210212
return fmt.Errorf(`store: unable to update entry %q: %v`, entry.URL, err)
211213
}

‎internal/storage/entry_query_builder.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
250250
e.author,
251251
e.share_code,
252252
e.content,
253+
e.web_content,
253254
e.status,
254255
e.starred,
255256
e.reading_time,
@@ -314,6 +315,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
314315
&entry.Author,
315316
&entry.ShareCode,
316317
&entry.Content,
318+
&entry.WebContent,
317319
&entry.Status,
318320
&entry.Starred,
319321
&entry.ReadingTime,
@@ -335,7 +337,6 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
335337
&iconID,
336338
&tz,
337339
)
338-
339340
if err != nil {
340341
return nil, fmt.Errorf("unable to fetch entry row: %v", err)
341342
}

‎internal/template/templates/views/entry.html

+29-15
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,17 @@ <h1 dir="auto">
7878
</li>
7979
<li>
8080
<a href="#"
81-
title="{{ t "entry.scraper.title" }}"
81+
title="{{ if .entry.WebContent }}{{ t "entry.scraper.title.rss" }}{{ else }}{{ t "entry.scraper.title" }}{{ end }}"
82+
data-title="{{ t "entry.scraper.title" }}"
83+
data-title-rss="{{ t "entry.scraper.title.rss" }}"
84+
data-label="{{ t "entry.scraper.label" }}"
85+
data-label-rss="{{ t "entry.scraper.label.rss" }}"
86+
data-label-loading="{{ t "entry.state.loading" }}"
8287
data-fetch-content-entry="true"
88+
data-current-content="{{ if .entry.WebContent }}web{{ else }}rss{{ end }}"
8389
data-fetch-content-url="{{ route "fetchContent" "entryID" .entry.ID }}"
84-
data-label-loading="{{ t "entry.state.loading" }}"
85-
>{{ icon "scraper" }}<span class="icon-label">{{ t "entry.scraper.label" }}</span></a>
90+
data-fetch-original-content-url="{{ route "fetchOriginal" "entryID" .entry.ID }}"
91+
>{{ icon "scraper" }}<span class="icon-label">{{ if .entry.WebContent }}{{ t "entry.scraper.label.rss" }}{{ else }}{{ t "entry.scraper.label" }}{{ end }}</span></a>
8692
</li>
8793
{{ if .entry.CommentsURL }}
8894
<li>
@@ -186,9 +192,17 @@ <h1 dir="auto">
186192
{{ end }}
187193
{{end}}
188194
{{ if .user }}
189-
{{ noescape (proxyFilter .entry.Content) }}
195+
{{ if .entry.WebContent }}
196+
{{ noescape (proxyFilter .entry.WebContent) }}
197+
{{ else }}
198+
{{ noescape (proxyFilter .entry.Content) }}
199+
{{ end }}
190200
{{ else }}
191-
{{ noescape .entry.Content }}
201+
{{ if .entry.WebContent }}
202+
{{ noescape .entry.WebContent }}
203+
{{ else }}
204+
{{ noescape .entry.Content }}
205+
{{ end }}
192206
{{ end }}
193207
</article>
194208
{{ if .entry.Enclosures }}
@@ -203,11 +217,11 @@ <h1 dir="auto">
203217
data-last-position="{{ .MediaProgression }}"
204218
data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}"
205219
>
206-
{{ if (and $.user (mustBeProxyfied "audio")) }}
207-
<source src="{{ proxyURL .URL }}" type="{{ .Html5MimeType }}">
208-
{{ else }}
209-
<source src="{{ .URL | safeURL }}" type="{{ .Html5MimeType }}">
210-
{{ end }}
220+
{{ if (and $.user (mustBeProxyfied "audio")) }}
221+
<source src="{{ proxyURL .URL }}" type="{{ .Html5MimeType }}">
222+
{{ else }}
223+
<source src="{{ .URL | safeURL }}" type="{{ .Html5MimeType }}">
224+
{{ end }}
211225
</audio>
212226
</div>
213227
{{ else if hasPrefix .MimeType "video/" }}
@@ -216,11 +230,11 @@ <h1 dir="auto">
216230
data-last-position="{{ .MediaProgression }}"
217231
data-save-url="{{ route "saveEnclosureProgression" "enclosureID" .ID }}"
218232
>
219-
{{ if (and $.user (mustBeProxyfied "video")) }}
220-
<source src="{{ proxyURL .URL }}" type="{{ .Html5MimeType }}">
221-
{{ else }}
222-
<source src="{{ .URL | safeURL }}" type="{{ .Html5MimeType }}">
223-
{{ end }}
233+
{{ if (and $.user (mustBeProxyfied "video")) }}
234+
<source src="{{ proxyURL .URL }}" type="{{ .Html5MimeType }}">
235+
{{ else }}
236+
<source src="{{ .URL | safeURL }}" type="{{ .Html5MimeType }}">
237+
{{ end }}
224238
</video>
225239
</div>
226240
{{ else if hasPrefix .MimeType "image/" }}

0 commit comments

Comments
 (0)