Skip to content

Commit 29ee5ec

Browse files
Daniel Yrovasdanielyrovas
Daniel Yrovas
authored andcommitted
keep rss & web content | rebase
1 parent 6592c1a commit 29ee5ec

30 files changed

+325
-193
lines changed

internal/api/entry.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,13 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) {
335335
return
336336
}
337337

338-
json.OK(w, r, map[string]string{"content": entry.Content})
338+
if request.QueryBoolParam(r, "save", false) {
339+
if err := h.store.UpdateEntryTitleAndContent(entry); err != nil {
340+
json.ServerError(w, r, err)
341+
}
342+
}
343+
344+
json.OK(w, r, map[string]string{"content": entry.Content, "web_content": entry.WebContent})
339345
}
340346

341347
func (h *handler) flushHistory(w http.ResponseWriter, r *http.Request) {

internal/database/migrations.go

+7
Original file line numberDiff line numberDiff line change
@@ -882,4 +882,11 @@ var migrations = []func(tx *sql.Tx) error{
882882
_, err = tx.Exec(sql)
883883
return err
884884
},
885+
func(tx *sql.Tx) (err error) {
886+
sql := `
887+
ALTER TABLE entries ADD COLUMN web_content text default '';
888+
`
889+
_, err = tx.Exec(sql)
890+
return err
891+
},
885892
}

internal/locale/translations/de_DE.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Diesen Artikel speichern",
7373
"entry.save.completed": "Erledigt!",
7474
"entry.save.toast.completed": "Artikel gespeichert",
75+
"entry.scraper.label.rss": "RSS-Inhalte anzeigen",
76+
"entry.scraper.title.rss": "RSS-Inhalte abrufen",
7577
"entry.scraper.label": "Herunterladen",
7678
"entry.scraper.title": "Inhalt herunterladen",
7779
"entry.scraper.completed": "Erledigt!",

internal/locale/translations/el_EL.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Αποθηκεύστε αυτό το άρθρο",
7373
"entry.save.completed": "Έγινε!",
7474
"entry.save.toast.completed": "Το άρθρο αποθηκεύτηκε",
75+
"entry.scraper.label.rss": "Εμφάνιση περιεχομένου RSS",
76+
"entry.scraper.title.rss": "Λήψη περιεχομένου RSS",
7577
"entry.scraper.label": "Λήψη",
7678
"entry.scraper.title": "Λήψη αρχικού περιεχομένου",
7779
"entry.scraper.completed": "Έγινε!",

internal/locale/translations/en_US.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Save this entry",
7373
"entry.save.completed": "Done!",
7474
"entry.save.toast.completed": "Entry saved",
75+
"entry.scraper.label.rss": "Show RSS Content",
76+
"entry.scraper.title.rss": "Fetch RSS content",
7577
"entry.scraper.label": "Download",
7678
"entry.scraper.title": "Fetch original content",
7779
"entry.scraper.completed": "Done!",

internal/locale/translations/es_ES.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Guardar este artículo",
7373
"entry.save.completed": "¡Hecho!",
7474
"entry.save.toast.completed": "Artículos guardados",
75+
"entry.scraper.label.rss": "Mostrar contenido RSS",
76+
"entry.scraper.title.rss": "Obtener contenido RSS",
7577
"entry.scraper.label": "Descargar",
7678
"entry.scraper.title": "Obtener contenido original",
7779
"entry.scraper.completed": "¡Hecho!",

internal/locale/translations/fi_FI.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Tallenna tämä artikkeli",
7373
"entry.save.completed": "Valmis!",
7474
"entry.save.toast.completed": "Artikkeli tallennettu",
75+
"entry.scraper.label.rss": "Näytä RSS-sisältö",
76+
"entry.scraper.title.rss": "Hae RSS-sisältöä",
7577
"entry.scraper.label": "Lataa",
7678
"entry.scraper.title": "Nouda alkuperäinen sisältö",
7779
"entry.scraper.completed": "Valmis!",

internal/locale/translations/fr_FR.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Sauvegarder cet article",
7373
"entry.save.completed": "Terminé !",
7474
"entry.save.toast.completed": "Article sauvegardé",
75+
"entry.scraper.label.rss": "Afficher le contenu RSS",
76+
"entry.scraper.title.rss": "Obtenir le contenu RSS",
7577
"entry.scraper.label": "Télécharger",
7678
"entry.scraper.title": "Récupérer le contenu original",
7779
"entry.scraper.completed": "Terminé !",

internal/locale/translations/hi_IN.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "एस लेख को सहेजे",
7373
"entry.save.completed": "कार्य समाप्त हुआ!",
7474
"entry.save.toast.completed": "लेख को सहेज लिया",
75+
"entry.scraper.label.rss": "RSS सामग्री दिखाएँ",
76+
"entry.scraper.title.rss": "RSS सामग्री प्राप्त करें",
7577
"entry.scraper.label": "डाउनलोड",
7678
"entry.scraper.title": "मूल विषयवस्तु लाए",
7779
"entry.scraper.completed": "कार्य समाप्त हुआ!",

internal/locale/translations/id_ID.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Simpan artikel ini",
7373
"entry.save.completed": "Selesai!",
7474
"entry.save.toast.completed": "Artikel tersimpan",
75+
"entry.scraper.label.rss": "Tampilkan Konten RSS",
76+
"entry.scraper.title.rss": "Dapatkan Konten RSS",
7577
"entry.scraper.label": "Unduh",
7678
"entry.scraper.title": "Ambil konten asli",
7779
"entry.scraper.completed": "Selesai!",

internal/locale/translations/it_IT.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Salva questo articolo",
7373
"entry.save.completed": "Fatto!",
7474
"entry.save.toast.completed": "Articolo salvato",
75+
"entry.scraper.label.rss": "Mostra contenuto RSS",
76+
"entry.scraper.title.rss": "Ottieni contenuto RSS",
7577
"entry.scraper.label": "Scarica",
7678
"entry.scraper.title": "Scarica il contenuto integrale",
7779
"entry.scraper.completed": "Fatto!",

internal/locale/translations/ja_JP.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "この記事を保存",
7373
"entry.save.completed": "完了!",
7474
"entry.save.toast.completed": "記事は保存されました",
75+
"entry.scraper.label.rss": "RSS コンテンツを表示",
76+
"entry.scraper.title.rss": "RSS コンテンツを取得",
7577
"entry.scraper.label": "ダウンロード",
7678
"entry.scraper.title": "オリジナルの内容を取得",
7779
"entry.scraper.completed": "完了!",

internal/locale/translations/nl_NL.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Artikel opslaan",
7373
"entry.save.completed": "Done!",
7474
"entry.save.toast.completed": "Artikel opgeslagen",
75+
"entry.scraper.label.rss": "Toon RSS-content",
76+
"entry.scraper.title.rss": "Haal RSS-content op",
7577
"entry.scraper.label": "Downloaden",
7678
"entry.scraper.title": "Fetch original content",
7779
"entry.scraper.completed": "Klaar!",

internal/locale/translations/pl_PL.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Zapisz ten artykuł",
7373
"entry.save.completed": "Gotowe!",
7474
"entry.save.toast.completed": "Artykuł zapisany",
75+
"entry.scraper.label.rss": "Pokaż treść RSS",
76+
"entry.scraper.title.rss": "Pobierz treść RSS",
7577
"entry.scraper.label": "Ściągnij",
7678
"entry.scraper.title": "Pobierz oryginalną treść",
7779
"entry.scraper.completed": "Gotowe!",

internal/locale/translations/pt_BR.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Salvar esse item",
7373
"entry.save.completed": "Feito!",
7474
"entry.save.toast.completed": "Item guardado",
75+
"entry.scraper.label.rss": "Mostrar conteúdo RSS",
76+
"entry.scraper.title.rss": "Obter conteúdo RSS",
7577
"entry.scraper.label": "Baixar",
7678
"entry.scraper.title": "Obter conteúdo completo",
7779
"entry.scraper.completed": "Feito!",

internal/locale/translations/ru_RU.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Сохранить эту статью",
7373
"entry.save.completed": "Готово!",
7474
"entry.save.toast.completed": "Статья сохранена",
75+
"entry.scraper.label.rss": "Показать содержимое RSS",
76+
"entry.scraper.title.rss": "Получить содержимое RSS",
7577
"entry.scraper.label": "Скачать",
7678
"entry.scraper.title": "Извлечь оригинальное содержимое",
7779
"entry.scraper.completed": "Готово!",

internal/locale/translations/tr_TR.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "Bu makaleyi kaydet",
7373
"entry.save.completed": "Bitti!",
7474
"entry.save.toast.completed": "Makale kaydedildi",
75+
"entry.scraper.label.rss": "RSS İçeriğini Göster",
76+
"entry.scraper.title.rss": "RSS İçeriğini Al",
7577
"entry.scraper.label": "İndir",
7678
"entry.scraper.title": "Orijinal içeriği çek",
7779
"entry.scraper.completed": "Bitti!",

internal/locale/translations/uk_UA.json

+2
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,9 @@
6969
"entry.state.saving": "Зберігаю...",
7070
"entry.state.loading": "Завантаження...",
7171
"entry.save.label": "Зберегти",
72+
"entry.scraper.label.rss": "Показати вміст RSS",
7273
"entry.save.title": "Зберегти цю статтю",
74+
"entry.scraper.title.rss": "Отримати вміст RSS",
7375
"entry.save.completed": "Готово!",
7476
"entry.save.toast.completed": "Стаття збережена",
7577
"entry.scraper.label": "Завантажити",

internal/locale/translations/zh_CN.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "保存这篇文章",
7373
"entry.save.completed": "完成",
7474
"entry.save.toast.completed": "已保存文章",
75+
"entry.scraper.label.rss": "显示 RSS 内容",
76+
"entry.scraper.title.rss": "获取 RSS 内容",
7577
"entry.scraper.label": "抓取全文",
7678
"entry.scraper.title": "抓取全文内容",
7779
"entry.scraper.completed": "抓取完成",

internal/locale/translations/zh_TW.json

+2
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@
7272
"entry.save.title": "儲存這篇文章",
7373
"entry.save.completed": "完成",
7474
"entry.save.toast.completed": "已儲存文章",
75+
"entry.scraper.label.rss": "顯示 RSS 內容",
76+
"entry.scraper.title.rss": "獲取 RSS 內容",
7577
"entry.scraper.label": "下載原文",
7678
"entry.scraper.title": "下載原文內容",
7779
"entry.scraper.completed": "下載完成",

internal/model/entry.go

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type Entry struct {
3030
CreatedAt time.Time `json:"created_at"`
3131
ChangedAt time.Time `json:"changed_at"`
3232
Content string `json:"content"`
33+
WebContent string `json:"web_content,omitempty"`
3334
Author string `json:"author"`
3435
ShareCode string `json:"share_code"`
3536
Starred bool `json:"starred"`

internal/reader/processor/processor.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,14 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
9898
)
9999
} else if content != "" {
100100
// We replace the entry content only if the scraper doesn't return any error.
101-
entry.Content = content
101+
entry.WebContent = content
102102
}
103103
}
104104

105105
rewrite.Rewriter(websiteURL, entry, feed.RewriteRules)
106106

107107
// The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered.
108-
entry.Content = sanitizer.Sanitize(websiteURL, entry.Content)
108+
entry.WebContent = sanitizer.Sanitize(websiteURL, entry.Content)
109109

110110
updateEntryReadingTime(store, feed, entry, entryIsNew, user)
111111
filteredEntries = append(filteredEntries, entry)
@@ -209,14 +209,14 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
209209
}
210210

211211
if content != "" {
212-
entry.Content = content
212+
entry.WebContent = content
213213
if user.ShowReadingTime {
214-
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
214+
entry.ReadingTime = readingtime.EstimateReadingTime(entry.WebContent, user.DefaultReadingSpeed, user.CJKReadingSpeed)
215215
}
216216
}
217217

218218
rewrite.Rewriter(websiteURL, entry, entry.Feed.RewriteRules)
219-
entry.Content = sanitizer.Sanitize(websiteURL, entry.Content)
219+
entry.WebContent = sanitizer.Sanitize(websiteURL, entry.WebContent)
220220

221221
return nil
222222
}

internal/reader/rewrite/rewriter.go

+21-21
Original file line numberDiff line numberDiff line change
@@ -24,35 +24,35 @@ type rule struct {
2424
func (rule rule) applyRule(entryURL string, entry *model.Entry) {
2525
switch rule.name {
2626
case "add_image_title":
27-
entry.Content = addImageTitle(entryURL, entry.Content)
27+
entry.WebContent = addImageTitle(entryURL, entry.WebContent)
2828
case "add_mailto_subject":
29-
entry.Content = addMailtoSubject(entryURL, entry.Content)
29+
entry.WebContent = addMailtoSubject(entryURL, entry.WebContent)
3030
case "add_dynamic_image":
31-
entry.Content = addDynamicImage(entryURL, entry.Content)
31+
entry.WebContent = addDynamicImage(entryURL, entry.WebContent)
3232
case "add_dynamic_iframe":
33-
entry.Content = addDynamicIframe(entryURL, entry.Content)
33+
entry.WebContent = addDynamicIframe(entryURL, entry.WebContent)
3434
case "add_youtube_video":
35-
entry.Content = addYoutubeVideo(entryURL, entry.Content)
35+
entry.WebContent = addYoutubeVideo(entryURL, entry.WebContent)
3636
case "add_invidious_video":
37-
entry.Content = addInvidiousVideo(entryURL, entry.Content)
37+
entry.WebContent = addInvidiousVideo(entryURL, entry.WebContent)
3838
case "add_youtube_video_using_invidious_player":
39-
entry.Content = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.Content)
39+
entry.WebContent = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.WebContent)
4040
case "add_youtube_video_from_id":
41-
entry.Content = addYoutubeVideoFromId(entry.Content)
41+
entry.WebContent = addYoutubeVideoFromId(entry.WebContent)
4242
case "add_pdf_download_link":
43-
entry.Content = addPDFLink(entryURL, entry.Content)
43+
entry.WebContent = addPDFLink(entryURL, entry.WebContent)
4444
case "nl2br":
45-
entry.Content = strings.ReplaceAll(entry.Content, "\n", "<br>")
45+
entry.WebContent = strings.ReplaceAll(entry.WebContent, "\n", "<br>")
4646
case "convert_text_link", "convert_text_links":
47-
entry.Content = replaceTextLinks(entry.Content)
47+
entry.WebContent = replaceTextLinks(entry.WebContent)
4848
case "fix_medium_images":
49-
entry.Content = fixMediumImages(entryURL, entry.Content)
49+
entry.WebContent = fixMediumImages(entryURL, entry.WebContent)
5050
case "use_noscript_figure_images":
51-
entry.Content = useNoScriptImages(entryURL, entry.Content)
51+
entry.WebContent = useNoScriptImages(entryURL, entry.WebContent)
5252
case "replace":
5353
// Format: replace("search-term"|"replace-term")
5454
if len(rule.args) >= 2 {
55-
entry.Content = replaceCustom(entry.Content, rule.args[0], rule.args[1])
55+
entry.WebContent = replaceCustom(entry.WebContent, rule.args[0], rule.args[1])
5656
} else {
5757
slog.Warn("Cannot find search and replace terms for replace rule",
5858
slog.Any("rule", rule),
@@ -72,29 +72,29 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) {
7272
case "remove":
7373
// Format: remove("#selector > .element, .another")
7474
if len(rule.args) >= 1 {
75-
entry.Content = removeCustom(entry.Content, rule.args[0])
75+
entry.WebContent = removeCustom(entry.WebContent, rule.args[0])
7676
} else {
7777
slog.Warn("Cannot find selector for remove rule",
7878
slog.Any("rule", rule),
7979
slog.String("entry_url", entryURL),
8080
)
8181
}
8282
case "add_castopod_episode":
83-
entry.Content = addCastopodEpisode(entryURL, entry.Content)
83+
entry.WebContent = addCastopodEpisode(entryURL, entry.WebContent)
8484
case "base64_decode":
8585
selector := "body"
8686
if len(rule.args) >= 1 {
8787
selector = rule.args[0]
8888
}
89-
entry.Content = applyFuncOnTextContent(entry.Content, selector, decodeBase64Content)
89+
entry.WebContent = applyFuncOnTextContent(entry.WebContent, selector, decodeBase64Content)
9090
case "add_hn_links_using_hack":
91-
entry.Content = addHackerNewsLinksUsing(entry.Content, "hack")
91+
entry.WebContent = addHackerNewsLinksUsing(entry.WebContent, "hack")
9292
case "add_hn_links_using_opener":
93-
entry.Content = addHackerNewsLinksUsing(entry.Content, "opener")
93+
entry.WebContent = addHackerNewsLinksUsing(entry.WebContent, "opener")
9494
case "parse_markdown":
95-
entry.Content = parseMarkdown(entry.Content)
95+
entry.WebContent = parseMarkdown(entry.WebContent)
9696
case "remove_tables":
97-
entry.Content = removeTables(entry.Content)
97+
entry.WebContent = removeTables(entry.WebContent)
9898
case "remove_clickbait":
9999
entry.Title = cases.Title(language.English).String(strings.ToLower(entry.Title))
100100
}

0 commit comments

Comments
 (0)