Skip to content

Commit 628dc2a

Browse files
author
Daniel Yrovas
committed
squash and rebase
1 parent 5ce912b commit 628dc2a

31 files changed

+308
-183
lines changed

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ miniflux
33
*.rpm
44
*.deb
55
.idea
6-
.vscode
6+
.vscode

‎internal/api/entry.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,13 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) {
279279
return
280280
}
281281

282-
json.OK(w, r, map[string]string{"content": entry.Content})
282+
if request.QueryBoolParam(r, "save", false) {
283+
if err := h.store.UpdateEntryContent(entry); err != nil {
284+
json.ServerError(w, r, err)
285+
}
286+
}
287+
288+
json.OK(w, r, map[string]string{"content": entry.Content, "web_content": entry.WebContent})
283289
}
284290

285291
func configureFilters(builder *storage.EntryQueryBuilder, r *http.Request) {

‎internal/database/migrations.go

+7
Original file line numberDiff line numberDiff line change
@@ -752,4 +752,11 @@ var migrations = []func(tx *sql.Tx) error{
752752
_, err = tx.Exec(sql)
753753
return err
754754
},
755+
func(tx *sql.Tx) (err error) {
756+
sql := `
757+
ALTER TABLE entries ADD COLUMN web_content text default '';
758+
`
759+
_, err = tx.Exec(sql)
760+
return err
761+
},
755762
}

‎internal/locale/translations/de_DE.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Diesen Artikel speichern",
6868
"entry.save.completed": "Erledigt!",
6969
"entry.save.toast.completed": "Artikel gespeichert",
70+
"entry.scraper.label.rss": "RSS-Inhalte anzeigen",
71+
"entry.scraper.title.rss": "RSS-Inhalte abrufen",
7072
"entry.scraper.label": "Herunterladen",
7173
"entry.scraper.title": "Inhalt herunterladen",
7274
"entry.scraper.completed": "Erledigt!",

‎internal/locale/translations/el_EL.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Αποθηκεύστε αυτό το άρθρο",
6868
"entry.save.completed": "Έγινε!",
6969
"entry.save.toast.completed": "Το άρθρο αποθηκεύτηκε",
70+
"entry.scraper.label.rss": "Εμφάνιση περιεχομένου RSS",
71+
"entry.scraper.title.rss": "Λήψη περιεχομένου RSS",
7072
"entry.scraper.label": "Λήψη",
7173
"entry.scraper.title": "Λήψη αρχικού περιεχομένου",
7274
"entry.scraper.completed": "Έγινε!",

‎internal/locale/translations/en_US.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Save this entry",
6868
"entry.save.completed": "Done!",
6969
"entry.save.toast.completed": "Entry saved",
70+
"entry.scraper.label.rss": "Show RSS Content",
71+
"entry.scraper.title.rss": "Fetch RSS content",
7072
"entry.scraper.label": "Download",
7173
"entry.scraper.title": "Fetch original content",
7274
"entry.scraper.completed": "Done!",

‎internal/locale/translations/es_ES.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Guardar este artículo",
6868
"entry.save.completed": "¡Hecho!",
6969
"entry.save.toast.completed": "Artículos guardados",
70+
"entry.scraper.label.rss": "Mostrar contenido RSS",
71+
"entry.scraper.title.rss": "Obtener contenido RSS",
7072
"entry.scraper.label": "Descargar",
7173
"entry.scraper.title": "Obtener contenido original",
7274
"entry.scraper.completed": "¡Hecho!",

‎internal/locale/translations/fi_FI.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Tallenna tämä artikkeli",
6868
"entry.save.completed": "Valmis!",
6969
"entry.save.toast.completed": "Artikkeli tallennettu",
70+
"entry.scraper.label.rss": "Näytä RSS-sisältö",
71+
"entry.scraper.title.rss": "Hae RSS-sisältöä",
7072
"entry.scraper.label": "Lataa",
7173
"entry.scraper.title": "Nouda alkuperäinen sisältö",
7274
"entry.scraper.completed": "Valmis!",

‎internal/locale/translations/fr_FR.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Sauvegarder cet article",
6868
"entry.save.completed": "Terminé !",
6969
"entry.save.toast.completed": "Article sauvegardé",
70+
"entry.scraper.label.rss": "Afficher le contenu RSS",
71+
"entry.scraper.title.rss": "Obtenir le contenu RSS",
7072
"entry.scraper.label": "Télécharger",
7173
"entry.scraper.title": "Récupérer le contenu original",
7274
"entry.scraper.completed": "Terminé !",

‎internal/locale/translations/hi_IN.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "एस लेख को सहेजे",
6868
"entry.save.completed": "कार्य समाप्त हुआ!",
6969
"entry.save.toast.completed": "लेख को सहेज लिया",
70+
"entry.scraper.label.rss": "RSS सामग्री दिखाएँ",
71+
"entry.scraper.title.rss": "RSS सामग्री प्राप्त करें",
7072
"entry.scraper.label": "डाउनलोड",
7173
"entry.scraper.title": "मूल विषयवस्तु लाए",
7274
"entry.scraper.completed": "कार्य समाप्त हुआ!",

‎internal/locale/translations/id_ID.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Simpan artikel ini",
6868
"entry.save.completed": "Selesai!",
6969
"entry.save.toast.completed": "Artikel tersimpan",
70+
"entry.scraper.label.rss": "Tampilkan Konten RSS",
71+
"entry.scraper.title.rss": "Dapatkan Konten RSS",
7072
"entry.scraper.label": "Unduh",
7173
"entry.scraper.title": "Ambil konten asli",
7274
"entry.scraper.completed": "Selesai!",

‎internal/locale/translations/it_IT.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Salva questo articolo",
6868
"entry.save.completed": "Fatto!",
6969
"entry.save.toast.completed": "Articolo salvato",
70+
"entry.scraper.label.rss": "Mostra contenuto RSS",
71+
"entry.scraper.title.rss": "Ottieni contenuto RSS",
7072
"entry.scraper.label": "Scarica",
7173
"entry.scraper.title": "Scarica il contenuto integrale",
7274
"entry.scraper.completed": "Fatto!",

‎internal/locale/translations/ja_JP.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "この記事を保存",
6868
"entry.save.completed": "完了!",
6969
"entry.save.toast.completed": "記事は保存されました",
70+
"entry.scraper.label.rss": "RSS コンテンツを表示",
71+
"entry.scraper.title.rss": "RSS コンテンツを取得",
7072
"entry.scraper.label": "ダウンロード",
7173
"entry.scraper.title": "オリジナルの内容を取得",
7274
"entry.scraper.completed": "完了!",

‎internal/locale/translations/nl_NL.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Artikel opslaan",
6868
"entry.save.completed": "Done!",
6969
"entry.save.toast.completed": "Artikel opgeslagen",
70+
"entry.scraper.label.rss": "Toon RSS-content",
71+
"entry.scraper.title.rss": "Haal RSS-content op",
7072
"entry.scraper.label": "Downloaden",
7173
"entry.scraper.title": "Fetch original content",
7274
"entry.scraper.completed": "Klaar!",

‎internal/locale/translations/pl_PL.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Zapisz ten artykuł",
6868
"entry.save.completed": "Gotowe!",
6969
"entry.save.toast.completed": "Artykuł zapisany",
70+
"entry.scraper.label.rss": "Pokaż treść RSS",
71+
"entry.scraper.title.rss": "Pobierz treść RSS",
7072
"entry.scraper.label": "Ściągnij",
7173
"entry.scraper.title": "Pobierz oryginalną treść",
7274
"entry.scraper.completed": "Gotowe!",

‎internal/locale/translations/pt_BR.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Salvar esse item",
6868
"entry.save.completed": "Feito!",
6969
"entry.save.toast.completed": "Item guardado",
70+
"entry.scraper.label.rss": "Mostrar conteúdo RSS",
71+
"entry.scraper.title.rss": "Obter conteúdo RSS",
7072
"entry.scraper.label": "Baixar",
7173
"entry.scraper.title": "Obter conteúdo completo",
7274
"entry.scraper.completed": "Feito!",

‎internal/locale/translations/ru_RU.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Сохранить эту статью",
6868
"entry.save.completed": "Готово!",
6969
"entry.save.toast.completed": "Статья сохранена",
70+
"entry.scraper.label.rss": "Показать содержимое RSS",
71+
"entry.scraper.title.rss": "Получить содержимое RSS",
7072
"entry.scraper.label": "Скачать",
7173
"entry.scraper.title": "Извлечь оригинальное содержимое",
7274
"entry.scraper.completed": "Готово!",

‎internal/locale/translations/tr_TR.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Bu makaleyi kaydet",
6868
"entry.save.completed": "Bitti!",
6969
"entry.save.toast.completed": "Makale kaydedildi",
70+
"entry.scraper.label.rss": "RSS İçeriğini Göster",
71+
"entry.scraper.title.rss": "RSS İçeriğini Al",
7072
"entry.scraper.label": "İndir",
7173
"entry.scraper.title": "Orijinal içeriği çek",
7274
"entry.scraper.completed": "Bitti!",

‎internal/locale/translations/uk_UA.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "Зберегти цю статтю",
6868
"entry.save.completed": "Готово!",
6969
"entry.save.toast.completed": "Стаття збережена",
70+
"entry.scraper.label.rss": "Показати вміст RSS",
71+
"entry.scraper.title.rss": "Отримати вміст RSS",
7072
"entry.scraper.label": "Завантажити",
7173
"entry.scraper.title": "Отримати оригінальний зміст",
7274
"entry.scraper.completed": "Готово!",
@@ -427,4 +429,4 @@
427429
"%d роки тому",
428430
"%d років тому"
429431
]
430-
}
432+
}

‎internal/locale/translations/zh_CN.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "保存这篇文章",
6868
"entry.save.completed": "完成",
6969
"entry.save.toast.completed": "已保存文章",
70+
"entry.scraper.label.rss": "显示 RSS 内容",
71+
"entry.scraper.title.rss": "获取 RSS 内容",
7072
"entry.scraper.label": "抓取全文",
7173
"entry.scraper.title": "抓取全文内容",
7274
"entry.scraper.completed": "抓取完成",

‎internal/locale/translations/zh_TW.json

+2
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
"entry.save.title": "儲存這篇文章",
6868
"entry.save.completed": "完成",
6969
"entry.save.toast.completed": "已儲存文章",
70+
"entry.scraper.label.rss": "顯示 RSS 內容",
71+
"entry.scraper.title.rss": "獲取 RSS 內容",
7072
"entry.scraper.label": "下載原文",
7173
"entry.scraper.title": "下載原文內容",
7274
"entry.scraper.completed": "下載完成",

‎internal/model/entry.go

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type Entry struct {
3030
CreatedAt time.Time `json:"created_at"`
3131
ChangedAt time.Time `json:"changed_at"`
3232
Content string `json:"content"`
33+
WebContent string `json:"web_content,omitempty"`
3334
Author string `json:"author"`
3435
ShareCode string `json:"share_code"`
3536
Starred bool `json:"starred"`

‎internal/reader/processor/processor.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,15 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
8181
logger.Error(`[Processor] Unable to crawl this entry: %q => %v`, entry.URL, scraperErr)
8282
} else if content != "" {
8383
// We replace the entry content only if the scraper doesn't return any error.
84-
entry.Content = content
84+
// TODO: document change
85+
entry.WebContent = content
8586
}
8687
}
8788

8889
rewrite.Rewriter(url, entry, feed.RewriteRules)
8990

9091
// The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered.
91-
entry.Content = sanitizer.Sanitize(url, entry.Content)
92+
entry.WebContent = sanitizer.Sanitize(url, entry.WebContent)
9293

9394
if entryIsNew {
9495
intg, err := store.Integration(feed.UserID)
@@ -169,18 +170,18 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
169170
}
170171

171172
if content != "" {
172-
entry.Content = content
173+
entry.WebContent = content
173174
entry.ReadingTime = calculateReadingTime(content, user)
174175
}
175176

176177
rewrite.Rewriter(url, entry, entry.Feed.RewriteRules)
177-
entry.Content = sanitizer.Sanitize(url, entry.Content)
178+
entry.WebContent = sanitizer.Sanitize(url, entry.WebContent)
178179

179180
return nil
180181
}
181182

182183
func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
183-
var url = entry.URL
184+
url := entry.URL
184185
if feed.UrlRewriteRules != "" {
185186
parts := customReplaceRuleRegex.FindStringSubmatch(feed.UrlRewriteRules)
186187

‎internal/reader/rewrite/rewriter.go

+19-19
Original file line numberDiff line numberDiff line change
@@ -61,33 +61,33 @@ func parseRules(rulesText string) (rules []rule) {
6161
func applyRule(entryURL string, entry *model.Entry, rule rule) {
6262
switch rule.name {
6363
case "add_image_title":
64-
entry.Content = addImageTitle(entryURL, entry.Content)
64+
entry.WebContent = addImageTitle(entryURL, entry.WebContent)
6565
case "add_mailto_subject":
66-
entry.Content = addMailtoSubject(entryURL, entry.Content)
66+
entry.WebContent = addMailtoSubject(entryURL, entry.WebContent)
6767
case "add_dynamic_image":
68-
entry.Content = addDynamicImage(entryURL, entry.Content)
68+
entry.WebContent = addDynamicImage(entryURL, entry.WebContent)
6969
case "add_youtube_video":
70-
entry.Content = addYoutubeVideo(entryURL, entry.Content)
70+
entry.WebContent = addYoutubeVideo(entryURL, entry.WebContent)
7171
case "add_invidious_video":
72-
entry.Content = addInvidiousVideo(entryURL, entry.Content)
72+
entry.WebContent = addInvidiousVideo(entryURL, entry.WebContent)
7373
case "add_youtube_video_using_invidious_player":
74-
entry.Content = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.Content)
74+
entry.WebContent = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.WebContent)
7575
case "add_youtube_video_from_id":
76-
entry.Content = addYoutubeVideoFromId(entry.Content)
76+
entry.WebContent = addYoutubeVideoFromId(entry.WebContent)
7777
case "add_pdf_download_link":
78-
entry.Content = addPDFLink(entryURL, entry.Content)
78+
entry.WebContent = addPDFLink(entryURL, entry.WebContent)
7979
case "nl2br":
80-
entry.Content = replaceLineFeeds(entry.Content)
80+
entry.WebContent = replaceLineFeeds(entry.WebContent)
8181
case "convert_text_link", "convert_text_links":
82-
entry.Content = replaceTextLinks(entry.Content)
82+
entry.WebContent = replaceTextLinks(entry.WebContent)
8383
case "fix_medium_images":
84-
entry.Content = fixMediumImages(entryURL, entry.Content)
84+
entry.WebContent = fixMediumImages(entryURL, entry.WebContent)
8585
case "use_noscript_figure_images":
86-
entry.Content = useNoScriptImages(entryURL, entry.Content)
86+
entry.WebContent = useNoScriptImages(entryURL, entry.WebContent)
8787
case "replace":
8888
// Format: replace("search-term"|"replace-term")
8989
if len(rule.args) >= 2 {
90-
entry.Content = replaceCustom(entry.Content, rule.args[0], rule.args[1])
90+
entry.WebContent = replaceCustom(entry.WebContent, rule.args[0], rule.args[1])
9191
} else {
9292
logger.Debug("[Rewrite] Cannot find search and replace terms for replace rule %s", rule)
9393
}
@@ -101,22 +101,22 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) {
101101
case "remove":
102102
// Format: remove("#selector > .element, .another")
103103
if len(rule.args) >= 1 {
104-
entry.Content = removeCustom(entry.Content, rule.args[0])
104+
entry.WebContent = removeCustom(entry.WebContent, rule.args[0])
105105
} else {
106106
logger.Debug("[Rewrite] Cannot find selector for remove rule %s", rule)
107107
}
108108
case "add_castopod_episode":
109-
entry.Content = addCastopodEpisode(entryURL, entry.Content)
109+
entry.WebContent = addCastopodEpisode(entryURL, entry.WebContent)
110110
case "base64_decode":
111111
if len(rule.args) >= 1 {
112-
entry.Content = applyFuncOnTextContent(entry.Content, rule.args[0], decodeBase64Content)
112+
entry.WebContent = applyFuncOnTextContent(entry.WebContent, rule.args[0], decodeBase64Content)
113113
} else {
114-
entry.Content = applyFuncOnTextContent(entry.Content, "body", decodeBase64Content)
114+
entry.WebContent = applyFuncOnTextContent(entry.WebContent, "body", decodeBase64Content)
115115
}
116116
case "parse_markdown":
117-
entry.Content = parseMarkdown(entry.Content)
117+
entry.WebContent = parseMarkdown(entry.WebContent)
118118
case "remove_tables":
119-
entry.Content = removeTables(entry.Content)
119+
entry.WebContent = removeTables(entry.WebContent)
120120
case "remove_clickbait":
121121
entry.Title = removeClickbait(entry.Title)
122122
}

0 commit comments

Comments
 (0)