diff --git a/api/feed.go b/api/feed.go
index fc7ae6a26a40c55d2d159d2e2e0a6ac30863d730..31a33a0edf34decca42a4da0f5ba5e89a8eb3193 100644
--- a/api/feed.go
+++ b/api/feed.go
@@ -51,6 +51,8 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.Password,
feedInfo.ScraperRules,
feedInfo.RewriteRules,
+ feedInfo.BlocklistRules,
+ feedInfo.KeeplistRules,
feedInfo.FetchViaProxy,
)
if err != nil {
diff --git a/api/payload.go b/api/payload.go
index 9b20901704d6e0d35cb5f276e05eca73b84550ba..8923078d6885652a6ded00a0b8384a1e782ceb3e 100644
--- a/api/payload.go
+++ b/api/payload.go
@@ -24,15 +24,17 @@ type entriesResponse struct {
}
type feedCreation struct {
- FeedURL string `json:"feed_url"`
- CategoryID int64 `json:"category_id"`
- UserAgent string `json:"user_agent"`
- Username string `json:"username"`
- Password string `json:"password"`
- Crawler bool `json:"crawler"`
- FetchViaProxy bool `json:"fetch_via_proxy"`
- ScraperRules string `json:"scraper_rules"`
- RewriteRules string `json:"rewrite_rules"`
+ FeedURL string `json:"feed_url"`
+ CategoryID int64 `json:"category_id"`
+ UserAgent string `json:"user_agent"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Crawler bool `json:"crawler"`
+ FetchViaProxy bool `json:"fetch_via_proxy"`
+ ScraperRules string `json:"scraper_rules"`
+ RewriteRules string `json:"rewrite_rules"`
+ BlocklistRules string `json:"blocklist_rules"`
+ KeeplistRules string `json:"keeplist_rules"`
}
type subscriptionDiscovery struct {
@@ -44,17 +46,19 @@ type subscriptionDiscovery struct {
}
type feedModification struct {
- FeedURL *string `json:"feed_url"`
- SiteURL *string `json:"site_url"`
- Title *string `json:"title"`
- ScraperRules *string `json:"scraper_rules"`
- RewriteRules *string `json:"rewrite_rules"`
- Crawler *bool `json:"crawler"`
- UserAgent *string `json:"user_agent"`
- Username *string `json:"username"`
- Password *string `json:"password"`
- CategoryID *int64 `json:"category_id"`
- Disabled *bool `json:"disabled"`
+ FeedURL *string `json:"feed_url"`
+ SiteURL *string `json:"site_url"`
+ Title *string `json:"title"`
+ ScraperRules *string `json:"scraper_rules"`
+ RewriteRules *string `json:"rewrite_rules"`
+ BlocklistRules *string `json:"blocklist_rules"`
+ KeeplistRules *string `json:"keeplist_rules"`
+ Crawler *bool `json:"crawler"`
+ UserAgent *string `json:"user_agent"`
+ Username *string `json:"username"`
+ Password *string `json:"password"`
+ CategoryID *int64 `json:"category_id"`
+ Disabled *bool `json:"disabled"`
}
func (f *feedModification) Update(feed *model.Feed) {
@@ -78,6 +82,14 @@ func (f *feedModification) Update(feed *model.Feed) {
feed.RewriteRules = *f.RewriteRules
}
+ if f.KeeplistRules != nil {
+ feed.KeeplistRules = *f.KeeplistRules
+ }
+
+ if f.BlocklistRules != nil {
+ feed.BlocklistRules = *f.BlocklistRules
+ }
+
if f.Crawler != nil {
feed.Crawler = *f.Crawler
}
diff --git a/client/core.go b/client/core.go
index 95ff8aa88464a60d31cfa6070390353f838f8007..c2aabf4fa238163df4e149618734492184fdbf57 100644
--- a/client/core.go
+++ b/client/core.go
@@ -92,6 +92,8 @@ type Feed struct {
ParsingErrorCount int `json:"parsing_error_count,omitempty"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
+ BlocklistRules string `json:"blocklist_rules"`
+ KeeplistRules string `json:"keeplist_rules"`
Crawler bool `json:"crawler"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
@@ -101,16 +103,18 @@ type Feed struct {
// FeedModification represents changes for a feed.
type FeedModification struct {
- FeedURL *string `json:"feed_url"`
- SiteURL *string `json:"site_url"`
- Title *string `json:"title"`
- ScraperRules *string `json:"scraper_rules"`
- RewriteRules *string `json:"rewrite_rules"`
- Crawler *bool `json:"crawler"`
- UserAgent *string `json:"user_agent"`
- Username *string `json:"username"`
- Password *string `json:"password"`
- CategoryID *int64 `json:"category_id"`
+ FeedURL *string `json:"feed_url"`
+ SiteURL *string `json:"site_url"`
+ Title *string `json:"title"`
+ ScraperRules *string `json:"scraper_rules"`
+ RewriteRules *string `json:"rewrite_rules"`
+ BlocklistRules *string `json:"blocklist_rules"`
+ KeeplistRules *string `json:"keeplist_rules"`
+ Crawler *bool `json:"crawler"`
+ UserAgent *string `json:"user_agent"`
+ Username *string `json:"username"`
+ Password *string `json:"password"`
+ CategoryID *int64 `json:"category_id"`
}
// FeedIcon represents the feed icon.
diff --git a/database/migration.go b/database/migration.go
index 8c79339326d47ab032b45c4fa7fecba9ed7d507f..a1f415ed15d7d4006e3bc881b63e1dba4374ccbe 100644
--- a/database/migration.go
+++ b/database/migration.go
@@ -12,7 +12,7 @@ import (
"miniflux.app/logger"
)
-const schemaVersion = 39
+const schemaVersion = 40
// Migrate executes database migrations.
func Migrate(db *sql.DB) {
diff --git a/database/sql.go b/database/sql.go
index 52265815979e71091810d68d3c576ecb6668c592..0ab746f6b5939ae4b821399c645cec65a5efbee8 100644
--- a/database/sql.go
+++ b/database/sql.go
@@ -197,6 +197,11 @@ create index entries_user_feed_idx on entries (user_id, feed_id);
`,
"schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc');
alter table users add column entry_direction entry_sorting_direction default 'asc';
+`,
+ "schema_version_40": `alter table feeds
+ add column blocklist_rules text not null default '',
+ add column keeplist_rules text not null default ''
+;
`,
"schema_version_5": `create table integrations (
user_id int not null,
@@ -258,6 +263,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_38": "e91d2f4075ceb7b8a16a25f350f36dee12cfd1ad86b8b6414c4cf2e9a003358c",
"schema_version_39": "b0f90b97502921d4681a07c64d180a91a0b4ccac7d3c1dbe30519ad6f1bf1737",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
+ "schema_version_40": "6a8fec92399f853ed6817aff4cfa43255dce4c19afad796e41519d09de62105e",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",
"schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4",
"schema_version_7": "33f298c9aa30d6de3ca28e1270df51c2884d7596f1283a75716e2aeb634cd05c",
diff --git a/database/sql/schema_version_40.sql b/database/sql/schema_version_40.sql
new file mode 100644
index 0000000000000000000000000000000000000000..c0b5ce769de53b3352aeea1cf88e8bc8568ca713
--- /dev/null
+++ b/database/sql/schema_version_40.sql
@@ -0,0 +1,4 @@
+alter table feeds
+ add column blocklist_rules text not null default '',
+ add column keeplist_rules text not null default ''
+;
diff --git a/go.sum b/go.sum
index 3236b211c7008b39ad10bfb113e5abd146733615..7884a3488b753c9b335e4d9d0aff07edfccc7090 100644
--- a/go.sum
+++ b/go.sum
@@ -348,7 +348,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BG
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -362,7 +361,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -371,14 +369,12 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c h1:38q6VNPWR010vN82/SB121GujZNIfAUb4YttE2rhGuc=
golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -399,14 +395,11 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -428,16 +421,13 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
@@ -457,7 +447,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/locale/translations.go b/locale/translations.go
index f9bde112dc86ad7ada1579780e9af51a18fcb83a..3995493e7698e31116eb3ffca9827bea485b4833 100644
--- a/locale/translations.go
+++ b/locale/translations.go
@@ -253,6 +253,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
+ "form.feed.label.blocklist_rules": "Regeln blockieren",
+ "form.feed.label.keeplist_rules": "Regeln einhalten",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ãœber Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
@@ -600,6 +602,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Override Default User Agent",
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
+ "form.feed.label.blocklist_rules": "Block Rules",
+ "form.feed.label.keeplist_rules": "Keep Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
@@ -927,6 +931,9 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
+ "form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",
+ "form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)",
+ "form.feed.label.blocklist_rules": "Reglas de Blacklist",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
@@ -1254,6 +1261,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
+ "form.feed.label.blocklist_rules": "Règles de blocage",
+ "form.feed.label.keeplist_rules": "Règles d'autorisation",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
@@ -1601,6 +1610,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Usa user agent personalizzato",
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
+ "form.feed.label.blocklist_rules": "Regole di blocco",
+ "form.feed.label.keeplist_rules": "Regole di autorizzazione",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
@@ -1928,6 +1939,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "ディフォルト㮠User Agent を上書ãã™ã‚‹",
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
+ "form.feed.label.blocklist_rules": "ブãƒãƒƒã‚¯ãƒ«ãƒ¼ãƒ«",
+ "form.feed.label.keeplist_rules": "許å¯è¦å‰‡",
"form.feed.label.ignore_http_cache": "HTTPã‚ャッシュを無視",
"form.feed.label.fetch_via_proxy": "プãƒã‚シ経由ã§ãƒ•ã‚§ãƒƒãƒ",
"form.feed.label.disabled": "ã“ã®ãƒ•ã‚£ãƒ¼ãƒ‰ã‚’æ›´æ–°ã—ãªã„",
@@ -2255,6 +2268,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Standaard User Agent overschrijven",
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
+ "form.feed.label.blocklist_rules": "Blokkeer regels",
+ "form.feed.label.keeplist_rules": "toestemmingsregels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
@@ -2602,6 +2617,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
+ "form.feed.label.blocklist_rules": "Zasady blokowania",
+ "form.feed.label.keeplist_rules": "Zasady zezwoleń",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Ðе обновлÑÑ‚ÑŒ Ñтот канал",
@@ -2953,6 +2970,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
"form.feed.label.scraper_rules": "Regras do scraper",
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
+ "form.feed.label.blocklist_rules": "Regras de bloqueio",
+ "form.feed.label.keeplist_rules": "Regras de permissão",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
@@ -3282,6 +3301,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
+ "form.feed.label.blocklist_rules": "Правила блокировки",
+ "form.feed.label.keeplist_rules": "правила разрешений",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокÑи",
"form.feed.label.disabled": "Ðе обновлÑÑ‚ÑŒ Ñтот канал",
@@ -3613,6 +3634,8 @@ var translations = map[string]string{
"form.feed.label.user_agent": "覆盖默认 User-Agent",
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "é‡å†™è§„则",
+ "form.feed.label.blocklist_rules": "å°é”规则",
+ "form.feed.label.keeplist_rules": "许å¯è§„则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓å˜",
"form.feed.label.fetch_via_proxy": "通过代ç†èŽ·å–",
"form.feed.label.disabled": "请勿刷新æ¤Feed",
@@ -3706,15 +3729,15 @@ var translations = map[string]string{
}
var translationsChecksums = map[string]string{
- "de_DE": "dbd9141c3b2f436d16ff660735904897c235406094f0e7d70cbceb509c6063b6",
- "en_US": "61ef12ca29c271905a594ba1fe600871c930e7cfa25f0721bf076cd056cf60d4",
- "es_ES": "74e1b08adc78c9bd2fe05df69d19232fca86cb5fc94379ee42cef34878e4b8e5",
- "fr_FR": "3b61ad39a8d5227aac7b49897a6f78828c7e4a2040fe9636af932515c89d9785",
- "it_IT": "048498833a7c9bb519cde0a61f955e15cbcf117b2006955305fb23cc652ed9f4",
- "ja_JP": "cd8cac53f606066d60b3bc5586c0a484ee97466e3cca4816228ce74faf5916fe",
- "nl_NL": "b76c996efdbc814688ad3db50a906397d45717b19f49a4702b4e65f50df23600",
- "pl_PL": "a27aecbb6cd96d0714b2c934f5d96e01ab229e031e9e5ea0de96cec8c5cae0bf",
- "pt_BR": "8e3cdaa0e39485a6eb5cf3942ec23b8f6a16e8efc551f52a35f74e444ebd70e3",
- "ru_RU": "92d224b6563777efe64e1972884547f27e7374074b9f4f1fc2eeb331f6c5597c",
- "zh_CN": "de59177700df00c27352d785c62f66ee49955b8b294f0eaf944ad7c4025838c7",
+ "de_DE": "8acdf65175293ab3684f945582f9f4adb442b5008afba3312a100bb75c7910d8",
+ "en_US": "0fdf8969fa460ffb6c21ad8afd624eef130dcd1bb18c489e3a7fd38e6ed2d563",
+ "es_ES": "86e8bf5fab817c536aa73bdf2aa0f5c09ddad6f7457b662a5e091304815aac05",
+ "fr_FR": "020deb5822f8bec9d94c9b9a387651efaef21f21239e7c203d76c71260bbbd54",
+ "it_IT": "77c5963a80c173e2257909b47e46cf7f1ecbbae773c61454fcc987d8b26e4a2a",
+ "ja_JP": "f2b6b61ac847f17031ab34284a3f3b6355c03fd1534164024e33d1659126e227",
+ "nl_NL": "4dba877744bb92911fc488c3cfa47aad605501d1ba351f7136b92cb4872fbd34",
+ "pl_PL": "10acf265cea096636f640d0e275b136fd06cf992d8a306eb418d6dcd21cb8b24",
+ "pt_BR": "dc853281eb52ae1225a36a9bd2c6bb3c62304834b3b1c092edcdec86823468b4",
+ "ru_RU": "46d039f6d50e48d841d09285d28810c609cd62dd821da4c68798490d57d1d090",
+ "zh_CN": "80e8a170246b0b90dffda393d897760ecb4e94b264cacc124db2be6845ac9361",
}
diff --git a/locale/translations/de_DE.json b/locale/translations/de_DE.json
index 0f41b21b15e01c024c1745a0887e298888cec0d8..ff17e58e0da7d9d755fac1039b22a1ef5e3178a4 100644
--- a/locale/translations/de_DE.json
+++ b/locale/translations/de_DE.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
+ "form.feed.label.blocklist_rules": "Regeln blockieren",
+ "form.feed.label.keeplist_rules": "Regeln einhalten",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ãœber Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
diff --git a/locale/translations/en_US.json b/locale/translations/en_US.json
index 47a5539e5874d52ff2b8b9f808b8e7ce1921ce7e..f4d8ae9ba46c09d489b716b8fe4f7909f1c7d859 100644
--- a/locale/translations/en_US.json
+++ b/locale/translations/en_US.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Override Default User Agent",
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
+ "form.feed.label.blocklist_rules": "Block Rules",
+ "form.feed.label.keeplist_rules": "Keep Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
diff --git a/locale/translations/es_ES.json b/locale/translations/es_ES.json
index d84d4664c15ff99252dbf9b3da0db4c99977710d..f945c621904a345b54bb97e8de88dc16778badb1 100644
--- a/locale/translations/es_ES.json
+++ b/locale/translations/es_ES.json
@@ -248,6 +248,9 @@
"form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
+ "form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",
+ "form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)",
+ "form.feed.label.blocklist_rules": "Reglas de Blacklist",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json
index 90f6c7e0ae3f778ece6ae529ca0a334175b62cbf..96b774e2356bb71f7905decba7aa9669dcc3cbbf 100644
--- a/locale/translations/fr_FR.json
+++ b/locale/translations/fr_FR.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
+ "form.feed.label.blocklist_rules": "Règles de blocage",
+ "form.feed.label.keeplist_rules": "Règles d'autorisation",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
diff --git a/locale/translations/it_IT.json b/locale/translations/it_IT.json
index a291feeb58fb65efbf4d6dd8b4ab7d7bece40d99..005f5103be4f91fcef2bb9101deb6337236f778c 100644
--- a/locale/translations/it_IT.json
+++ b/locale/translations/it_IT.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Usa user agent personalizzato",
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
+ "form.feed.label.blocklist_rules": "Regole di blocco",
+ "form.feed.label.keeplist_rules": "Regole di autorizzazione",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
diff --git a/locale/translations/ja_JP.json b/locale/translations/ja_JP.json
index b721e79027255be499ce7398c4df3f7b402d6054..8f88271cf5e14c67c3298bfb199e3e26a1cbefd7 100644
--- a/locale/translations/ja_JP.json
+++ b/locale/translations/ja_JP.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "ディフォルト㮠User Agent を上書ãã™ã‚‹",
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
+ "form.feed.label.blocklist_rules": "ブãƒãƒƒã‚¯ãƒ«ãƒ¼ãƒ«",
+ "form.feed.label.keeplist_rules": "許å¯è¦å‰‡",
"form.feed.label.ignore_http_cache": "HTTPã‚ャッシュを無視",
"form.feed.label.fetch_via_proxy": "プãƒã‚シ経由ã§ãƒ•ã‚§ãƒƒãƒ",
"form.feed.label.disabled": "ã“ã®ãƒ•ã‚£ãƒ¼ãƒ‰ã‚’æ›´æ–°ã—ãªã„",
diff --git a/locale/translations/nl_NL.json b/locale/translations/nl_NL.json
index e977ad57069454a672ac9d065c7cc4b0c6111e48..df640143dc86a251faa98fbb00aeb20b27b358f1 100644
--- a/locale/translations/nl_NL.json
+++ b/locale/translations/nl_NL.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Standaard User Agent overschrijven",
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
+ "form.feed.label.blocklist_rules": "Blokkeer regels",
+ "form.feed.label.keeplist_rules": "toestemmingsregels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
diff --git a/locale/translations/pl_PL.json b/locale/translations/pl_PL.json
index af52a4d063912903c402de9ec50171ddd54c8249..d7c594c1ef2fce832d28795bc288ece3e983eefc 100644
--- a/locale/translations/pl_PL.json
+++ b/locale/translations/pl_PL.json
@@ -250,6 +250,8 @@
"form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
+ "form.feed.label.blocklist_rules": "Zasady blokowania",
+ "form.feed.label.keeplist_rules": "Zasady zezwoleń",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Ðе обновлÑÑ‚ÑŒ Ñтот канал",
diff --git a/locale/translations/pt_BR.json b/locale/translations/pt_BR.json
index 53cf38dc048094e58ea5d64425df45600a67be15..a3dcee3ac6af4f5b5ae200514f7a9607e8ade17d 100644
--- a/locale/translations/pt_BR.json
+++ b/locale/translations/pt_BR.json
@@ -248,6 +248,8 @@
"form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
"form.feed.label.scraper_rules": "Regras do scraper",
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
+ "form.feed.label.blocklist_rules": "Regras de bloqueio",
+ "form.feed.label.keeplist_rules": "Regras de permissão",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
diff --git a/locale/translations/ru_RU.json b/locale/translations/ru_RU.json
index df5d1b9b275be791d2631b94f0cb3cd6eb623693..15558dd8d8f099df879c66e96181164991cf38cd 100644
--- a/locale/translations/ru_RU.json
+++ b/locale/translations/ru_RU.json
@@ -250,6 +250,8 @@
"form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
+ "form.feed.label.blocklist_rules": "Правила блокировки",
+ "form.feed.label.keeplist_rules": "правила разрешений",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокÑи",
"form.feed.label.disabled": "Ðе обновлÑÑ‚ÑŒ Ñтот канал",
diff --git a/locale/translations/zh_CN.json b/locale/translations/zh_CN.json
index 7eb8f13c758423f3abb4097029ac06104057241f..2b6dc6d8f2d622e91c5c94466eddd8766fbc0325 100644
--- a/locale/translations/zh_CN.json
+++ b/locale/translations/zh_CN.json
@@ -246,6 +246,8 @@
"form.feed.label.user_agent": "覆盖默认 User-Agent",
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "é‡å†™è§„则",
+ "form.feed.label.blocklist_rules": "å°é”规则",
+ "form.feed.label.keeplist_rules": "许å¯è§„则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓å˜",
"form.feed.label.fetch_via_proxy": "通过代ç†èŽ·å–",
"form.feed.label.disabled": "请勿刷新æ¤Feed",
diff --git a/model/feed.go b/model/feed.go
index f13f0be9352cf1693323c4edb611b878db4a0e7f..4f17f5f88ca64fe75653029149a210fc2276cf12 100644
--- a/model/feed.go
+++ b/model/feed.go
@@ -29,6 +29,8 @@ type Feed struct {
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
Crawler bool `json:"crawler"`
+ BlocklistRules string `json:"blocklist_rules"`
+ KeeplistRules string `json:"keeplist_rules"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
@@ -72,7 +74,7 @@ func (f *Feed) WithCategoryID(categoryID int64) {
}
// WithBrowsingParameters defines browsing parameters.
-func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) {
+func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules, blacklistRules, keeplistRules string, fetchViaProxy bool) {
f.Crawler = crawler
f.UserAgent = userAgent
f.Username = username
@@ -80,6 +82,8 @@ func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, passwor
f.ScraperRules = scraperRules
f.RewriteRules = rewriteRules
f.FetchViaProxy = fetchViaProxy
+ f.BlocklistRules = blacklistRules
+ f.KeeplistRules = keeplistRules
}
// WithError adds a new error message and increment the error counter.
diff --git a/model/feed_test.go b/model/feed_test.go
index 8dc9fa0fe5fb4f4d13f41358bab7f65dfd3e8d72..e8abf8dd7962b280eb0472346bbd9186b7903127 100644
--- a/model/feed_test.go
+++ b/model/feed_test.go
@@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) {
func TestFeedBrowsingParams(t *testing.T) {
feed := &Feed{}
- feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false)
+ feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", "Look a Rule", "Oh wow another Rule", false)
if !feed.Crawler {
t.Error(`The crawler must be activated`)
diff --git a/reader/feed/handler.go b/reader/feed/handler.go
index 5c81d08a074ac8a10b484324c9a907bb4c730aac..96609544bca98767701a7e4108fd909859c37d56 100644
--- a/reader/feed/handler.go
+++ b/reader/feed/handler.go
@@ -34,7 +34,7 @@ type Handler struct {
}
// CreateFeed fetch, parse and store a new feed.
-func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) {
+func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules, blocklistRules, keeplistRules string, fetchViaProxy bool) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@@ -65,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.UserID = userID
subscription.WithCategoryID(categoryID)
- subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy)
+ subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, blocklistRules, keeplistRules, fetchViaProxy)
subscription.WithClientResponse(response)
subscription.CheckedNow()
diff --git a/reader/processor/processor.go b/reader/processor/processor.go
index 811451761fbc919fde7bae5301a48e610ed829cf..6b48c7378e1973e2c0999ce66d961f8d9ab68470 100644
--- a/reader/processor/processor.go
+++ b/reader/processor/processor.go
@@ -5,6 +5,7 @@
package processor
import (
+ "regexp"
"time"
"miniflux.app/config"
@@ -19,9 +20,11 @@ import (
// ProcessFeedEntries downloads original web page for entries and apply filters.
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
+
+ filterFeedEntries(feed)
+
for _, entry := range feed.Entries {
logger.Debug("[Feed #%d] Processing entry %s", feed.ID, entry.URL)
-
if feed.Crawler {
if !store.EntryURLExists(feed.ID, entry.URL) {
startTime := time.Now()
@@ -51,6 +54,37 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
}
}
+/*
+Filters feed entries based on regex rules
+First we filter based on our keep list, then we remove those entries that match the block list
+*/
+func filterFeedEntries(feed *model.Feed) {
+ var filteredEntries []*model.Entry
+
+ if len(feed.KeeplistRules) > 0 {
+ for _, entry := range feed.Entries {
+ match, _ := regexp.MatchString(feed.KeeplistRules, entry.Title)
+ if match == true {
+ filteredEntries = append(filteredEntries, entry)
+ }
+ }
+ } else {
+ filteredEntries = feed.Entries
+ }
+ if len(feed.BlocklistRules) > 0 {
+ k := 0
+ for _, entry := range filteredEntries {
+ match, _ := regexp.MatchString(feed.BlocklistRules, entry.Title)
+ if match != true {
+ filteredEntries[k] = entry
+ k++
+ }
+ }
+ filteredEntries = filteredEntries[:k]
+ }
+ feed.Entries = filteredEntries
+}
+
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(entry *model.Entry) error {
startTime := time.Now()
diff --git a/reader/processor/processor_test.go b/reader/processor/processor_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..1c72609dcfa1a45ae093725b6f297e455b550838
--- /dev/null
+++ b/reader/processor/processor_test.go
@@ -0,0 +1,88 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package processor // import "miniflux.app/reader/processor"
+
+import (
+ "miniflux.app/reader/parser"
+ "testing"
+)
+
+func TestKeeplistRules(t *testing.T) {
+ data := `<?xml version="1.0"?>
+ <rss version="2.0">
+ <channel>
+ <title>SomeGood News</title>
+ <link>http://foo.bar/</link>
+ <item>
+ <title>Kitten News</title>
+ <link>http://kitties.today/daily-kitten</link>
+ <description>Kitten picture of the day.</description>
+ <pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
+ <guid>http://kitties.today</guid>
+ </item>
+ <item>
+ <title>Daily Covid DoomScrolling News</title>
+ <link>http://covid.doom/daily-panic-dose</link>
+ <description>Did you know that you can get COVID IN YOUR DREAMS?.</description>
+ <pubDate>Tue, 03 Jun 2020 09:39:21 GMT</pubDate>
+ <guid>http://covid.doom</guid>
+ </item>
+ </channel>
+ </rss>`
+
+ feed, err := parser.ParseFeed(data)
+ if err != nil {
+ t.Error(err)
+ }
+ if len(feed.Entries) != 2 {
+ t.Errorf("Error parsing feed")
+ }
+
+ //case insensitive
+ feed.KeeplistRules = "(?i)kitten"
+ filterFeedEntries(feed)
+ if len(feed.Entries) != 1 {
+ t.Errorf("Keeplist filter rule did not properly filter the feed")
+ }
+}
+
+func TestBlocklistRules(t *testing.T) {
+ data := `<?xml version="1.0"?>
+ <rss version="2.0">
+ <channel>
+ <title>SomeGood News</title>
+ <link>http://foo.bar/</link>
+ <item>
+ <title>Kitten News</title>
+ <link>http://kitties.today/daily-kitten</link>
+ <description>Kitten picture of the day.</description>
+ <pubDate>Tue, 03 Jun 2003 09:39:21 GMT</pubDate>
+ <guid>http://kitties.today</guid>
+ </item>
+ <item>
+ <title>Daily Covid DoomScrolling News</title>
+ <link>http://covid.doom/daily-panic-dose</link>
+ <description>Did you know that you can get COVID IN YOUR DREAMS?.</description>
+ <pubDate>Tue, 03 Jun 2020 09:39:21 GMT</pubDate>
+ <guid>http://covid.doom</guid>
+ </item>
+ </channel>
+ </rss>`
+
+ feed, err := parser.ParseFeed(data)
+ if err != nil {
+ t.Error(err)
+ }
+ if len(feed.Entries) != 2 {
+ t.Errorf("Error parsing feed")
+ }
+
+ //case insensitive
+ feed.BlocklistRules = "(?i)covid"
+ filterFeedEntries(feed)
+ if len(feed.Entries) != 1 {
+ t.Errorf("Keeplist filter rule did not properly filter the feed")
+ }
+}
diff --git a/storage/feed.go b/storage/feed.go
index 697ef095bc678c865e47d5c0bde22737bafd4af0..81b93a3c21991f8fae0bd129c8d32a6e8f59abf3 100644
--- a/storage/feed.go
+++ b/storage/feed.go
@@ -27,6 +27,8 @@ var feedListQuery = `
f.parsing_error_msg,
f.scraper_rules,
f.rewrite_rules,
+ f.blocklist_rules,
+ f.keeplist_rules,
f.crawler,
f.user_agent,
f.username,
@@ -180,6 +182,8 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F
f.parsing_error_msg,
f.scraper_rules,
f.rewrite_rules,
+ f.blocklist_rules,
+ f.keeplist_rules,
f.crawler,
f.user_agent,
f.username,
@@ -290,6 +294,8 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
&feed.ParsingErrorMsg,
&feed.ScraperRules,
&feed.RewriteRules,
+ &feed.BlocklistRules,
+ &feed.KeeplistRules,
&feed.Crawler,
&feed.UserAgent,
&feed.Username,
@@ -375,6 +381,8 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
f.parsing_error_msg,
f.scraper_rules,
f.rewrite_rules,
+ f.blocklist_rules,
+ f.keeplist_rules,
f.crawler,
f.user_agent,
f.username,
@@ -407,6 +415,8 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
&feed.ParsingErrorMsg,
&feed.ScraperRules,
&feed.RewriteRules,
+ &feed.BlocklistRules,
+ &feed.KeeplistRules,
&feed.Crawler,
&feed.UserAgent,
&feed.Username,
@@ -453,10 +463,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
disabled,
scraper_rules,
rewrite_rules,
+ blocklist_rules,
+ keeplist_rules,
fetch_via_proxy
)
VALUES
- ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
+ ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
RETURNING
id
`
@@ -476,6 +488,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.Disabled,
feed.ScraperRules,
feed.RewriteRules,
+ feed.BlocklistRules,
+ feed.KeeplistRules,
feed.FetchViaProxy,
).Scan(&feed.ID)
if err != nil {
@@ -523,16 +537,18 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
parsing_error_count=$9,
scraper_rules=$10,
rewrite_rules=$11,
- crawler=$12,
- user_agent=$13,
- username=$14,
- password=$15,
- disabled=$16,
- next_check_at=$17,
- ignore_http_cache=$18,
- fetch_via_proxy=$19
+ blocklist_rules=$12,
+ keeplist_rules=$13,
+ crawler=$14,
+ user_agent=$15,
+ username=$16,
+ password=$17,
+ disabled=$18,
+ next_check_at=$19,
+ ignore_http_cache=$20,
+ fetch_via_proxy=$21
WHERE
- id=$20 AND user_id=$21
+ id=$22 AND user_id=$23
`
_, err = s.db.Exec(query,
feed.FeedURL,
@@ -546,6 +562,8 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.ParsingErrorCount,
feed.ScraperRules,
feed.RewriteRules,
+ feed.BlocklistRules,
+ feed.KeeplistRules,
feed.Crawler,
feed.UserAgent,
feed.Username,
diff --git a/template/html/add_subscription.html b/template/html/add_subscription.html
index 74a6963fe1f2d8d5280d25c72f136b77e322fc02..d4653aeb819d9dfd46a85589bb8b2433560752f6 100644
--- a/template/html/add_subscription.html
+++ b/template/html/add_subscription.html
@@ -55,6 +55,12 @@
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
+
+ <label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
+ <input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
+
+ <label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
+ <input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
</div>
</details>
diff --git a/template/html/edit_feed.html b/template/html/edit_feed.html
index 8242da9eba778a549d822f04c57da2756510b866..f477525c6802206cab3bce83de42643cae949e46 100644
--- a/template/html/edit_feed.html
+++ b/template/html/edit_feed.html
@@ -64,6 +64,12 @@
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
+ <label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
+ <input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
+
+ <label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
+ <input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
+
<label for="form-category">{{ t "form.feed.label.category" }}</label>
<select id="form-category" name="category_id">
{{ range .categories }}
diff --git a/template/views.go b/template/views.go
index ba6a5c81a98f068354ca36889ea3bdc84899095d..0a52e2e7b51a90f7e6aaef872bc31e0decaec39e 100644
--- a/template/views.go
+++ b/template/views.go
@@ -86,6 +86,12 @@ var templateViewsMap = map[string]string{
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
+
+ <label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
+ <input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
+
+ <label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
+ <input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
</div>
</details>
@@ -589,6 +595,12 @@ var templateViewsMap = map[string]string{
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
+ <label for="form-blocklist-rules">{{ t "form.feed.label.blocklist_rules" }}</label>
+ <input type="text" name="blocklist_rules" id="form-blocklist-rules" value="{{ .form.BlocklistRules }}">
+
+ <label for="form-keeplist-rules">{{ t "form.feed.label.keeplist_rules" }}</label>
+ <input type="text" name="keeplist_rules" id="form-keeplist-rules" value="{{ .form.KeeplistRules }}">
+
<label for="form-category">{{ t "form.feed.label.category" }}</label>
<select id="form-category" name="category_id">
{{ range .categories }}
@@ -1582,7 +1594,7 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
- "add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032",
+ "add_subscription": "82bf0dfadd64d3b6eda323a8174dd1446665b0804d27ca8718a828488627b287",
"api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284",
"bookmark_entries": "eacbbdce7fa85ec66c4c12f02879daab562a17ff79f1aac1805617e83e3a3a42",
"categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9",
@@ -1593,7 +1605,7 @@ var templateViewsMapChecksums = map[string]string{
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",
- "edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b",
+ "edit_feed": "5de7626448c48de384a0388227ab0c3b75b1ec19b5de440c91039180852cc5dc",
"edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a",
"entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c",
"feed_entries": "89977ea86b8d43305d587b70e6d9c45c2c88249b3966f2d31051dc7a5f1c48b6",
diff --git a/tests/feed_test.go b/tests/feed_test.go
index eba68bae1931863475497f49e0b8e6c9a55c39f3..e7841a4a991a671240cf2a8373839c62ff5d5fcf 100644
--- a/tests/feed_test.go
+++ b/tests/feed_test.go
@@ -195,6 +195,31 @@ func TestUpdateFeedRewriteRules(t *testing.T) {
}
}
+func TestUpdateFeedKeeplistRules(t *testing.T) {
+ client := createClient(t)
+ feed, _ := createFeed(t, client)
+
+ keeplistRules := "test"
+ updatedFeed, err := client.UpdateFeed(feed.ID, &miniflux.FeedModification{KeeplistRules: &keeplistRules})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if updatedFeed.KeeplistRules != keeplistRules {
+ t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules)
+ }
+
+ keeplistRules = ""
+ updatedFeed, err = client.UpdateFeed(feed.ID, &miniflux.FeedModification{KeeplistRules: &keeplistRules})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if updatedFeed.KeeplistRules != keeplistRules {
+ t.Fatalf(`Wrong KeeplistRules value, got "%v" instead of "%v"`, updatedFeed.KeeplistRules, keeplistRules)
+ }
+}
+
func TestUpdateFeedUserAgent(t *testing.T) {
client := createClient(t)
feed, _ := createFeed(t, client)
diff --git a/ui/feed_edit.go b/ui/feed_edit.go
index 199058287ef5f1434441718f65d3165df40e8fbf..07d6b68f6c59062eb24f3033f16a5f174b31e016 100644
--- a/ui/feed_edit.go
+++ b/ui/feed_edit.go
@@ -47,6 +47,8 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
Title: feed.Title,
ScraperRules: feed.ScraperRules,
RewriteRules: feed.RewriteRules,
+ BlocklistRules: feed.BlocklistRules,
+ KeeplistRules: feed.KeeplistRules,
Crawler: feed.Crawler,
UserAgent: feed.UserAgent,
CategoryID: feed.Category.ID,
diff --git a/ui/form/feed.go b/ui/form/feed.go
index eeeeb7b07069dd4bd477a67316281b081f3d4080..23db7952405ae333cc70bceec60069b9501388fb 100644
--- a/ui/form/feed.go
+++ b/ui/form/feed.go
@@ -19,6 +19,8 @@ type FeedForm struct {
Title string
ScraperRules string
RewriteRules string
+ BlocklistRules string
+ KeeplistRules string
Crawler bool
UserAgent string
CategoryID int64
@@ -45,6 +47,8 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.FeedURL = f.FeedURL
feed.ScraperRules = f.ScraperRules
feed.RewriteRules = f.RewriteRules
+ feed.BlocklistRules = f.BlocklistRules
+ feed.KeeplistRules = f.KeeplistRules
feed.Crawler = f.Crawler
feed.UserAgent = f.UserAgent
feed.ParsingErrorCount = 0
@@ -63,7 +67,6 @@ func NewFeedForm(r *http.Request) *FeedForm {
if err != nil {
categoryID = 0
}
-
return &FeedForm{
FeedURL: r.FormValue("feed_url"),
SiteURL: r.FormValue("site_url"),
@@ -71,6 +74,8 @@ func NewFeedForm(r *http.Request) *FeedForm {
ScraperRules: r.FormValue("scraper_rules"),
UserAgent: r.FormValue("user_agent"),
RewriteRules: r.FormValue("rewrite_rules"),
+ BlocklistRules: r.FormValue("blocklist_rules"),
+ KeeplistRules: r.FormValue("keeplist_rules"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
Username: r.FormValue("feed_username"),
diff --git a/ui/form/subscription.go b/ui/form/subscription.go
index 74be3c0c7101dad1171d14a5b39873e765e2ebb9..31aef6742f19c19daabd7232c23d2d55b3325ddf 100644
--- a/ui/form/subscription.go
+++ b/ui/form/subscription.go
@@ -13,15 +13,17 @@ import (
// SubscriptionForm represents the subscription form.
type SubscriptionForm struct {
- URL string
- CategoryID int64
- Crawler bool
- FetchViaProxy bool
- UserAgent string
- Username string
- Password string
- ScraperRules string
- RewriteRules string
+ URL string
+ CategoryID int64
+ Crawler bool
+ FetchViaProxy bool
+ UserAgent string
+ Username string
+ Password string
+ ScraperRules string
+ RewriteRules string
+ BlocklistRules string
+ KeeplistRules string
}
// Validate makes sure the form values are valid.
@@ -41,14 +43,15 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
}
return &SubscriptionForm{
- URL: r.FormValue("url"),
- Crawler: r.FormValue("crawler") == "1",
- FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
- CategoryID: int64(categoryID),
- UserAgent: r.FormValue("user_agent"),
- Username: r.FormValue("feed_username"),
- Password: r.FormValue("feed_password"),
- ScraperRules: r.FormValue("scraper_rules"),
- RewriteRules: r.FormValue("rewrite_rules"),
+ URL: r.FormValue("url"),
+ Crawler: r.FormValue("crawler") == "1",
+ CategoryID: int64(categoryID),
+ UserAgent: r.FormValue("user_agent"),
+ Username: r.FormValue("feed_username"),
+ Password: r.FormValue("feed_password"),
+ ScraperRules: r.FormValue("scraper_rules"),
+ RewriteRules: r.FormValue("rewrite_rules"),
+ BlocklistRules: r.FormValue("blocklist_rules"),
+ KeeplistRules: r.FormValue("keeplist_rules"),
}
}
diff --git a/ui/subscription_choose.go b/ui/subscription_choose.go
index 68390e18a278ac94ab53f557a5bc5aa31d86abea..151cedb3681c175799d9094d437dc5ffc62a3d31 100644
--- a/ui/subscription_choose.go
+++ b/ui/subscription_choose.go
@@ -57,6 +57,8 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
+ subscriptionForm.BlocklistRules,
+ subscriptionForm.KeeplistRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
diff --git a/ui/subscription_submit.go b/ui/subscription_submit.go
index 0eb1178833ae3c223af87ec50f632e271ab501ae..11751a8d5a415ca221c3995a0feb7aa22fe2a8af 100644
--- a/ui/subscription_submit.go
+++ b/ui/subscription_submit.go
@@ -85,6 +85,8 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
+ subscriptionForm.BlocklistRules,
+ subscriptionForm.KeeplistRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {