From 3b41237e4b4f08903ea435d6df4762c9dda573ec Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 12 Jan 2017 08:45:46 -0500 Subject: [PATCH 01/45] fix #2301 --- src/js/logger-ui.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/logger-ui.js b/src/js/logger-ui.js index ba05a6bd19b30..ad398f17352e0 100644 --- a/src/js/logger-ui.js +++ b/src/js/logger-ui.js @@ -543,7 +543,7 @@ var renderLogEntry = function(entry) { var time = logDate; time.setTime(entry.tstamp - logDateTimezoneOffset); tr.cells[0].textContent = padTo2(time.getUTCHours()) + ':' + - padTo2(time.getMinutes()) + ':' + + padTo2(time.getUTCMinutes()) + ':' + padTo2(time.getSeconds()); if ( entry.tab ) { From 54032e520b4259f010517e55381cdaa286d2d1c0 Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 17 Jan 2017 18:18:28 -0500 Subject: [PATCH 02/45] fix https://github.com/gorhill/uBO-Extra/issues/19 --- src/js/traffic.js | 143 ++++++++++++++++++---------------------------- 1 file changed, 56 insertions(+), 87 deletions(-) diff --git a/src/js/traffic.js b/src/js/traffic.js index 1c67e0de52c28..6e4587d0fca45 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -386,95 +386,69 @@ var onBeforeBehindTheSceneRequest = function(details) { // To handle: // - inline script tags +// - websockets // - media elements larger than n kB var onHeadersReceived = function(details) { // Do not interfere with behind-the-scene requests. var tabId = details.tabId; - if ( vAPI.isBehindTheSceneTabId(tabId) ) { - return; - } + if ( vAPI.isBehindTheSceneTabId(tabId) ) { return; } - var requestType = details.type; + var µb = µBlock, + requestType = details.type; if ( requestType === 'main_frame' ) { - return onRootFrameHeadersReceived(details); - } - - if ( requestType === 'sub_frame' ) { - return onFrameHeadersReceived(details); + µb.tabContextManager.push(tabId, details.url); } - if ( requestType === 'image' || requestType === 'media' ) { - return foilLargeMediaElement(details); - } -}; - -/******************************************************************************/ - -var onRootFrameHeadersReceived = function(details) { - var µb = µBlock, - tabId = details.tabId; - - µb.tabContextManager.push(tabId, details.url); - - // Lookup the page store associated with this tab id. var pageStore = µb.pageStoreFromTabId(tabId); - if ( !pageStore ) { + if ( pageStore === null ) { + if ( requestType !== 'main_frame' ) { return; } pageStore = µb.bindTabToPageStats(tabId, 'beforeRequest'); } - // I can't think of how pageStore could be null at this point. - - return processCSP(details, pageStore, pageStore.createContextFromPage()); -}; + if ( pageStore.getNetFilteringSwitch() === false ) { return; } -/******************************************************************************/ - -var onFrameHeadersReceived = function(details) { - // Lookup the page store associated with this tab id. - var pageStore = µBlock.pageStoreFromTabId(details.tabId); - if ( !pageStore ) { - return; + if ( requestType === 'image' || requestType === 'media' ) { + return foilLargeMediaElement(pageStore, details); } - // Frame id of frame request is their own id, while the request is made - // in the context of the parent. - return processCSP( - details, - pageStore, - pageStore.createContextFromFrameId(details.frameId) - ); + // https://github.com/gorhill/uBO-Extra/issues/19 + // Turns out scripts must also be considered as potential embedded + // contexts (as workers) and as such we may need to inject content + // security policy directives. + if ( requestType === 'script' || requestType === 'main_frame' || requestType === 'sub_frame' ) { + return processCSP(pageStore, details); + } }; /******************************************************************************/ -var processCSP = function(details, pageStore, context) { +var processCSP = function(pageStore, details) { var µb = µBlock, tabId = details.tabId, requestURL = details.url, loggerEnabled = µb.logger.isEnabled(); + var context = pageStore.createContextFromPage(); context.requestURL = requestURL; context.requestHostname = µb.URI.hostnameFromURI(requestURL); + if ( details.type !== 'main_frame' ) { + context.pageHostname = context.pageDomain = context.requestHostname; + } - context.requestType = 'inline-script'; - var inlineScriptResult = pageStore.filterRequestNoCache(context), + var inlineScriptResult, blockInlineScript; + if ( details.type !== 'script' ) { + context.requestType = 'inline-script'; + inlineScriptResult = pageStore.filterRequestNoCache(context); blockInlineScript = µb.isBlockResult(inlineScriptResult); + } context.requestType = 'websocket'; µb.staticNetFilteringEngine.matchStringExactType(context, requestURL, 'websocket'); var websocketResult = µb.staticNetFilteringEngine.toResultString(loggerEnabled), blockWebsocket = µb.isBlockResult(websocketResult); - // https://github.com/gorhill/uBlock/issues/2050 - // Blanket-blocking websockets is exceptional, so we test whether the - // page is whitelisted if and only if there is a hit against a websocket - // filter. - if ( blockWebsocket && pageStore.getNetFilteringSwitch() === false ) { - websocketResult = ''; - blockWebsocket = false; - } - var headersChanged = false; + var headersChanged; if ( blockInlineScript || blockWebsocket ) { headersChanged = foilWithCSP( details.responseHeaders, @@ -484,34 +458,33 @@ var processCSP = function(details, pageStore, context) { } if ( loggerEnabled ) { - µb.logger.writeOne( - tabId, - 'net', - inlineScriptResult, - 'inline-script', - requestURL, - context.rootHostname, - context.pageHostname - ); - } - - if ( loggerEnabled && blockWebsocket ) { - µb.logger.writeOne( - tabId, - 'net', - websocketResult, - 'websocket', - requestURL, - context.rootHostname, - context.pageHostname - ); + if ( blockInlineScript !== undefined ) { + µb.logger.writeOne( + tabId, + 'net', + inlineScriptResult, + 'inline-script', + requestURL, + context.rootHostname, + context.pageHostname + ); + } + if ( websocketResult !== '' ) { + µb.logger.writeOne( + tabId, + 'net', + websocketResult, + 'websocket', + requestURL, + context.rootHostname, + context.pageHostname + ); + } } context.dispose(); - if ( headersChanged !== true ) { - return; - } + if ( headersChanged !== true ) { return; } µb.updateBadgeAsync(tabId); @@ -523,19 +496,14 @@ var processCSP = function(details, pageStore, context) { // https://github.com/gorhill/uBlock/issues/1163 // "Block elements by size" -var foilLargeMediaElement = function(details) { +var foilLargeMediaElement = function(pageStore, details) { var µb = µBlock; var i = headerIndexFromName('content-length', details.responseHeaders); if ( i === -1 ) { return; } var tabId = details.tabId, - pageStore = µb.pageStoreFromTabId(tabId); - if ( pageStore === null || pageStore.getNetFilteringSwitch() === false ) { - return; - } - - var size = parseInt(details.responseHeaders[i].value, 10) || 0, + size = parseInt(details.responseHeaders[i].value, 10) || 0, result = pageStore.filterLargeMediaElement(size); if ( result === undefined ) { return; } @@ -691,7 +659,8 @@ vAPI.net.onHeadersReceived = { 'main_frame', 'sub_frame', 'image', - 'media' + 'media', + 'script' ], extra: [ 'blocking', 'responseHeaders' ], callback: onHeadersReceived From 69fc59305ec60607593474b238e476f808ec7294 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 13:01:13 -0500 Subject: [PATCH 03/45] fix https://github.com/uBlockOrigin/uAssets/issues/263#issuecomment-272615772 --- src/js/tab.js | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/js/tab.js b/src/js/tab.js index 4ddbc6dda25e4..9b9976d6263ce 100644 --- a/src/js/tab.js +++ b/src/js/tab.js @@ -561,26 +561,23 @@ vAPI.tabs.onPopupUpdated = (function() { context.requestHostname = µb.URI.hostnameFromURI(targetURL); context.requestType = 'popup'; - // https://github.com/gorhill/uBlock/commit/1d448b85b2931412508aa01bf899e0b6f0033626#commitcomment-14944764 - // Ignore bad target URL. On Firefox, an `about:blank` tab may be - // opened for a new tab before it is filled in with the real target - // URL. // https://github.com/gorhill/uBlock/issues/1735 // Do not bail out on `data:` URI, they are commonly used for popups. // https://github.com/uBlockOrigin/uAssets/issues/255 // Do not bail out on `about:blank`: an `about:blank` popup can be // opened, with the sole purpose to serve as an intermediary in // a sequence of chained popups. - if ( - context.requestHostname === '' && - targetURL.startsWith('data:') === false && - targetURL !== 'about:blank' - ) { - return ''; - } + // https://github.com/uBlockOrigin/uAssets/issues/263#issuecomment-272615772 + // Do not bail out, period: the static filtering engine must be + // able to examine all sorts of URLs for popup filtering purpose. - // Dynamic filtering makes sense only when we have a valid hostname. - if ( openerHostname !== '' ) { + // Dynamic filtering makes sense only when we have a valid opener + // hostname. + // https://github.com/gorhill/uBlock/commit/1d448b85b2931412508aa01bf899e0b6f0033626#commitcomment-14944764 + // Ignore bad target URL. On Firefox, an `about:blank` tab may be + // opened for a new tab before it is filled in with the real target + // URL. + if ( openerHostname !== '' && targetURL !== 'about:blank' ) { // Check per-site switch first if ( µb.hnSwitches.evaluateZ('no-popups', openerHostname) ) { if ( typeof clickedURL === 'string' && areDifferentURLs(targetURL, clickedURL) ) { From 3b9fd49c508812dfe2545836c63bd5d8cfd2ead4 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Wed, 18 Jan 2017 13:17:47 -0500 Subject: [PATCH 04/45] Assets management refactored (#2314) * refactoring assets management code * finalizing refactoring of assets management * various code review of new assets management code * fix #2281 * fix #1961 * fix #1293 * fix #1275 * fix update scheduler timing logic * forward compatibility (to be removed once 1.11+ is widespread) * more codereview; give admins ability to specify own assets.json * "assetKey" is more accurate than "path" * fix group count update when building dom incrementally * reorganize content (order, added URLs, etc.) * ability to customize updater through advanced settings * better spinner icon --- assets/assets.json | 585 +++++++++ src/3p-filters.html | 16 +- src/_locales/en/messages.json | 4 +- src/background.html | 1 - src/css/3p-filters.css | 91 +- src/js/3p-filters.js | 433 +++---- src/js/assets.js | 2094 +++++++++++-------------------- src/js/background.js | 105 +- src/js/logger.js | 4 +- src/js/messaging.js | 89 +- src/js/redirect-engine.js | 24 +- src/js/reverselookup-worker.js | 18 +- src/js/reverselookup.js | 38 +- src/js/scriptlets/subscriber.js | 9 +- src/js/settings.js | 5 +- src/js/start.js | 27 +- src/js/storage.js | 957 +++++++------- src/js/ublock.js | 8 +- tools/make-assets.sh | 4 +- tools/make-chromium.sh | 6 +- 20 files changed, 2082 insertions(+), 2436 deletions(-) create mode 100644 assets/assets.json diff --git a/assets/assets.json b/assets/assets.json new file mode 100644 index 0000000000000..883b87ba01ee4 --- /dev/null +++ b/assets/assets.json @@ -0,0 +1,585 @@ +{ + "assets.json": { + "content": "internal", + "updateAfter": 13, + "contentURL": [ + "https://raw.githubusercontent.com/gorhill/uBlock/master/assets/assets.json", + "assets/assets.json" + ] + }, + "public_suffix_list.dat": { + "content": "internal", + "updateAfter": 19, + "contentURL": [ + "https://publicsuffix.org/list/public_suffix_list.dat", + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat" + ] + }, + "ublock-resources": { + "content": "internal", + "updateAfter": 7, + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resources.txt", + "assets/ublock/resources.txt" + ] + }, + "ublock-filters": { + "content": "filters", + "group": "default", + "title": "uBlock filters", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", + "assets/ublock/filters.txt" + ] + }, + "ublock-badware": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Badware risks", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt", + "assets/ublock/badware.txt" + ], + "supportURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks", + "instructionURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks" + }, + "ublock-experimental": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Experimental", + "off": true, + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/experimental.txt", + "assets/ublock/experimental.txt" + ], + "supportURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters", + "instructionURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters" + }, + "ublock-privacy": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Privacy", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt", + "assets/ublock/privacy.txt" + ] + }, + "ublock-unbreak": { + "content": "filters", + "group": "default", + "title": "uBlock filters – Unbreak", + "contentURL": [ + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt", + "assets/ublock/unbreak.txt" + ] + }, + "awrl-0": { + "content": "filters", + "group": "ads", + "off": true, + "title": "Adblock Warning Removal List", + "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt", + "supportURL": "https://forums.lanik.us/" + }, + "reek-0": { + "content": "filters", + "group": "ads", + "off": true, + "title": "Anti-Adblock Killer | Reek", + "contentURL": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", + "supportURL": "https://github.com/reek/anti-adblock-killer", + "instructionURL": "https://github.com/reek/anti-adblock-killer#instruction" + }, + "easylist": { + "content": "filters", + "group": "ads", + "title": "EasyList", + "contentURL": [ + "https://easylist.to/easylist/easylist.txt", + "https://easylist-downloads.adblockplus.org/easylist.txt", + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easylist.txt", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "easylist-nocosmetic": { + "content": "filters", + "group": "ads", + "off": true, + "title": "EasyList without element hiding rules", + "contentURL": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", + "supportURL": "https://forums.lanik.us/" + }, + "disconnect-tracking": { + "content": "filters", + "group": "privacy", + "off": true, + "title": "Basic tracking list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" + }, + "easyprivacy": { + "content": "filters", + "group": "privacy", + "title": "EasyPrivacy", + "contentURL": [ + "https://easylist.to/easylist/easyprivacy.txt", + "https://easylist-downloads.adblockplus.org/easyprivacy.txt", + "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-enhanced": { + "content": "filters", + "group": "privacy", + "off": true, + "title": "Fanboy’s Enhanced Tracking List", + "contentURL": "https://www.fanboy.co.nz/enhancedstats.txt", + "supportURL": "https://forums.lanik.us/" + }, + "disconnect-malvertising": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Malvertising filter list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt" + }, + "malware-0": { + "content": "filters", + "group": "malware", + "title": "Malware Domain List", + "contentURL": [ + "https://www.malwaredomainlist.com/hostslist/hosts.txt", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt" + ] + }, + "malware-1": { + "content": "filters", + "group": "malware", + "title": "Malware domains", + "contentURL": [ + "https://mirror.cedia.org.ec/malwaredomains/justdomains", + "https://mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" + }, + "malware-2": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Malware domains (long-lived)", + "contentURL": [ + "https://mirror1.malwaredomains.com/files/immortal_domains.txt", + "https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" + }, + "disconnect-malware": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Malware filter list by Disconnect", + "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt" + }, + "spam404-0": { + "content": "filters", + "group": "malware", + "off": true, + "title": "Spam404", + "contentURL": "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt", + "supportURL": "http://www.spam404.com/" + }, + "fanboy-thirdparty_social": { + "content": "filters", + "group": "social", + "off": true, + "title": "Anti-ThirdpartySocial (see warning inside list)", + "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-annoyance": { + "content": "filters", + "group": "social", + "off": true, + "title": "Fanboy’s Annoyance List", + "contentURL": [ + "https://easylist.to/easylist/fanboy-annoyance.txt", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "fanboy-social": { + "content": "filters", + "group": "social", + "off": true, + "title": "Fanboy’s Social Blocking List", + "contentURL": [ + "https://easylist.to/easylist/fanboy-social.txt", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt" + ], + "supportURL": "https://forums.lanik.us/" + }, + "dpollock-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "off": true, + "title": "Dan Pollock’s hosts file", + "contentURL": "http://someonewhocares.org/hosts/hosts", + "supportURL": "http://someonewhocares.org/hosts/" + }, + "fanboy-ultimate": { + "content": "filters", + "group": "multipurpose", + "off": true, + "title": "Fanboy+Easylist-Merged Ultimate List", + "contentURL": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", + "supportURL": "https://forums.lanik.us/" + }, + "hphosts": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "off": true, + "title": "hpHosts’ Ad and tracking servers", + "contentURL": "https://hosts-file.net/.%5Cad_servers.txt", + "supportURL": "https://hosts-file.net/" + }, + "mvps-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "off": true, + "title": "MVPS HOSTS", + "contentURL": "http://winhelp2002.mvps.org/hosts.txt", + "supportURL": "http://winhelp2002.mvps.org/" + }, + "plowe-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 13, + "title": "Peter Lowe’s Ad and tracking server list", + "contentURL": [ + "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", + "assets/thirdparties/pgl.yoyo.org/as/serverlist", + "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt" + ], + "supportURL": "https://pgl.yoyo.org/adservers/" + }, + "ara-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ara: Liste AR", + "lang": "ar", + "contentURL": "https://easylist-downloads.adblockplus.org/Liste_AR.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=98" + }, + "BGR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "BGR: Bulgarian Adblock list", + "lang": "bg", + "contentURL": "https://stanev.org/abp/adblock_bg.txt", + "supportURL": "https://stanev.org/abp/" + }, + "CHN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CHN: EasyList China (中文)", + "lang": "zh", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistchina.txt", + "supportURL": "http://abpchina.org/forum/forum.php" + }, + "CHN-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CHN: CJX's EasyList Lite (main focus on Chinese sites)", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" + }, + "CHN-2": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CHN: CJX's Annoyance List", + "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt", + "supportURL": "https://github.com/cjx82630/cjxlist" + }, + "CZE-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "CZE, SVK: EasyList Czech and Slovak", + "lang": "cs", + "contentURL": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", + "supportURL": "https://github.com/tomasko126/easylistczechandslovak" + }, + "DEU-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "DEU: EasyList Germany", + "lang": "de", + "contentURL": [ + "https://easylist.to/easylistgermany/easylistgermany.txt", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt" + ], + "supportURL": "https://forums.lanik.us/viewforum.php?f=90" + }, + "DNK-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "DNK: Schacks Adblock Plus liste", + "lang": "da", + "contentURL": "https://adblock.dk/block.csv", + "supportURL": "https://henrik.schack.dk/adblock/" + }, + "EST-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "EST: Eesti saitidele kohandatud filter", + "lang": "et", + "contentURL": "http://adblock.ee/list.php", + "supportURL": "http://adblock.ee/" + }, + "EU-prebake": { + "content": "filters", + "group": "regions", + "off": true, + "title": "EU: Prebake - Filter Obtrusive Cookie Notices", + "contentURL": "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt", + "supportURL": "https://github.com/liamja/Prebake" + }, + "FIN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "FIN: Finnish Addition to Easylist", + "lang": "fi", + "contentURL": "http://adb.juvander.net/Finland_adb.txt", + "supportURL": "http://www.juvander.fi/AdblockFinland" + }, + "FRA-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "FRA: EasyList Liste FR", + "lang": "fr", + "contentURL": "https://easylist-downloads.adblockplus.org/liste_fr.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=91" + }, + "GRC-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "GRC: Greek AdBlock Filter", + "lang": "el", + "contentURL": "https://www.void.gr/kargig/void-gr-filters.txt", + "supportURL": "https://github.com/kargig/greek-adblockplus-filter" + }, + "HUN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "HUN: hufilter", + "lang": "hu", + "contentURL": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", + "supportURL": "https://github.com/szpeter80/hufilter" + }, + "IDN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "IDN: ABPindo", + "lang": "id", + "contentURL": [ + "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt" + ], + "supportURL": "https://github.com/ABPindo/indonesianadblockrules" + }, + "ISL-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ISL: Icelandic ABP List", + "lang": "is", + "contentURL": "http://adblock.gardar.net/is.abp.txt", + "supportURL": "http://adblock.gardar.net/" + }, + "ISR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ISR: EasyList Hebrew", + "lang": "he", + "contentURL": "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt", + "supportURL": "https://github.com/easylist/EasyListHebrew" + }, + "ITA-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ITA: EasyList Italy", + "lang": "it", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=96" + }, + "ITA-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "ITA: ABP X Files", + "contentURL": "https://raw.githubusercontent.com/gioxx/xfiles/master/filtri.txt", + "supportURL": "http://noads.it/" + }, + "JPN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "JPN: ABP Japanese filters (日本用フィルタ)", + "lang": "ja", + "contentURL": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt", + "supportURL": "https://github.com/k2jp/abp-japanese-filters/wiki/Support_Policy" + }, + "KOR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "KOR: Korean Adblock List", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt", + "supportURL": "https://github.com/gfmaster/adblock-korea-contrib" + }, + "KOR-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "KOR: YousList", + "lang": "ko", + "contentURL": "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt", + "supportURL": "https://github.com/yous/YousList" + }, + "KOR-2": { + "content": "filters", + "group": "regions", + "off": true, + "title": "KOR: Fanboy's Korean", + "contentURL": "https://www.fanboy.co.nz/fanboy-korean.txt", + "supportURL": "https://forums.lanik.us/" + }, + "LTU-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "LTU: Adblock Plus Lithuania", + "lang": "lt", + "contentURL": "http://margevicius.lt/easylistlithuania.txt", + "supportURL": "http://margevicius.lt/easylist_lithuania/" + }, + "LVA-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "LVA: Latvian List", + "lang": "lv", + "contentURL": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", + "supportURL": "https://notabug.org/latvian-list/adblock-latvian" + }, + "NLD-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "NLD: EasyList Dutch", + "lang": "nl", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=100" + }, + "POL-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "POL: polskie filtry do Adblocka i uBlocka", + "lang": "pl", + "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", + "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" + }, + "RUS-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: RU AdList (Дополнительная региональная подписка)", + "lang": "ru", + "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: BitBlock List (Дополнительная подписка фильтров)", + "contentURL": "https://easylist-downloads.adblockplus.org/bitblock.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=102" + }, + "RUS-2": { + "content": "filters", + "group": "regions", + "off": true, + "title": "RUS: Adguard Russian Filter", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/1.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?69-%D0%A4%D0%B8%D0%BB%D1%8C%D1%82%D1%80%D1%8B-Adguard" + }, + "spa-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "spa: EasyList Spanish", + "lang": "es", + "contentURL": "https://easylist-downloads.adblockplus.org/easylistspanish.txt", + "supportURL": "https://forums.lanik.us/viewforum.php?f=103" + }, + "SVN-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "SVN: Slovenian List", + "lang": "sl", + "contentURL": "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt", + "supportURL": "https://github.com/betterwebleon/slovenian-list" + }, + "SWE-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "SWE: Fanboy's Swedish", + "lang": "sv", + "contentURL": "https://www.fanboy.co.nz/fanboy-swedish.txt", + "supportURL": "https://forums.lanik.us/" + }, + "TUR-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "TUR: Adguard Turkish Filter", + "lang": "tr", + "contentURL": "https://filters.adtidy.org/extension/chromium/filters/13.txt", + "supportURL": "https://forum.adguard.com/forumdisplay.php?51-Filter-Rules" + }, + "VIE-0": { + "content": "filters", + "group": "regions", + "off": true, + "title": "VIE: Fanboy's Vietnamese", + "lang": "vi", + "contentURL": "https://www.fanboy.co.nz/fanboy-vietnam.txt", + "supportURL": "https://forums.lanik.us/" + } +} diff --git a/src/3p-filters.html b/src/3p-filters.html index 8bce9d01aaab4..48086f71fe271 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -40,16 +40,10 @@

-
-
- -
-
- diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index b31059c90154c..653cf75c40c94 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -580,8 +580,8 @@ "description": "Message asking user to confirm reset" }, "errorCantConnectTo":{ - "message":"Unable to connect to {{url}}", - "description":"English: Network error: unable to connect to {{url}}" + "message":"Network error: {{msg}}", + "description":"English: Network error: {{msg}}" }, "subscriberConfirm":{ "message":"uBlock₀: Add the following URL to your custom filter lists?\n\nTitle: \"{{title}}\"\nURL: {{url}}", diff --git a/src/background.html b/src/background.html index a8463068e76c3..0d56942fc2b0e 100644 --- a/src/background.html +++ b/src/background.html @@ -8,7 +8,6 @@ - diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index 99f87cfd48235..8aa6760f9cf51 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -1,3 +1,7 @@ +@keyframes spin { + 100% { transform: rotate(360deg); -webkit-transform: rotate(360deg); } + } + ul { padding: 0; list-style-type: none; @@ -88,7 +92,7 @@ body[dir=rtl] #buttonApply { span.status { border: 1px solid transparent; color: #444; - display: inline-block; + display: none; font-size: smaller; line-height: 1; margin: 0 0 0 0.5em; @@ -99,6 +103,16 @@ span.unsecure { background-color: hsl(0, 100%, 88%); border-color: hsl(0, 100%, 83%); } +li.listEntry.unsecure span.unsecure { + display: inline; + } +span.obsolete { + background-color: hsl(36, 100%, 80%); + border-color: hsl(36, 100%, 75%); + } +li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { + display: inline; + } span.purge { border-color: #ddd; background-color: #eee; @@ -107,10 +121,16 @@ span.purge { span.purge:hover { opacity: 1; } -span.obsolete, -span.new { - background-color: hsl(36, 100%, 80%); - border-color: hsl(36, 100%, 75%); +li.listEntry.cached span.purge { + display: inline; + } +span.updating { + border: none; + padding: 0; + } +li.listEntry.updating span.updating { + animation: spin 2s linear infinite; + display: inline-block; } #externalListsDiv { margin: 2em auto 0 2em; @@ -125,64 +145,3 @@ body[dir=rtl] #externalListsDiv { width: 100%; word-wrap: normal; } -body #busyOverlay { - background-color: transparent; - bottom: 0; - cursor: wait; - display: none; - left: 0; - position: fixed; - right: 0; - top: 0; - z-index: 1000; - } -body.busy #busyOverlay { - display: block; - } -#busyOverlay > div:nth-of-type(1) { - background-color: white; - bottom: 0; - left: 0; - opacity: 0.75; - position: absolute; - right: 0; - top: 0; - } -#busyOverlay > div:nth-of-type(2) { - background-color: #eee; - border: 1px solid transparent; - border-color: #80b3ff #80b3ff hsl(216, 100%, 75%); - border-radius: 3px; - box-sizing: border-box; - height: 3em; - left: 10%; - position: absolute; - bottom: 75%; - width: 80%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(1) { - background-color: hsl(216, 100%, 75%); - background-image: linear-gradient(#a8cbff, #80b3ff); - background-repeat: repeat-x; - border: 0; - box-sizing: border-box; - color: #222; - height: 100%; - left: 0; - padding: 0; - position: absolute; - width: 25%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(2) { - background-color: transparent; - border: 0; - box-sizing: border-box; - height: 100%; - left: 0; - line-height: 3em; - overflow: hidden; - position: absolute; - text-align: center; - top: 0; - width: 100%; - } diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 0b960e81c0911..4b7dfa1b9dfea 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,38 +21,30 @@ /* global uDom */ +'use strict'; + /******************************************************************************/ (function() { -'use strict'; - /******************************************************************************/ -var userListName = vAPI.i18n('1pPageName'); var listDetails = {}; var parseCosmeticFilters = true; var ignoreGenericCosmeticFilters = false; +var selectedListsHashBefore = ''; var externalLists = ''; -var cacheWasPurged = false; -var needUpdate = false; -var hasCachedContent = false; /******************************************************************************/ var onMessage = function(msg) { switch ( msg.what ) { + case 'assetUpdated': + updateAssetStatus(msg); + break; case 'staticFilteringDataChanged': renderFilterLists(); break; - - case 'forceUpdateAssetsProgress': - renderBusyOverlay(true, msg.progress); - if ( msg.done ) { - messaging.send('dashboard', { what: 'reloadAllFilters' }); - } - break; - default: break; } @@ -69,20 +61,15 @@ var renderNumber = function(value) { /******************************************************************************/ -// TODO: get rid of background page dependencies - var renderFilterLists = function() { - var listGroupTemplate = uDom('#templates .groupEntry'); - var listEntryTemplate = uDom('#templates .listEntry'); - var listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'); - var renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString; - var lastUpdateString = vAPI.i18n('3pLastUpdate'); + var listGroupTemplate = uDom('#templates .groupEntry'), + listEntryTemplate = uDom('#templates .listEntry'), + listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'), + renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString, + lastUpdateString = vAPI.i18n('3pLastUpdate'); - // Assemble a pretty blacklist name if possible + // Assemble a pretty list name if possible var listNameFromListKey = function(listKey) { - if ( listKey === listDetails.userFiltersPath ) { - return userListName; - } var list = listDetails.current[listKey] || listDetails.available[listKey]; var listTitle = list ? list.title : ''; if ( listTitle === '' ) { @@ -91,73 +78,68 @@ var renderFilterLists = function() { return listTitle; }; - var liFromListEntry = function(listKey) { + var liFromListEntry = function(listKey, li) { var entry = listDetails.available[listKey]; - var li = listEntryTemplate.clone(); - + li = li ? li : listEntryTemplate.clone().nodeAt(0); + li.setAttribute('data-listkey', listKey); + var elem = li.querySelector('input[type="checkbox"]'); if ( entry.off !== true ) { - li.descendants('input').attr('checked', ''); + elem.setAttribute('checked', ''); + } else { + elem.removeAttribute('checked'); } - - var elem = li.descendants('a:nth-of-type(1)'); - elem.attr('href', 'asset-viewer.html?url=' + encodeURI(listKey)); - elem.attr('type', 'text/html'); - elem.attr('data-listkey', listKey); - elem.text(listNameFromListKey(listKey) + '\u200E'); - + elem = li.querySelector('a:nth-of-type(1)'); + elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); + elem.setAttribute('type', 'text/html'); + elem.textContent = listNameFromListKey(listKey) + '\u200E'; + elem = li.querySelector('a:nth-of-type(2)'); if ( entry.instructionURL ) { - elem = li.descendants('a:nth-of-type(2)'); - elem.attr('href', entry.instructionURL); - elem.css('display', ''); + elem.setAttribute('href', entry.instructionURL); + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); } - + elem = li.querySelector('a:nth-of-type(3)'); if ( entry.supportName ) { - elem = li.descendants('a:nth-of-type(3)'); - elem.attr('href', entry.supportURL); - elem.text('(' + entry.supportName + ')'); - elem.css('display', ''); + elem.setAttribute('href', entry.supportURL); + elem.textContent = '(' + entry.supportName + ')'; + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); } - - elem = li.descendants('span.counts'); + elem = li.querySelector('span.counts'); var text = listStatsTemplate .replace('{{used}}', renderNumber(!entry.off && !isNaN(+entry.entryUsedCount) ? entry.entryUsedCount : 0)) .replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?'); - elem.text(text); - - // https://github.com/gorhill/uBlock/issues/78 - // Badge for non-secure connection - var remoteURL = listKey; - if ( remoteURL.lastIndexOf('http:', 0) !== 0 ) { - remoteURL = entry.homeURL || ''; - } - if ( remoteURL.lastIndexOf('http:', 0) === 0 ) { - li.descendants('span.status.unsecure').css('display', ''); - } + elem.textContent = text; // https://github.com/chrisaljoudi/uBlock/issues/104 var asset = listDetails.cache[listKey] || {}; + // https://github.com/gorhill/uBlock/issues/78 + // Badge for non-secure connection + var remoteURL = asset.remoteURL; + li.classList.toggle( + 'unsecure', + typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 + ); // Badge for update status - if ( entry.off !== true ) { - if ( asset.repoObsolete ) { - li.descendants('span.status.new').css('display', ''); - needUpdate = true; - } else if ( asset.cacheObsolete ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } else if ( entry.external && !asset.cached ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } - } - - // In cache + li.classList.toggle( + 'obsolete', + entry.off !== true && asset.obsolete === true + ); + // Badge for cache status + li.classList.toggle( + 'cached', + asset.cached === true && asset.writeTime > 0 + ); if ( asset.cached ) { - elem = li.descendants('span.status.purge'); - elem.css('display', ''); - elem.attr('title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.lastModified))); - hasCachedContent = true; + li.querySelector('.status.purge').setAttribute( + 'title', + lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) + ); } + li.classList.remove('discard'); return li; }; @@ -176,27 +158,31 @@ var renderFilterLists = function() { }; var liFromListGroup = function(groupKey, listKeys) { - var liGroup = listGroupTemplate.clone(); - var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); - if ( groupName !== '' ) { - liGroup.descendants('span.geName').text(groupName); - liGroup.descendants('span.geCount').text(listEntryCountFromGroup(listKeys)); + var liGroup = document.querySelector('#lists > .groupEntry[data-groupkey="' + groupKey + '"]'); + if ( liGroup === null ) { + liGroup = listGroupTemplate.clone().nodeAt(0); + var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1)); + if ( groupName !== '' ) { + liGroup.querySelector('.geName').textContent = groupName; + } } - var ulGroup = liGroup.descendants('ul'); - if ( !listKeys ) { - return liGroup; + if ( liGroup.querySelector('.geName:empty') === null ) { + liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys); } + var ulGroup = liGroup.querySelector('.listEntries'); + if ( !listKeys ) { return liGroup; } listKeys.sort(function(a, b) { return (listDetails.available[a].title || '').localeCompare(listDetails.available[b].title || ''); }); for ( var i = 0; i < listKeys.length; i++ ) { - ulGroup.append(liFromListEntry(listKeys[i])); + var liEntry = liFromListEntry(listKeys[i], ulGroup.children[i]); + if ( liEntry.parentElement === null ) { + ulGroup.appendChild(liEntry); + } } return liGroup; }; - // https://www.youtube.com/watch?v=unCVi4hYRlY#t=30m18s - var groupsFromLists = function(lists) { var groups = {}; var listKeys = Object.keys(lists); @@ -219,14 +205,16 @@ var renderFilterLists = function() { listDetails = details; parseCosmeticFilters = details.parseCosmeticFilters; ignoreGenericCosmeticFilters = details.ignoreGenericCosmeticFilters; - needUpdate = false; - hasCachedContent = false; + + // Incremental rendering: this will allow us to easily discard unused + // DOM list entries. + uDom('#lists .listEntries .listEntry').addClass('discard'); // Visually split the filter lists in purpose-based groups - var ulLists = uDom('#lists').empty(), liGroup; - var groups = groupsFromLists(details.available); - var groupKey, i; - var groupKeys = [ + var ulLists = document.querySelector('#lists'), + groups = groupsFromLists(details.available), + liGroup, i, groupKey, + groupKeys = [ 'default', 'ads', 'privacy', @@ -239,31 +227,44 @@ var renderFilterLists = function() { for ( i = 0; i < groupKeys.length; i++ ) { groupKey = groupKeys[i]; liGroup = liFromListGroup(groupKey, groups[groupKey]); - liGroup.toggleClass( + liGroup.setAttribute('data-groupkey', groupKey); + liGroup.classList.toggle( 'collapsed', vAPI.localStorage.getItem('collapseGroup' + (i + 1)) === 'y' ); - ulLists.append(liGroup); + if ( liGroup.parentElement === null ) { + ulLists.appendChild(liGroup); + } delete groups[groupKey]; } // For all groups not covered above (if any left) groupKeys = Object.keys(groups); for ( i = 0; i < groupKeys.length; i++ ) { groupKey = groupKeys[i]; - ulLists.append(liFromListGroup(groupKey, groups[groupKey])); + ulLists.appendChild(liFromListGroup(groupKey, groups[groupKey])); } + uDom('#lists .listEntries .listEntry.discard').remove(); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); + uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); + uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); + uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); uDom('#listsOfBlockedHostsPrompt').text( vAPI.i18n('3pListsOfBlockedHostsPrompt') .replace('{{netFilterCount}}', renderNumber(details.netFilterCount)) .replace('{{cosmeticFilterCount}}', renderNumber(details.cosmeticFilterCount)) ); - uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); - uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); - uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); + + // Compute a hash of the lists currently enabled in memory. + var selectedListsBefore = []; + for ( var key in listDetails.current ) { + if ( listDetails.current[key].off !== true ) { + selectedListsBefore.push(key); + } + } + selectedListsHashBefore = selectedListsBefore.sort().join(); renderWidgets(); - renderBusyOverlay(details.manualUpdate, details.manualUpdateProgress); }; messaging.send('dashboard', { what: 'getLists' }, onListsReceived); @@ -271,33 +272,22 @@ var renderFilterLists = function() { /******************************************************************************/ -// Progress must be normalized to [0, 1], or can be undefined. +// This is to give a visual hint that the selection of blacklists has changed. -var renderBusyOverlay = function(state, progress) { - progress = progress || {}; - var showProgress = typeof progress.value === 'number'; - if ( showProgress ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:first-child').css( - 'width', - (progress.value * 100).toFixed(1) + '%' - ); - var text = progress.text || ''; - if ( text !== '' ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:last-child').text(text); - } - } - uDom('#busyOverlay > div:nth-of-type(2)').css('display', showProgress ? '' : 'none'); - uDom('body').toggleClass('busy', !!state); +var renderWidgets = function() { + uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); + uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); }; /******************************************************************************/ -// This is to give a visual hint that the selection of blacklists has changed. - -var renderWidgets = function() { - uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); - uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged()); - uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent); +var updateAssetStatus = function(details) { + var li = uDom('#lists .listEntry[data-listkey="' + details.key + '"]'); + li.toggleClass('obsolete', !details.cached); + li.toggleClass('cached', details.cached); + li.removeClass('updating'); + renderWidgets(); }; /******************************************************************************/ @@ -307,98 +297,49 @@ var renderWidgets = function() { var listsSelectionChanged = function() { if ( listDetails.parseCosmeticFilters !== parseCosmeticFilters || - listDetails.parseCosmeticFilters && listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters + listDetails.parseCosmeticFilters && + listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters ) { return true; } - - if ( cacheWasPurged ) { - return true; - } - - var availableLists = listDetails.available; - var currentLists = listDetails.current; - var location, availableOff, currentOff; - - // This check existing entries - for ( location in availableLists ) { - if ( availableLists.hasOwnProperty(location) === false ) { - continue; - } - availableOff = availableLists[location].off === true; - currentOff = currentLists[location] === undefined || currentLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } + var selectedListsAfter = [], + listEntries = uDom('#lists .listEntry[data-listkey] > input[type="checkbox"]:checked'); + for ( var i = 0, n = listEntries.length; i < n; i++ ) { + selectedListsAfter.push(listEntries.at(i).ancestors('.listEntry[data-listkey]').attr('data-listkey')); } - // This check removed entries - for ( location in currentLists ) { - if ( currentLists.hasOwnProperty(location) === false ) { - continue; - } - currentOff = currentLists[location].off === true; - availableOff = availableLists[location] === undefined || availableLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } - } - - return false; -}; - -/******************************************************************************/ - -// Return whether content need update. - -var listsContentChanged = function() { - return needUpdate; + return selectedListsHashBefore !== selectedListsAfter.sort().join(); }; /******************************************************************************/ var onListCheckboxChanged = function() { - var href = uDom(this).parent().descendants('a').first().attr('data-listkey'); - if ( typeof href !== 'string' ) { - return; - } - if ( listDetails.available[href] === undefined ) { - return; - } - listDetails.available[href].off = !this.checked; renderWidgets(); }; /******************************************************************************/ var onPurgeClicked = function() { - var button = uDom(this); - var li = button.parent(); - var href = li.descendants('a').first().attr('data-listkey'); - if ( !href ) { - return; - } + var button = uDom(this), + liEntry = button.ancestors('[data-listkey]'), + listKey = liEntry.attr('data-listkey'); + if ( !listKey ) { return; } - messaging.send('dashboard', { what: 'purgeCache', path: href }); - button.remove(); + messaging.send('dashboard', { what: 'purgeCache', assetKey: listKey }); // If the cached version is purged, the installed version must be assumed // to be obsolete. // https://github.com/gorhill/uBlock/issues/1733 // An external filter list must not be marked as obsolete, they will always // be fetched anyways if there is no cached copy. - var entry = listDetails.current && listDetails.current[href]; - if ( entry && entry.off !== true && /^[a-z]+:\/\//.test(href) === false ) { - if ( typeof entry.homeURL !== 'string' || entry.homeURL === '' ) { - li.descendants('span.status.new').css('display', ''); - } else { - li.descendants('span.status.obsolete').css('display', ''); - } - needUpdate = true; + var entry = listDetails.current && listDetails.current[listKey]; + if ( entry && entry.off !== true ) { + liEntry.addClass('obsolete'); + uDom('#buttonUpdate').removeClass('disabled'); } + liEntry.removeClass('cached'); - if ( li.descendants('input').first().prop('checked') ) { - cacheWasPurged = true; + if ( liEntry.descendants('input').first().prop('checked') ) { renderWidgets(); } }; @@ -419,22 +360,21 @@ var selectFilterLists = function(callback) { }); // Filter lists - var switches = []; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + var listKeys = [], + liEntries = uDom('#lists .listEntry'), liEntry, + i = liEntries.length; while ( i-- ) { - li = lis.at(i); - switches.push({ - location: li.descendants('a').attr('data-listkey'), - off: li.descendants('input').prop('checked') === false - }); + liEntry = liEntries.at(i); + if ( liEntry.descendants('input').first().prop('checked') ) { + listKeys.push(liEntry.attr('data-listkey')); + } } messaging.send( 'dashboard', { what: 'selectFilterLists', - switches: switches + keys: listKeys }, callback ); @@ -444,49 +384,34 @@ var selectFilterLists = function(callback) { var buttonApplyHandler = function() { uDom('#buttonApply').removeClass('enabled'); - - renderBusyOverlay(true); - var onSelectionDone = function() { messaging.send('dashboard', { what: 'reloadAllFilters' }); }; - selectFilterLists(onSelectionDone); - - cacheWasPurged = false; }; /******************************************************************************/ var buttonUpdateHandler = function() { - uDom('#buttonUpdate').removeClass('enabled'); - - if ( needUpdate ) { - renderBusyOverlay(true); - - var onSelectionDone = function() { - messaging.send('dashboard', { what: 'forceUpdateAssets' }); - }; - - selectFilterLists(onSelectionDone); - - cacheWasPurged = false; - } + var onSelectionDone = function() { + uDom('#lists .listEntry.obsolete').addClass('updating'); + messaging.send('dashboard', { what: 'forceUpdateAssets' }); + }; + selectFilterLists(onSelectionDone); }; /******************************************************************************/ -var buttonPurgeAllHandler = function() { +var buttonPurgeAllHandler = function(ev) { uDom('#buttonPurgeAll').removeClass('enabled'); - - renderBusyOverlay(true); - - var onCompleted = function() { - cacheWasPurged = true; - renderFilterLists(); - }; - - messaging.send('dashboard', { what: 'purgeAllCaches' }, onCompleted); + messaging.send( + 'dashboard', + { + what: 'purgeAllCaches', + hard: ev.ctrlKey && ev.shiftKey + }, + renderFilterLists + ); }; /******************************************************************************/ @@ -562,7 +487,7 @@ var groupEntryClickHandler = function() { /******************************************************************************/ -var getCloudData = function() { +var toCloudData = function() { var bin = { parseCosmeticFilters: uDom.nodeFromId('parseCosmeticFilters').checked, ignoreGenericCosmeticFilters: uDom.nodeFromId('ignoreGenericCosmeticFilters').checked, @@ -570,24 +495,22 @@ var getCloudData = function() { externalLists: externalLists }; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + var liEntries = uDom('#lists .listEntry'), liEntry; + var i = liEntries.length; while ( i-- ) { - li = lis.at(i); - if ( li.descendants('input').prop('checked') ) { - bin.selectedLists.push(li.descendants('a').attr('data-listkey')); + liEntry = liEntries.at(i); + if ( liEntry.descendants('input').prop('checked') ) { + bin.selectedLists.push(liEntry.attr('data-listkey')); } } return bin; }; -var setCloudData = function(data, append) { - if ( typeof data !== 'object' || data === null ) { - return; - } +var fromCloudData = function(data, append) { + if ( typeof data !== 'object' || data === null ) { return; } - var elem, checked; + var elem, checked, i, n; elem = uDom.nodeFromId('parseCosmeticFilters'); checked = data.parseCosmeticFilters === true || append && elem.checked; @@ -597,30 +520,34 @@ var setCloudData = function(data, append) { checked = data.ignoreGenericCosmeticFilters === true || append && elem.checked; elem.checked = listDetails.ignoreGenericCosmeticFilters = checked; - var lis = uDom('#lists .listEntry'), li, listKey; - var i = lis.length; - while ( i-- ) { - li = lis.at(i); - elem = li.descendants('input'); - listKey = li.descendants('a').attr('data-listkey'); - checked = data.selectedLists.indexOf(listKey) !== -1 || - append && elem.prop('checked'); - elem.prop('checked', checked); - listDetails.available[listKey].off = !checked; + var listKey; + for ( i = 0, n = data.selectedLists.length; i < n; i++ ) { + listKey = data.selectedLists[i]; + if ( listDetails.aliases[listKey] ) { + data.selectedLists[i] = listDetails.aliases[listKey]; + } + } + var selectedSet = new Set(data.selectedLists), + listEntries = uDom('#lists .listEntry'), + listEntry, input; + for ( i = 0, n = listEntries.length; i < n; i++ ) { + listEntry = listEntries.at(i); + listKey = listEntry.attr('data-listkey'); + input = listEntry.descendants('input').first(); + if ( append && input.prop('checked') ) { continue; } + input.prop('checked', selectedSet.has(listKey) ); } elem = uDom.nodeFromId('externalLists'); - if ( !append ) { - elem.value = ''; - } + if ( !append ) { elem.value = ''; } elem.value += data.externalLists || ''; renderWidgets(); externalListsChangeHandler(); }; -self.cloud.onPush = getCloudData; -self.cloud.onPull = setCloudData; +self.cloud.onPush = toCloudData; +self.cloud.onPull = fromCloudData; /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index 88909948b5b81..f4daf8f28f7dc 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,311 +19,54 @@ Home: https://github.com/gorhill/uBlock */ -/* global YaMD5 */ - 'use strict'; -/******************************************************************************* - -File system structure: - assets - ublock - ... - thirdparties - ... - user - filters.txt - ... - -*/ - /******************************************************************************/ -// Low-level asset files manager - µBlock.assets = (function() { /******************************************************************************/ -var oneSecond = 1000; -var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -var oneDay = 24 * oneHour; - -/******************************************************************************/ +var reIsExternalPath = /^(?:[a-z-]+):\/\//, + reIsUserAsset = /^user-/, + errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); -var projectRepositoryRoot = µBlock.projectServerRoot; -var assetsRepositoryRoot = 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/'; -var nullFunc = function() {}; -var reIsExternalPath = /^(file|ftps?|https?|resource):\/\//; -var reIsUserPath = /^assets\/user\//; -var reIsCachePath = /^cache:\/\//; -var lastRepoMetaTimestamp = 0; -var lastRepoMetaIsRemote = false; -var refreshRepoMetaPeriod = 5 * oneHour; -var errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); -var xhrTimeout = vAPI.localStorage.getItem('xhrTimeout') || 30000; -var onAssetRemovedListener = null; - -var exports = { - autoUpdate: true, - autoUpdateDelay: 4 * oneDay, - - // https://github.com/chrisaljoudi/uBlock/issues/426 - remoteFetchBarrier: 0 +var api = { }; /******************************************************************************/ -var AssetEntry = function() { - this.localChecksum = ''; - this.repoChecksum = ''; - this.expireTimestamp = 0; -}; - -var RepoMetadata = function() { - this.entries = {}; - this.waiting = []; -}; - -var repoMetadata = null; - -// We need these to persist beyond repoMetaData -var homeURLs = {}; - -/******************************************************************************/ +var observers = []; -var stringIsNotEmpty = function(s) { - return typeof s === 'string' && s !== ''; +api.addObserver = function(observer) { + if ( observers.indexOf(observer) === -1 ) { + observers.push(observer); + } }; -/******************************************************************************/ - -var cacheIsObsolete = function(t) { - return typeof t !== 'number' || (Date.now() - t) >= exports.autoUpdateDelay; +api.removeObserver = function(observer) { + var pos; + while ( (pos = observers.indexOf(observer)) !== -1 ) { + observers.splice(pos, 1); + } }; -/******************************************************************************/ - -var cachedAssetsManager = (function() { - var exports = {}; - var entries = null; - var cachedAssetPathPrefix = 'cached_asset_content://'; - - var getEntries = function(callback) { - if ( entries !== null ) { - callback(entries); - return; - } - // Flush cached non-user assets if these are from a prior version. - // https://github.com/gorhill/httpswitchboard/issues/212 - var onLastVersionRead = function(store) { - var currentVersion = vAPI.app.version; - var lastVersion = store.extensionLastVersion || '0.0.0.0'; - if ( currentVersion !== lastVersion ) { - vAPI.cacheStorage.set({ 'extensionLastVersion': currentVersion }); - } - callback(entries); - }; - var onLoaded = function(bin) { - // https://github.com/gorhill/httpswitchboard/issues/381 - // Maybe the index was requested multiple times and already - // fetched by one of the occurrences. - if ( entries === null ) { - var lastError = vAPI.lastError(); - if ( lastError ) { - console.error( - 'µBlock> cachedAssetsManager> getEntries():', - lastError.message - ); - } - entries = bin.cached_asset_entries || {}; - } - vAPI.cacheStorage.get('extensionLastVersion', onLastVersionRead); - }; - vAPI.cacheStorage.get('cached_asset_entries', onLoaded); - }; - exports.entries = getEntries; - - exports.load = function(path, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - 'path': path, - 'content': '' - }; - var cachedContentPath = cachedAssetPathPrefix + path; - var onLoaded = function(bin) { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µBlock> cachedAssetsManager.load():', details.error); - cbError(details); - return; - } - // Not sure how this can happen, but I've seen it happen. It could - // be because the save occurred while I was stepping in the code - // though, which means it would not occur during normal operation. - // Still, just to be safe. - if ( stringIsNotEmpty(bin[cachedContentPath]) === false ) { - exports.remove(path); - details.error = 'Error: not found'; - cbError(details); - return; - } - details.content = bin[cachedContentPath]; - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries[path] === undefined ) { - details.error = 'Error: not found'; - cbError(details); - return; - } - vAPI.cacheStorage.get(cachedContentPath, onLoaded); - }; - getEntries(onEntries); - }; - - exports.save = function(path, content, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - path: path, - content: content - }; - if ( content === '' ) { - exports.remove(path); - cbSuccess(details); - return; +var fireNotification = function(topic, details) { + var result; + for ( var i = 0; i < observers.length; i++ ) { + if ( observers[i](topic, details) === false ) { + result = false; } - var cachedContentPath = cachedAssetPathPrefix + path; - var bin = {}; - bin[cachedContentPath] = content; - var removedItems = []; - var onSaved = function() { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µBlock> cachedAssetsManager.save():', details.error); - cbError(details); - return; - } - // Saving over an existing item must be seen as removing an - // existing item and adding a new one. - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener(removedItems); - } - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries.hasOwnProperty(path) ) { - removedItems.push(path); - } - entries[path] = Date.now(); - bin.cached_asset_entries = entries; - vAPI.cacheStorage.set(bin, onSaved); - }; - getEntries(onEntries); - }; - - exports.remove = function(pattern, before) { - var onEntries = function(entries) { - var keystoRemove = []; - var removedItems = []; - var paths = Object.keys(entries); - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( typeof pattern === 'string' && path !== pattern ) { - continue; - } - if ( pattern instanceof RegExp && !pattern.test(path) ) { - continue; - } - if ( typeof before === 'number' && entries[path] >= before ) { - continue; - } - removedItems.push(path); - keystoRemove.push(cachedAssetPathPrefix + path); - delete entries[path]; - } - if ( keystoRemove.length ) { - vAPI.cacheStorage.remove(keystoRemove); - vAPI.cacheStorage.set({ 'cached_asset_entries': entries }); - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener(removedItems); - } - } - }; - getEntries(onEntries); - }; - - exports.removeAll = function(callback) { - var onEntries = function() { - // Careful! do not remove 'assets/user/' - exports.remove(/^https?:\/\/[a-z0-9]+/); - exports.remove(/^assets\/(ublock|thirdparties)\//); - exports.remove(/^cache:\/\//); - exports.remove('assets/checksums.txt'); - if ( typeof callback === 'function' ) { - callback(null); - } - }; - getEntries(onEntries); - }; - - exports.rmrf = function() { - exports.remove(/./); - }; - - exports.exists = function(path) { - return entries !== null && entries.hasOwnProperty(path); - }; - - getEntries(function(){}); - - return exports; -})(); - -/******************************************************************************/ - -var toRepoURL = function(path) { - if ( path.startsWith('assets/ublock/filter-lists.json') ) { - return projectRepositoryRoot + path; - } - - if ( path.startsWith('assets/checksums.txt') ) { - return path.replace( - /^assets\/checksums.txt/, - assetsRepositoryRoot + 'checksums/ublock0.txt' - ); - } - - if ( path.startsWith('assets/thirdparties/') ) { - return path.replace( - /^assets\/thirdparties\//, - assetsRepositoryRoot + 'thirdparties/' - ); - } - - if ( path.startsWith('assets/ublock/') ) { - return path.replace( - /^assets\/ublock\//, - assetsRepositoryRoot + 'filters/' - ); } - - // At this point, `path` is assumed to point to a resource specific to - // this project. - return projectRepositoryRoot + path; + return result; }; /******************************************************************************/ var getTextFileFromURL = function(url, onLoad, onError) { - // console.log('µBlock.assets/getTextFileFromURL("%s"):', url); + if ( reIsExternalPath.test(url) === false ) { + url = vAPI.getURL(url); + } if ( typeof onError !== 'function' ) { onError = onLoad; @@ -353,6 +96,7 @@ var getTextFileFromURL = function(url, onLoad, onError) { var onErrorReceived = function() { this.onload = this.onerror = this.ontimeout = null; + µBlock.logger.writeOne('', 'error', errorCantConnectTo.replace('{{msg}}', '')); onError.call(this); }; @@ -362,7 +106,7 @@ var getTextFileFromURL = function(url, onLoad, onError) { var xhr = new XMLHttpRequest(); try { xhr.open('get', url, true); - xhr.timeout = xhrTimeout; + xhr.timeout = µBlock.hiddenSettings.assetFetchTimeout * 1000 || 30000; xhr.onload = onResponseReceived; xhr.onerror = onErrorReceived; xhr.ontimeout = onErrorReceived; @@ -373,1289 +117,897 @@ var getTextFileFromURL = function(url, onLoad, onError) { } }; -/******************************************************************************/ +/******************************************************************************* -var updateLocalChecksums = function() { - var localChecksums = []; - var entries = repoMetadata.entries; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entry = entries[path]; - if ( entry.localChecksum !== '' ) { - localChecksums.push(entry.localChecksum + ' ' + path); + TODO(seamless migration): + This block of code will be removed when I am confident all users have + moved to a version of uBO which does not require the old way of caching + assets. + + api.listKeyAliases: a map of old asset keys to new asset keys. + + migrate(): to seamlessly migrate the old cache manager to the new one: + - attempt to preserve and move content of cached assets to new locations; + - removes all traces of now obsolete cache manager entries in cacheStorage. + + This code will typically execute only once, when the newer version of uBO + is first installed and executed. + +**/ + +api.listKeyAliases = { + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", + "assets/user/filters.txt": "user-filters", + "assets/ublock/resources.txt": "ublock-resources", + "assets/ublock/filters.txt": "ublock-filters", + "assets/ublock/privacy.txt": "ublock-privacy", + "assets/ublock/unbreak.txt": "ublock-unbreak", + "assets/ublock/badware.txt": "ublock-badware", + "assets/ublock/experimental.txt": "ublock-experimental", + "https://easylist-downloads.adblockplus.org/easylistchina.txt": "CHN-0", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "CHN-1", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt": "CHN-2", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt": "DEU-0", + "https://adblock.dk/block.csv": "DNK-0", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt": "easylist", + "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "easylist-nocosmetic", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt": "easyprivacy", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "fanboy-annoyance", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt": "fanboy-social", + "https://easylist-downloads.adblockplus.org/liste_fr.txt": "FRA-0", + "http://adblock.gardar.net/is.abp.txt": "ISL-0", + "https://easylist-downloads.adblockplus.org/easylistitaly.txt": "ITA-0", + "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "ITA-1", + "https://easylist-downloads.adblockplus.org/advblock.txt": "RUS-0", + "https://easylist-downloads.adblockplus.org/bitblock.txt": "RUS-1", + "https://filters.adtidy.org/extension/chromium/filters/1.txt": "RUS-2", + "https://adguard.com/en/filter-rules.html?id=1": "RUS-2", + "https://easylist-downloads.adblockplus.org/easylistdutch.txt": "NLD-0", + "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt": "LVA-0", + "http://hosts-file.net/.%5Cad_servers.txt": "hphosts", + "http://adblock.ee/list.php": "EST-0", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt": "disconnect-malvertising", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt": "disconnect-malware", + "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt": "disconnect-tracking", + "https://www.certyficate.it/adblock/adblock.txt": "POL-0", + "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt": "awrl-0", + "http://adb.juvander.net/Finland_adb.txt": "FIN-0", + "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt": "KOR-0", + "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt": "KOR-1", + "https://www.fanboy.co.nz/fanboy-korean.txt": "KOR-2", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0", + "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt": "JPN-0", + "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake", + "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0", + "http://margevicius.lt/easylistlithuania.txt": "LTU-0", + "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-0", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-1", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-2", + "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", + "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0", + "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0", + "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "HUN-0", + "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "CZE-0", + "http://someonewhocares.org/hosts/hosts": "dpollock-0", + "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt": "spam404-0", + "http://stanev.org/abp/adblock_bg.txt": "BGR-0", + "http://winhelp2002.mvps.org/hosts.txt": "mvps-0", + "https://www.fanboy.co.nz/enhancedstats.txt": "fanboy-enhanced", + "https://www.fanboy.co.nz/fanboy-antifacebook.txt": "fanboy-thirdparty_social", + "https://easylist-downloads.adblockplus.org/easylistspanish.txt": "spa-0", + "https://www.fanboy.co.nz/fanboy-swedish.txt": "SWE-0", + "https://www.fanboy.co.nz/r/fanboy-ultimate.txt": "fanboy-ultimate", + "https://filters.adtidy.org/extension/chromium/filters/13.txt": "TUR-0", + "https://adguard.com/filter-rules.html?id=13": "TUR-0", + "https://www.fanboy.co.nz/fanboy-vietnam.txt": "VIE-0", + "https://www.void.gr/kargig/void-gr-filters.txt": "GRC-0", + "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt": "SVN-0" +}; + +var migrate = function(callback) { + var entries, + moveCount = 0, + toRemove = []; + + var countdown = function(change) { + moveCount -= (change || 0); + if ( moveCount !== 0 ) { return; } + vAPI.cacheStorage.remove(toRemove); + saveAssetCacheRegistry(); + callback(); + }; + + var onContentRead = function(oldKey, newKey, bin) { + var content = bin && bin['cached_asset_content://' + oldKey] || undefined; + if ( content ) { + assetCacheRegistry[newKey] = { + readTime: Date.now(), + writeTime: entries[oldKey] + }; + if ( reIsExternalPath.test(oldKey) ) { + assetCacheRegistry[newKey].remoteURL = oldKey; + } + bin = {}; + bin['cache/' + newKey] = content; + vAPI.cacheStorage.set(bin); } - } - cachedAssetsManager.save('assets/checksums.txt', localChecksums.join('\n')); -}; + countdown(1); + }; -/******************************************************************************/ + var onEntries = function(bin) { + entries = bin && bin['cached_asset_entries']; + if ( !entries ) { return callback(); } + if ( bin && bin['assetCacheRegistry'] ) { + assetCacheRegistry = bin['assetCacheRegistry']; + } + var aliases = api.listKeyAliases; + for ( var oldKey in entries ) { + if ( oldKey.endsWith('assets/user/filters.txt') ) { continue; } + var newKey = aliases[oldKey]; + if ( !newKey && /^https?:\/\//.test(oldKey) ) { + newKey = oldKey; + } + if ( newKey ) { + vAPI.cacheStorage.get( + 'cached_asset_content://' + oldKey, + onContentRead.bind(null, oldKey, newKey) + ); + moveCount += 1; + } + toRemove.push('cached_asset_content://' + oldKey); + } + toRemove.push('cached_asset_entries', 'extensionLastVersion'); + countdown(); + }; -// Gather meta data of all assets. + vAPI.cacheStorage.get( + [ 'cached_asset_entries', 'assetCacheRegistry' ], + onEntries + ); +}; -var getRepoMetadata = function(callback) { - callback = callback || nullFunc; +/******************************************************************************* - // https://github.com/chrisaljoudi/uBlock/issues/515 - // Handle re-entrancy here, i.e. we MUST NOT tamper with the waiting list - // of callers, if any, except to add one at the end of the list. - if ( repoMetadata !== null && repoMetadata.waiting.length !== 0 ) { - repoMetadata.waiting.push(callback); - return; + The purpose of the asset source registry is to keep key detail information + about an asset: + - Where to load it from: this may consist of one or more URLs, either local + or remote. + - After how many days an asset should be deemed obsolete -- i.e. in need of + an update. + - The origin and type of an asset. + - The last time an asset was registered. + +**/ + +var assetSourceRegistryStatus, + assetSourceRegistry = Object.create(null); + +var registerAssetSource = function(assetKey, dict) { + var entry = assetSourceRegistry[assetKey] || {}; + for ( var prop in dict ) { + if ( dict.hasOwnProperty(prop) === false ) { continue; } + if ( dict[prop] === undefined ) { + delete entry[prop]; + } else { + entry[prop] = dict[prop]; + } } - - if ( exports.remoteFetchBarrier === 0 && lastRepoMetaIsRemote === false ) { - lastRepoMetaTimestamp = 0; + var contentURL = dict.contentURL; + if ( contentURL !== undefined ) { + if ( typeof contentURL === 'string' ) { + contentURL = entry.contentURL = [ contentURL ]; + } else if ( Array.isArray(contentURL) === false ) { + contentURL = entry.contentURL = []; + } + var remoteURLCount = 0; + for ( var i = 0; i < contentURL.length; i++ ) { + if ( reIsExternalPath.test(contentURL[i]) ) { + remoteURLCount += 1; + } + } + entry.hasLocalURL = remoteURLCount !== contentURL.length; + entry.hasRemoteURL = remoteURLCount !== 0; + } else if ( entry.contentURL === undefined ) { + entry.contentURL = []; } - if ( (Date.now() - lastRepoMetaTimestamp) >= refreshRepoMetaPeriod ) { - repoMetadata = null; + if ( typeof entry.updateAfter !== 'number' ) { + entry.updateAfter = 5; } - if ( repoMetadata !== null ) { - callback(repoMetadata); - return; + if ( entry.submitter ) { + entry.submitTime = Date.now(); // To detect stale entries } + assetSourceRegistry[assetKey] = entry; +}; - lastRepoMetaTimestamp = Date.now(); - lastRepoMetaIsRemote = exports.remoteFetchBarrier === 0; - - var defaultChecksums; - var localChecksums; - var repoChecksums; +var unregisterAssetSource = function(assetKey) { + assetCacheRemove(assetKey); + delete assetSourceRegistry[assetKey]; +}; - var checksumsReceived = function() { - if ( - defaultChecksums === undefined || - localChecksums === undefined || - repoChecksums === undefined - ) { - return; - } - // Remove from cache assets which no longer exist in the repo - var entries = repoMetadata.entries; - var checksumsChanged = false; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entry = entries[path]; - // https://github.com/gorhill/uBlock/issues/760 - // If the resource does not have a cached instance, we must reset - // the checksum to its value at install time. - if ( - stringIsNotEmpty(defaultChecksums[path]) && - entry.localChecksum !== defaultChecksums[path] && - cachedAssetsManager.exists(path) === false - ) { - entry.localChecksum = defaultChecksums[path]; - checksumsChanged = true; - } - // If repo checksums could not be fetched, assume no change. - // https://github.com/gorhill/uBlock/issues/602 - // Added: if repo checksum is that of the empty string, - // assume no change - if ( - repoChecksums === '' || - entry.repoChecksum === 'd41d8cd98f00b204e9800998ecf8427e' - ) { - entry.repoChecksum = entry.localChecksum; - } - if ( entry.repoChecksum !== '' || entry.localChecksum === '' ) { - continue; - } - checksumsChanged = true; - cachedAssetsManager.remove(path); - entry.localChecksum = ''; - } - if ( checksumsChanged ) { - updateLocalChecksums(); - } - // Notify all waiting callers - // https://github.com/chrisaljoudi/uBlock/issues/515 - // VERY IMPORTANT: because of re-entrancy, we MUST: - // - process the waiting callers in a FIFO manner - // - not cache repoMetadata.waiting.length, we MUST use the live - // value, because it can change while looping - // - not change the waiting list until they are all processed - for ( var i = 0; i < repoMetadata.waiting.length; i++ ) { - repoMetadata.waiting[i](repoMetadata); - } - repoMetadata.waiting.length = 0; +var saveAssetSourceRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry }); }; - - var validateChecksums = function(details) { - if ( details.error || details.content === '' ) { - return ''; - } - if ( /^(?:[0-9a-f]{32}\s+\S+(?:\s+|$))+/.test(details.content) === false ) { - return ''; + return function(lazily) { + if ( timer !== undefined ) { + clearTimeout(timer); } - // https://github.com/gorhill/uBlock/issues/602 - // External filter lists are not meant to appear in checksums.txt. - // TODO: remove this code once v1.1.0.0 is everywhere. - var out = []; - var listMap = µBlock.oldListToNewListMap; - var lines = details.content.split(/\s*\n\s*/); - var line, matches; - for ( var i = 0; i < lines.length; i++ ) { - line = lines[i]; - matches = line.match(/^[0-9a-f]+ (.+)$/); - if ( matches === null || listMap.hasOwnProperty(matches[1]) ) { - continue; - } - out.push(line); + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); } - return out.join('\n'); }; +})(); - var parseChecksums = function(text, eachFn) { - var lines = text.split(/\n+/); - var i = lines.length; - var fields; - while ( i-- ) { - fields = lines[i].trim().split(/\s+/); - if ( fields.length !== 2 ) { - continue; - } - eachFn(fields[1], fields[0]); - } - }; +var updateAssetSourceRegistry = function(json) { + var newDict; + try { + newDict = JSON.parse(json); + } catch (ex) { + } + if ( newDict instanceof Object === false ) { return; } - var onLocalChecksumsLoaded = function(details) { - var entries = repoMetadata.entries; - var processChecksum = function(path, checksum) { - if ( entries.hasOwnProperty(path) === false ) { - entries[path] = new AssetEntry(); + getAssetSourceRegistry(function(oldDict) { + var assetKey; + // Remove obsolete entries + for ( assetKey in oldDict ) { + if ( newDict[assetKey] === undefined ) { + unregisterAssetSource(assetKey); } - entries[path].localChecksum = checksum; - }; - if ( (localChecksums = validateChecksums(details)) ) { - parseChecksums(localChecksums, processChecksum); } - checksumsReceived(); - }; - - var onRepoChecksumsLoaded = function(details) { - var entries = repoMetadata.entries; - var processChecksum = function(path, checksum) { - if ( entries.hasOwnProperty(path) === false ) { - entries[path] = new AssetEntry(); - } - entries[path].repoChecksum = checksum; - }; - if ( (repoChecksums = validateChecksums(details)) ) { - parseChecksums(repoChecksums, processChecksum); + // Add/update existing entries + for ( assetKey in newDict ) { + registerAssetSource(assetKey, newDict[assetKey]); } - checksumsReceived(); - }; - - // https://github.com/gorhill/uBlock/issues/760 - // We need the checksum values at install time, because some resources - // may have been purged, in which case the checksum must be reset to the - // value at install time. - var onDefaultChecksumsLoaded = function() { - defaultChecksums = Object.create(null); - var processChecksum = function(path, checksum) { - defaultChecksums[path] = checksum; - }; - parseChecksums(this.responseText || '', processChecksum); - checksumsReceived(); - }; - - repoMetadata = new RepoMetadata(); - repoMetadata.waiting.push(callback); - readRepoFile('assets/checksums.txt', onRepoChecksumsLoaded); - getTextFileFromURL(vAPI.getURL('assets/checksums.txt'), onDefaultChecksumsLoaded); - readLocalFile('assets/checksums.txt', onLocalChecksumsLoaded); + saveAssetSourceRegistry(); + }); }; -// https://www.youtube.com/watch?v=-t3WYfgM4x8 - -/******************************************************************************/ - -exports.setHomeURL = function(path, homeURL) { - if ( typeof homeURL !== 'string' || homeURL === '' ) { +var getAssetSourceRegistry = function(callback) { + // Already loaded. + if ( assetSourceRegistryStatus === 'ready' ) { + callback(assetSourceRegistry); return; } - homeURLs[path] = homeURL; -}; -/******************************************************************************/ + // Being loaded. + if ( Array.isArray(assetSourceRegistryStatus) ) { + assetSourceRegistryStatus.push(callback); + return; + } -// Get a local asset, do not look-up repo or remote location if local asset -// is not found. + // Not loaded: load it. + assetSourceRegistryStatus = [ callback ]; -var readLocalFile = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; + var registryReady = function() { + var callers = assetSourceRegistryStatus; + assetSourceRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetSourceRegistry); } - callback(details); - }; - - var onInstallFileLoaded = function() { - //console.log('µBlock> readLocalFile("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µBlock> readLocalFile("%s") / onInstallFileError()', path); - reportBack('', 'Error'); }; - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readLocalFile("%s") / onCachedContentLoaded()', path); - reportBack(details.content); + // First-install case. + var createRegistry = function() { + getTextFileFromURL( + µBlock.assetsBootstrapLocation || 'assets/assets.json', + function() { + updateAssetSourceRegistry(this.responseText); + registryReady(); + } + ); }; - var onCachedContentError = function(details) { - //console.error('µBlock> readLocalFile("%s") / onCachedContentError()', path); - if ( reIsExternalPath.test(path) ) { - reportBack('', 'Error: asset not found'); - return; - } - // It's ok for user data to not be found - if ( reIsUserPath.test(path) ) { - reportBack(''); + vAPI.cacheStorage.get('assetSourceRegistry', function(bin) { + if ( !bin || !bin.assetSourceRegistry ) { + createRegistry(); return; } - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; + assetSourceRegistry = bin.assetSourceRegistry; + registryReady(); + }); +}; - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); +api.registerAssetSource = function(assetKey, details) { + getAssetSourceRegistry(function() { + registerAssetSource(assetKey, details); + saveAssetSourceRegistry(true); + }); }; -// https://www.youtube.com/watch?v=r9KVpuFPtHc +api.unregisterAssetSource = function(assetKey) { + getAssetSourceRegistry(function() { + unregisterAssetSource(assetKey); + saveAssetSourceRegistry(true); + }); +}; -/******************************************************************************/ +/******************************************************************************* + + The purpose of the asset cache registry is to keep track of all assets + which have been persisted into the local cache. -// Get the repository copy of a built-in asset. +**/ -var readRepoFile = function(path, callback) { - // https://github.com/chrisaljoudi/uBlock/issues/426 - if ( exports.remoteFetchBarrier !== 0 ) { - readLocalFile(path, callback); +var assetCacheRegistryStatus, + assetCacheRegistryStartTime = Date.now(), + assetCacheRegistry = {}; + +var getAssetCacheRegistry = function(callback) { + // Already loaded. + if ( assetCacheRegistryStatus === 'ready' ) { + callback(assetCacheRegistry); return; } - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content, - 'error': err - }; - callback(details); - }; + // Being loaded. + if ( Array.isArray(assetCacheRegistryStatus) ) { + assetCacheRegistryStatus.push(callback); + return; + } - var repositoryURL = toRepoURL(path); + // Not loaded: load it. + assetCacheRegistryStatus = [ callback ]; - var onRepoFileLoaded = function() { - //console.log('µBlock> readRepoFile("%s") / onRepoFileLoaded()', path); - // https://github.com/gorhill/httpswitchboard/issues/263 - if ( this.status === 200 ) { - reportBack(this.responseText); - } else { - reportBack('', 'Error: ' + this.statusText); + var registryReady = function() { + var callers = assetCacheRegistryStatus; + assetCacheRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetCacheRegistry); } }; - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - reportBack('', 'Error'); + var migrationDone = function() { + vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { + if ( bin && bin.assetCacheRegistry ) { + assetCacheRegistry = bin.assetCacheRegistry; + } + registryReady(); + }); }; - // '_=...' is to skip browser cache - getTextFileFromURL( - repositoryURL + '?_=' + Date.now(), - onRepoFileLoaded, - onRepoFileError - ); + migrate(migrationDone); }; -/******************************************************************************/ - -// An asset from an external source with a copy shipped with the extension: -// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL -// External --> -// Repository --> has checksum (to detect need for update only) -// Cache --> has expiration timestamp (in cache) -// Local --> install time version +var saveAssetCacheRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry }); + }; + return function(lazily) { + if ( timer !== undefined ) { clearTimeout(timer); } + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; +})(); -var readRepoCopyAsset = function(path, callback) { - var assetEntry; - var homeURL = homeURLs[path]; +var assetCacheRead = function(assetKey, callback) { + var internalKey = 'cache/' + assetKey; var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } + var details = { assetKey: assetKey, content: content }; + if ( err ) { details.error = err; } callback(details); }; - var updateChecksum = function() { - if ( assetEntry !== undefined && assetEntry.repoChecksum !== assetEntry.localChecksum ) { - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); + var onAssetRead = function(bin) { + if ( !bin || !bin[internalKey] ) { + return reportBack('', 'E_NOTFOUND'); } + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + return reportBack('', 'E_NOTFOUND'); + } + entry.readTime = Date.now(); + saveAssetCacheRegistry(true); + reportBack(bin[internalKey]); }; - var onInstallFileLoaded = function() { - //console.log('µBlock> readRepoCopyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µBlock> readRepoCopyAsset("%s") / onInstallFileError():', path, this.statusText); - reportBack('', 'Error'); + var onReady = function() { + vAPI.cacheStorage.get(internalKey, onAssetRead); }; - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileLoaded()', path); - reportBack(details.content); - }; + getAssetCacheRegistry(onReady); +}; - var onCachedContentError = function(details) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileError()', path); - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; +var assetCacheWrite = function(assetKey, details, callback) { + var internalKey = 'cache/' + assetKey; + var content = ''; + if ( typeof details === 'string' ) { + content = details; + } else if ( details instanceof Object ) { + content = details.content || ''; + } - var repositoryURL = toRepoURL(path); - var repositoryURLSkipCache = repositoryURL + '?_=' + Date.now(); + if ( content === '' ) { + return assetCacheRemove(assetKey, callback); + } - var onRepoFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s"): error', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; + var reportBack = function(content) { + var details = { assetKey: assetKey, content: content }; + if ( typeof callback === 'function' ) { + callback(details); } - //console.log('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); + fireNotification('after-asset-updated', details); }; - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + var onReady = function() { + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + entry = assetCacheRegistry[assetKey] = {}; + } + entry.writeTime = entry.readTime = Date.now(); + if ( details instanceof Object && typeof details.url === 'string' ) { + entry.remoteURL = details.url; + } + var bin = { assetCacheRegistry: assetCacheRegistry }; + bin[internalKey] = content; + vAPI.cacheStorage.set(bin); + reportBack(content); }; + getAssetCacheRegistry(onReady); +}; - var onHomeFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s"): no response', path, homeURL); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); +var assetCacheRemove = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + removedEntries = [], + removedContent = []; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; } - return; + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + removedEntries.push(assetKey); + removedContent.push('cache/' + assetKey); + delete cacheDict[assetKey]; } - //console.log('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s")', path, homeURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onHomeFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', homeURL)); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + if ( removedContent.length !== 0 ) { + vAPI.cacheStorage.remove(removedContent); + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); + } + for ( var i = 0; i < removedEntries.length; i++ ) { + fireNotification('after-asset-updated', { assetKey: removedEntries[i] }); } }; - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Auto-update enabled AND (not in cache OR in cache but obsolete) - var timestamp = entries[path]; - var inCache = typeof timestamp === 'number'; - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && stringIsNotEmpty(homeURL) - ) { - if ( inCache === false || cacheIsObsolete(timestamp) ) { - //console.log('µBlock> readRepoCopyAsset("%s") / onCacheMetaReady(): not cached or obsolete', path); - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - return; + getAssetCacheRegistry(onReady); +}; + +var assetCacheMarkAsDirty = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + cacheEntry, + mustSave = false; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { + continue; } + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + cacheEntry = cacheDict[assetKey]; + if ( !cacheEntry.writeTime ) { continue; } + cacheDict[assetKey].writeTime = 0; + mustSave = true; } - - // In cache - if ( inCache ) { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; + if ( mustSave ) { + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); } - - // Not in cache - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); }; - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; + getAssetCacheRegistry(onReady); +}; - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } +/******************************************************************************/ - // Repo copy changed: fetch from home URL - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µBlock> readRepoCopyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - if ( stringIsNotEmpty(homeURL) ) { - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - } else { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } - return; - } +var stringIsNotEmpty = function(s) { + return typeof s === 'string' && s !== ''; +}; + +/******************************************************************************* + + User assets are NOT persisted in the cache storage. User assets are + recognized by the asset key which always starts with 'user-'. + + TODO(seamless migration): + Can remove instances of old user asset keys when I am confident all users + are using uBO v1.11 and beyond. + +**/ - // Load from cache - cachedAssetsManager.entries(onCacheMetaReady); +var readUserAsset = function(assetKey, callback) { + var reportBack = function(content) { + callback({ assetKey: assetKey, content: content }); }; - getRepoMetadata(onRepoMetaReady); + var onLoaded = function(bin) { + if ( !bin ) { return reportBack(''); } + var content = ''; + if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) { + content = bin['cached_asset_content://assets/user/filters.txt']; + vAPI.cacheStorage.remove('cached_asset_content://assets/user/filters.txt'); + } + if ( typeof bin['assets/user/filters.txt'] === 'string' ) { + content = bin['assets/user/filters.txt']; + // TODO(seamless migration): + // Uncomment once all moved to v1.11+. + //vAPI.storage.remove('assets/user/filters.txt'); + } + if ( typeof bin[assetKey] === 'string' ) { + // TODO(seamless migration): + // Replace conditional with assignment once all moved to v1.11+ + if ( content !== bin[assetKey] ) { + saveUserAsset(assetKey, content); + } + } else if ( content !== '' ) { + saveUserAsset(assetKey, content); + } + return reportBack(content); + }; + var toRead = assetKey; + if ( assetKey === µBlock.userFiltersPath ) { + toRead = [ + assetKey, + 'assets/user/filters.txt', + 'cached_asset_content://assets/user/filters.txt' + ]; + } + vAPI.storage.get(toRead, onLoaded); }; -// https://www.youtube.com/watch?v=uvUW4ozs7pY +var saveUserAsset = function(assetKey, content, callback) { + var bin = {}; + bin[assetKey] = content; + // TODO(seamless migration): + // This is for forward compatibility. Only for a limited time. Remove when + // everybody moved to 1.11.0 and beyond. + // >>>>>>>> + if ( assetKey === µBlock.userFiltersPath ) { + bin['assets/user/filters.txt'] = content; + } + // <<<<<<<< + var onSaved = function() { + if ( callback instanceof Function ) { + callback({ assetKey: assetKey, content: content }); + } + }; + vAPI.storage.set(bin, onSaved); +}; /******************************************************************************/ -// An important asset shipped with the extension -- typically small, or -// doesn't change often: -// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL -// Repository --> has checksum (to detect need for update and corruption) -// Cache --> whatever from above -// Local --> install time version - -var readRepoOnlyAsset = function(path, callback) { +api.get = function(assetKey, callback) { + if ( assetKey === µBlock.userFiltersPath ) { + readUserAsset(assetKey, callback); + return; + } - var assetEntry; + var assetDetails = {}, + contentURLs, + contentURL; var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; + var details = { assetKey: assetKey, content: content }; if ( err ) { - details.error = err; + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; } callback(details); }; - var onInstallFileLoaded = function() { - //console.log('µBlock> readRepoOnlyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); + var onContentNotLoaded = function() { + var isExternal; + while ( (contentURL = contentURLs.shift()) ) { + isExternal = reIsExternalPath.test(contentURL); + if ( isExternal === false || assetDetails.hasLocalURL !== true ) { + break; + } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + getTextFileFromURL(contentURL, onContentLoaded, onContentNotLoaded); }; - var onInstallFileError = function() { - console.error('µBlock> readRepoOnlyAsset("%s") / onInstallFileError()', path); - reportBack('', 'Error'); + var onContentLoaded = function() { + if ( stringIsNotEmpty(this.responseText) === false ) { + onContentNotLoaded(); + return; + } + if ( reIsExternalPath.test(contentURL) ) { + assetCacheWrite(assetKey, { + content: this.responseText, + url: contentURL + }); + } + reportBack(this.responseText); }; var onCachedContentLoaded = function(details) { - //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentError()', path); - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); - }; - - var repositoryURL = toRepoURL(path + '?_=' + Date.now()); - - var onRepoFileLoaded = function() { - if ( typeof this.responseText !== 'string' ) { - console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): no response', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - if ( YaMD5.hashStr(this.responseText) !== assetEntry.repoChecksum ) { - console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): bad md5 checksum', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } - - // Asset added or changed: load from repo URL and then cache result - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - getTextFileFromURL(repositoryURL, onRepoFileLoaded, onRepoFileError); - return; - } - - // Load from cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + if ( details.content !== '' ) { + return reportBack(details.content); + } + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + onContentNotLoaded(); + }); }; - getRepoMetadata(onRepoMetaReady); -}; - -/******************************************************************************/ - -// Asset doesn't exist. Just for symmetry purpose. - -var readNilAsset = function(path, callback) { - callback({ - 'path': path, - 'content': '', - 'error': 'Error: asset not found' - }); + assetCacheRead(assetKey, onCachedContentLoaded); }; /******************************************************************************/ -// An external asset: -// Path --> starts with 'http' -// External --> https://..., http://... -// Cache --> has expiration timestamp (in cache) +var getRemote = function(assetKey, callback) { + var assetDetails = {}, + contentURLs, + contentURL; -var readExternalAsset = function(path, callback) { var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; + var details = { assetKey: assetKey, content: content }; if ( err ) { - details.error = err; + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; } callback(details); }; - var onCachedContentLoaded = function(details) { - //console.log('µBlock> readExternalAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - console.error('µBlock> readExternalAsset("%s") / onCachedContentError()', path); - reportBack('', 'Error'); - }; - - var onExternalFileLoaded = function() { - // https://github.com/chrisaljoudi/uBlock/issues/708 - // A successful download should never return an empty file: turn this - // into an error condition. + var onRemoteContentLoaded = function() { if ( stringIsNotEmpty(this.responseText) === false ) { - onExternalFileError(); + registerAssetSource(assetKey, { error: { time: Date.now(), error: 'No content' } }); + tryLoading(); return; } - //console.log('µBlock> readExternalAsset("%s") / onExternalFileLoaded1()', path); - cachedAssetsManager.save(path, this.responseText); + assetCacheWrite(assetKey, { + content: this.responseText, + url: contentURL + }); + registerAssetSource(assetKey, { error: undefined }); reportBack(this.responseText); }; - var onExternalFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', path)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + var onRemoteContentError = function() { + registerAssetSource(assetKey, { error: { time: Date.now(), error: this.statusText } }); + tryLoading(); }; - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Not in cache OR - // - // - Auto-update enabled AND in cache but obsolete - var timestamp = entries[path]; - var notInCache = typeof timestamp !== 'number'; - var updateCache = exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - cacheIsObsolete(timestamp); - if ( notInCache || updateCache ) { - getTextFileFromURL(path, onExternalFileLoaded, onExternalFileError); - return; + var tryLoading = function() { + while ( (contentURL = contentURLs.shift()) ) { + if ( reIsExternalPath.test(contentURL) ) { break; } } - - // In cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - cachedAssetsManager.entries(onCacheMetaReady); -}; - -/******************************************************************************/ - -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved - -var readUserAsset = function(path, callback) { - // TODO: remove when confident all users no longer have their custom - // filters saved into vAPI.cacheStorage. - var onCachedContentLoaded = function(details) { - saveUserAsset(path, details.content); - //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; - - var onCachedContentError = function() { - saveUserAsset(path, ''); - //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); - }; - - var onLoaded = function(bin) { - var content = bin && bin[path]; - if ( typeof content === 'string' ) { - callback({ 'path': path, 'content': content }); - return; - } - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - vAPI.storage.get(path, onLoaded); -}; - -var saveUserAsset = function(path, content, callback) { - var bin = {}; - bin[path] = content; - var onSaved = function() { - // Saving over an existing asset must be seen as removing an - // existing asset and adding a new one. - if ( onAssetRemovedListener instanceof Function ) { - onAssetRemovedListener([ path ]); - } - if ( callback instanceof Function ) { - callback({ path: path, content: content }); + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); } + getTextFileFromURL(contentURL, onRemoteContentLoaded, onRemoteContentError); }; - vAPI.storage.set(bin, onSaved); -}; -/******************************************************************************/ - -// Asset available only from the cache. -// Cache data: -// Path --> starts with 'cache://' -// Cache --> whatever - -var readCacheAsset = function(path, callback) { - var onCachedContentLoaded = function(details) { - //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; - - var onCachedContentError = function() { - //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); - }; - - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); -}; - -/******************************************************************************/ - -// Assets -// -// A copy of an asset from an external source shipped with the extension: -// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL -// External --> -// Repository --> has checksum (to detect obsolescence) -// Cache --> has expiration timestamp (to detect obsolescence) -// Local --> install time version -// -// An important asset shipped with the extension (usually small, or doesn't -// change often): -// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL -// Repository --> has checksum (to detect obsolescence or data corruption) -// Cache --> whatever from above -// Local --> install time version -// -// An external filter list: -// Path --> starts with 'http' -// External --> -// Cache --> has expiration timestamp (to detect obsolescence) -// -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved -// -// When a checksum is present, it is used to determine whether the asset -// needs to be updated. -// When an expiration timestamp is present, it is used to determine whether -// the asset needs to be updated. -// -// If no update required, an asset if first fetched from the cache. If the -// asset is not cached it is fetched from the closest location: local for -// an asset shipped with the extension, external for an asset not shipped -// with the extension. - -exports.get = function(path, callback) { - - if ( reIsUserPath.test(path) ) { - readUserAsset(path, callback); - return; - } - - if ( reIsCachePath.test(path) ) { - readCacheAsset(path, callback); - return; - } - - if ( reIsExternalPath.test(path) ) { - readExternalAsset(path, callback); - return; - } - - var onRepoMetaReady = function(meta) { - var assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - readNilAsset(path, callback); - return; - } - - // Asset is repo copy of external content - if ( stringIsNotEmpty(homeURLs[path]) ) { - readRepoCopyAsset(path, callback); - return; + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; } - - // Asset is repo only - readRepoOnlyAsset(path, callback); - }; - - getRepoMetadata(onRepoMetaReady); + tryLoading(); + }); }; -// https://www.youtube.com/watch?v=98y0Q7nLGWk - -/******************************************************************************/ - -exports.getLocal = readLocalFile; - /******************************************************************************/ -exports.put = function(path, content, callback) { - if ( reIsUserPath.test(path) ) { - saveUserAsset(path, content, callback); - return; +api.put = function(assetKey, content, callback) { + if ( reIsUserAsset.test(assetKey) ) { + return saveUserAsset(assetKey, content, callback); } - - cachedAssetsManager.save(path, content, callback); -}; - -/******************************************************************************/ - -exports.rmrf = function() { - cachedAssetsManager.rmrf(); -}; - -/******************************************************************************/ - -exports.rename = function(from, to, callback) { - var done = function() { - if ( typeof callback === 'function' ) { - callback(); - } - }; - - var fromLoaded = function(details) { - cachedAssetsManager.remove(from); - cachedAssetsManager.save(to, details.content, callback); - done(); - }; - - var toLoaded = function(details) { - // `to` already exists: do nothing - if ( details.content !== '' ) { - return done(); - } - cachedAssetsManager.load(from, fromLoaded); - }; - - // If `to` content already exists, do nothing. - cachedAssetsManager.load(to, toLoaded); + assetCacheWrite(assetKey, content, callback); }; /******************************************************************************/ -exports.metadata = function(callback) { - var out = {}; - - // https://github.com/chrisaljoudi/uBlock/issues/186 - // We need to check cache obsolescence when both cache and repo meta data - // has been gathered. - var checkCacheObsolescence = function() { - var entry, homeURL; - for ( var path in out ) { - if ( out.hasOwnProperty(path) === false ) { - continue; - } - entry = out[path]; - // https://github.com/gorhill/uBlock/issues/528 - // Not having a homeURL property does not mean the filter list - // is not external. - homeURL = reIsExternalPath.test(path) ? path : homeURLs[path]; - entry.cacheObsolete = stringIsNotEmpty(homeURL) && - cacheIsObsolete(entry.lastModified); - } - callback(out); - }; +api.metadata = function(callback) { + var assetRegistryReady = false, + cacheRegistryReady = false; - var onRepoMetaReady = function(meta) { - var entries = meta.entries; - var entryRepo, entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryRepo = entries[path]; - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; + var onReady = function() { + var assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)), + cacheDict = assetCacheRegistry, + assetEntry, cacheEntry, + now = Date.now(), obsoleteAfter; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry ) { + assetEntry.cached = true; + assetEntry.writeTime = cacheEntry.writeTime; + obsoleteAfter = cacheEntry.writeTime + assetEntry.updateAfter * 86400000; + assetEntry.obsolete = obsoleteAfter < now; + assetEntry.remoteURL = cacheEntry.remoteURL; + } else { + assetEntry.writeTime = 0; + obsoleteAfter = 0; + assetEntry.obsolete = true; } - entryOut.localChecksum = entryRepo.localChecksum; - entryOut.repoChecksum = entryRepo.repoChecksum; - entryOut.homeURL = homeURLs[path] || ''; - entryOut.supportURL = entryRepo.supportURL || ''; - entryOut.repoObsolete = entryOut.localChecksum !== entryOut.repoChecksum; } - checkCacheObsolescence(); + callback(assetDict); }; - var onCacheMetaReady = function(entries) { - var entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; - } - entryOut.lastModified = entries[path]; - // User data is not literally cache data - if ( reIsUserPath.test(path) ) { - continue; - } - entryOut.cached = true; - if ( reIsExternalPath.test(path) ) { - entryOut.homeURL = path; - } - } - getRepoMetadata(onRepoMetaReady); - }; + getAssetSourceRegistry(function() { + assetRegistryReady = true; + if ( cacheRegistryReady ) { onReady(); } + }); - cachedAssetsManager.entries(onCacheMetaReady); + getAssetCacheRegistry(function() { + cacheRegistryReady = assetCacheRegistry; + if ( assetRegistryReady ) { onReady(); } + }); }; /******************************************************************************/ -exports.purge = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); +api.purge = function(pattern, callback) { + assetCacheMarkAsDirty(pattern, callback); }; -exports.purgeCacheableAsset = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); - lastRepoMetaTimestamp = 0; +api.remove = function(pattern, callback) { + assetCacheRemove(pattern, callback); }; -exports.purgeAll = function(callback) { - cachedAssetsManager.removeAll(callback); - lastRepoMetaTimestamp = 0; +api.rmrf = function() { + assetCacheRemove(/./); }; /******************************************************************************/ -exports.onAssetRemoved = { - addListener: function(callback) { - onAssetRemovedListener = callback instanceof Function ? callback : null; - } -}; +// Asset updater area. +var updaterStatus, + updaterTimer, + updaterAssetDelayDefault = 120000, + updaterAssetDelay = updaterAssetDelayDefault, + updaterUpdated = [], + updaterFetched = new Set(); -/******************************************************************************/ - -return exports; - -})(); - -/******************************************************************************/ -/******************************************************************************/ - -µBlock.assetUpdater = (function() { - -/******************************************************************************/ - -var µb = µBlock; - -var updateDaemonTimer = null; -var autoUpdateDaemonTimerPeriod = 11 * 60 * 1000; // 11 minutes -var manualUpdateDaemonTimerPeriod = 5 * 1000; // 5 seconds - -var updateCycleFirstPeriod = 7 * 60 * 1000; // 7 minutes -var updateCycleNextPeriod = 11 * 60 * 60 * 1000; // 11 hours -var updateCycleTime = 0; - -var toUpdate = {}; -var toUpdateCount = 0; -var updated = {}; -var updatedCount = 0; -var metadata = null; - -var onStartListener = null; -var onCompletedListener = null; -var onAssetUpdatedListener = null; - -var exports = { - manualUpdate: false, - manualUpdateProgress: { - value: 0, - text: null - } -}; - -/******************************************************************************/ - -var onOneUpdated = function(details) { - // Resource fetched, we can safely restart the daemon. - scheduleUpdateDaemon(); - - var path = details.path; - if ( details.error ) { - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount)); - //console.debug('µBlock.assetUpdater/onOneUpdated: "%s" failed', path); - return; - } - - //console.debug('µBlock.assetUpdater/onOneUpdated: "%s"', path); - updated[path] = true; - updatedCount += 1; - - if ( typeof onAssetUpdatedListener === 'function' ) { - onAssetUpdatedListener(details); - } - - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount + 1)); +var updateFirst = function() { + updaterStatus = 'updating'; + updaterFetched.clear(); + updaterUpdated = []; + fireNotification('before-assets-updated'); + updateNext(); }; -/******************************************************************************/ - -var updateOne = function() { - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var metaEntry; - var updatingCount = 0; - var updatingText = null; +var updateNext = function() { + var assetDict, cacheDict; - for ( var path in toUpdate ) { - if ( toUpdate.hasOwnProperty(path) === false ) { - continue; + // This will remove a cached asset when it's no longer in use. + var garbageCollectOne = function(assetKey) { + var cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { + assetCacheRemove(assetKey); } - if ( toUpdate[path] !== true ) { - continue; - } - toUpdate[path] = false; - toUpdateCount -= 1; - if ( metadata.hasOwnProperty(path) === false ) { - continue; - } - metaEntry = metadata[path]; - if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) { - continue; - } - - // Will restart the update daemon once the resource is received: the - // fetching of a resource may take some time, possibly beyond the - // next scheduled daemon cycle, so this ensure the daemon won't do - // anything else before the resource is fetched (or times out). - suspendUpdateDaemon(); - - //console.debug('µBlock.assetUpdater/updateOne: assets.get("%s")', path); - µb.assets.get(path, onOneUpdated); - updatingCount = 1; - updatingText = metaEntry.homeURL || path; - break; - } - - manualUpdateNotify( - false, - (updatedCount + updatingCount/2) / (updatedCount + toUpdateCount + updatingCount + 1), - updatingText - ); -}; - -/******************************************************************************/ - -// Update one asset, fetch metadata if not done yet. - -var safeUpdateOne = function() { - if ( metadata !== null ) { - updateOne(); - return; - } - - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var onMetadataReady = function(response) { - scheduleUpdateDaemon(); - metadata = response; - updateOne(); }; - suspendUpdateDaemon(); - µb.assets.metadata(onMetadataReady); -}; - -/******************************************************************************/ - -var safeStartListener = function(callback) { - // Because this can be called from outside the daemon's main loop - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - var onStartListenerDone = function(assets) { - scheduleUpdateDaemon(); - assets = assets || {}; - for ( var path in assets ) { - if ( assets.hasOwnProperty(path) === false ) { + var findOne = function() { + var now = Date.now(), + assetEntry, cacheEntry; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + if ( assetEntry.hasRemoteURL !== true ) { continue; } + if ( updaterFetched.has(assetKey) ) { continue; } + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { continue; } - if ( toUpdate.hasOwnProperty(path) ) { - continue; + if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { + return assetKey; } - //console.debug('assets.js > µBlock.assetUpdater/safeStartListener: "%s"', path); - toUpdate[path] = true; - toUpdateCount += 1; - } - if ( typeof callback === 'function' ) { - callback(); + garbageCollectOne(assetKey); } }; - if ( typeof onStartListener === 'function' ) { - suspendUpdateDaemon(); - onStartListener(onStartListenerDone); - } else { - onStartListenerDone(null); - } -}; - -/******************************************************************************/ - -var updateDaemon = function() { - updateDaemonTimer = null; - scheduleUpdateDaemon(); - - µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate; - - if ( µb.assets.autoUpdate !== true ) { - return; - } - - // Start an update cycle? - if ( updateCycleTime !== 0 ) { - if ( Date.now() >= updateCycleTime ) { - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle started'); - reset(); - safeStartListener(); + var updatedOne = function(details) { + if ( details.content !== '' ) { + updaterUpdated.push(details.assetKey); + if ( details.assetKey === 'assets.json' ) { + updateAssetSourceRegistry(details.content); + } } - return; - } - - // Any asset to update? - if ( toUpdateCount !== 0 ) { - safeUpdateOne(); - return; - } - // Nothing left to update - - // In case of manual update, fire progress notifications - manualUpdateNotify(true, 1, ''); - - // If anything was updated, notify listener - if ( updatedCount !== 0 ) { - if ( typeof onCompletedListener === 'function' ) { - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle completed'); - onCompletedListener({ - updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy - updatedCount: updatedCount - }); + if ( findOne() !== undefined ) { + vAPI.setTimeout(updateNext, updaterAssetDelay); + } else { + updateDone(); } - } - - // Schedule next update cycle - if ( updateCycleTime === 0 ) { - reset(); - //console.debug('µBlock.assetUpdater/updateDaemon: update cycle re-scheduled'); - updateCycleTime = Date.now() + updateCycleNextPeriod; - } -}; - -/******************************************************************************/ - -var scheduleUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - } - updateDaemonTimer = vAPI.setTimeout( - updateDaemon, - exports.manualUpdate ? manualUpdateDaemonTimerPeriod : autoUpdateDaemonTimerPeriod - ); -}; - -var suspendUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - updateDaemonTimer = null; - } -}; - -scheduleUpdateDaemon(); - -/******************************************************************************/ - -var reset = function() { - toUpdate = {}; - toUpdateCount = 0; - updated = {}; - updatedCount = 0; - updateCycleTime = 0; - metadata = null; -}; - -/******************************************************************************/ - -var manualUpdateNotify = function(done, value, text) { - if ( exports.manualUpdate === false ) { - return; - } + }; - exports.manualUpdate = !done; - exports.manualUpdateProgress.value = value || 0; - if ( typeof text === 'string' ) { - exports.manualUpdateProgress.text = text; - } + var updateOne = function() { + var assetKey = findOne(); + if ( assetKey === undefined ) { + return updateDone(); + } + updaterFetched.add(assetKey); + getRemote(assetKey, updatedOne); + }; - vAPI.messaging.broadcast({ - what: 'forceUpdateAssetsProgress', - done: !exports.manualUpdate, - progress: exports.manualUpdateProgress, - updatedCount: updatedCount + getAssetSourceRegistry(function(dict) { + assetDict = dict; + if ( !cacheDict ) { return; } + updateOne(); }); - // When manually updating, whatever launched the manual update is - // responsible to launch a reload of the filter lists. - if ( exports.manualUpdate !== true ) { - reset(); - } + getAssetCacheRegistry(function(dict) { + cacheDict = dict; + if ( !assetDict ) { return; } + updateOne(); + }); }; -/******************************************************************************/ - -// Manual update: just a matter of forcing the update daemon to work on a -// tighter schedule. - -exports.force = function() { - if ( exports.manualUpdate ) { - return; - } - - reset(); - - exports.manualUpdate = true; - - var onStartListenerDone = function() { - if ( toUpdateCount === 0 ) { - updateCycleTime = Date.now() + updateCycleNextPeriod; - manualUpdateNotify(true, 1); - } else { - manualUpdateNotify(false, 0); - safeUpdateOne(); - } - }; - - safeStartListener(onStartListenerDone); +var updateDone = function() { + var assetKeys = updaterUpdated.slice(0); + updaterFetched.clear(); + updaterUpdated = []; + updaterStatus = undefined; + updaterAssetDelay = updaterAssetDelayDefault; + fireNotification('after-assets-updated', { assetKeys: assetKeys }); }; -/******************************************************************************/ - -exports.onStart = { - addEventListener: function(callback) { - onStartListener = callback || null; - if ( typeof onStartListener === 'function' ) { - updateCycleTime = Date.now() + updateCycleFirstPeriod; +api.updateStart = function(details) { + var oldUpdateDelay = updaterAssetDelay, + newUpdateDelay = details.delay || updaterAssetDelayDefault; + updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); + if ( updaterStatus !== undefined ) { + if ( newUpdateDelay < oldUpdateDelay ) { + clearTimeout(updaterTimer); + updaterTimer = vAPI.setTimeout(updateNext, updaterAssetDelay); } + return; } + updateFirst(); }; -/******************************************************************************/ - -exports.onAssetUpdated = { - addEventListener: function(callback) { - onAssetUpdatedListener = callback || null; +api.updateStop = function() { + if ( updaterTimer ) { + clearTimeout(updaterTimer); + updaterTimer = undefined; } -}; - -/******************************************************************************/ - -exports.onCompleted = { - addEventListener: function(callback) { - onCompletedListener = callback || null; + if ( updaterStatus !== undefined ) { + updateDone(); } }; /******************************************************************************/ -// Typically called when an update has been forced. - -exports.restart = function() { - reset(); - updateCycleTime = Date.now() + updateCycleNextPeriod; -}; - -/******************************************************************************/ - -// Call when disabling uBlock, to ensure it doesn't stick around as a detached -// window object in Firefox. - -exports.shutdown = function() { - suspendUpdateDaemon(); - reset(); -}; +return api; /******************************************************************************/ -return exports; - })(); /******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js index 00b82a1c64997..791e56be94d6f 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,20 +19,16 @@ Home: https://github.com/gorhill/uBlock */ -/* exported µBlock */ - 'use strict'; /******************************************************************************/ -var µBlock = (function() { +var µBlock = (function() { // jshint ignore:line /******************************************************************************/ var oneSecond = 1000; var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -// var oneDay = 24 * oneHour; /******************************************************************************/ @@ -71,8 +67,12 @@ return { }, hiddenSettingsDefault: { + assetFetchTimeout: 30, + autoUpdateAssetFetchPeriod: 120, + autoUpdatePeriod: 7, ignoreRedirectFilters: false, ignoreScriptInjectFilters: false, + manualUpdateAssetFetchPeriod: 2000, popupFontSize: 'unset', suspendTabsUntilReady: false }, @@ -119,92 +119,15 @@ return { lastBackupTime: 0 }, - // EasyList, EasyPrivacy and many others have an 4-day update period, - // as per list headers. - updateAssetsEvery: 97 * oneHour, - projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/', - userFiltersPath: 'assets/user/filters.txt', - pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', - - // permanent lists - permanentLists: { - // User - 'assets/user/filters.txt': { - group: 'default' - }, - // uBlock - 'assets/ublock/filters.txt': { - title: 'uBlock filters', - group: 'default' - }, - 'assets/ublock/privacy.txt': { - title: 'uBlock filters – Privacy', - group: 'default' - }, - 'assets/ublock/unbreak.txt': { - title: 'uBlock filters – Unbreak', - group: 'default' - }, - 'assets/ublock/badware.txt': { - title: 'uBlock filters – Badware risks', - group: 'default', - supportURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks', - instructionURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks' - }, - 'assets/ublock/experimental.txt': { - title: 'uBlock filters – Experimental', - group: 'default', - off: true, - supportURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters', - instructionURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters' - } - }, + // Allows to fully customize uBO's assets, typically set through admin + // settings. The content of 'assets.json' will also tell which filter + // lists to enable by default when uBO is first installed. + assetsBootstrapLocation: 'assets/assets.json', - // current lists - remoteBlacklists: {}, - oldListToNewListMap: { - "assets/thirdparties/adblock.gardar.net/is.abp.txt": "http://adblock.gardar.net/is.abp.txt", - "assets/thirdparties/adblock.schack.dk/block.txt": "https://adblock.dk/block.csv", - "https://adblock.schack.dk/block.txt": "https://adblock.dk/block.csv", - "assets/thirdparties/dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/advblock.txt": "https://easylist-downloads.adblockplus.org/advblock.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/bitblock.txt": "https://easylist-downloads.adblockplus.org/bitblock.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistchina.txt": "https://easylist-downloads.adblockplus.org/easylistchina.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistdutch.txt": "https://easylist-downloads.adblockplus.org/easylistdutch.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistgermany.txt": "https://easylist-downloads.adblockplus.org/easylistgermany.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylistitaly.txt": "https://easylist-downloads.adblockplus.org/easylistitaly.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-social.txt": "https://easylist-downloads.adblockplus.org/fanboy-social.txt", - "assets/thirdparties/easylist-downloads.adblockplus.org/liste_fr.txt": "https://easylist-downloads.adblockplus.org/liste_fr.txt", - "assets/thirdparties/gitorious.org/adblock-latvian/adblock-latvian/raw/master_lists/latvian-list.txt": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt", - "assets/thirdparties/home.fredfiber.no/langsholt/adblock.txt": "http://home.fredfiber.no/langsholt/adblock.txt", - "assets/thirdparties/hosts-file.net/ad-servers": "http://hosts-file.net/.%5Cad_servers.txt", - "assets/thirdparties/http://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "assets/thirdparties/liste-ar-adblock.googlecode.com/hg/Liste_AR.txt": "https://liste-ar-adblock.googlecode.com/hg/Liste_AR.txt", - "assets/thirdparties/margevicius.lt/easylistlithuania.txt": "http://margevicius.lt/easylistlithuania.txt", - "assets/thirdparties/mirror1.malwaredomains.com/files/immortal_domains.txt": "http://malwaredomains.lehigh.edu/files/immortal_domains.txt", - "assets/thirdparties/raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt": "https://raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt", - "assets/thirdparties/raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt", - "assets/thirdparties/raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt", - "assets/thirdparties/raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt", - "assets/thirdparties/raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt", - "assets/thirdparties/someonewhocares.org/hosts/hosts": "http://someonewhocares.org/hosts/hosts", - "assets/thirdparties/spam404bl.com/spam404scamlist.txt": "https://spam404bl.com/spam404scamlist.txt", - "assets/thirdparties/stanev.org/abp/adblock_bg.txt": "http://stanev.org/abp/adblock_bg.txt", - "assets/thirdparties/winhelp2002.mvps.org/hosts.txt": "http://winhelp2002.mvps.org/hosts.txt", - "assets/thirdparties/www.fanboy.co.nz/enhancedstats.txt": "https://www.fanboy.co.nz/enhancedstats.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-antifacebook.txt": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-korean.txt": "https://www.fanboy.co.nz/fanboy-korean.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-swedish.txt": "https://www.fanboy.co.nz/fanboy-swedish.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-ultimate.txt": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt", - "assets/thirdparties/www.fanboy.co.nz/fanboy-vietnam.txt": "https://www.fanboy.co.nz/fanboy-vietnam.txt", - "assets/thirdparties/www.void.gr/kargig/void-gr-filters.txt": "https://www.void.gr/kargig/void-gr-filters.txt", - "assets/thirdparties/www.zoso.ro/pages/rolist.txt": "", - "https://iadb.azurewebsites.net/Finland_adb.txt": "http://adb.juvander.net/Finland_adb.txt", - "https://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", - "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt" - }, + userFiltersPath: 'user-filters', + pslAssetKey: 'public_suffix_list.dat', + + availableFilterLists: {}, selfieAfter: 23 * oneMinute, diff --git a/src/js/logger.js b/src/js/logger.js index b9a7bee328748..700521ab5eada 100644 --- a/src/js/logger.js +++ b/src/js/logger.js @@ -19,15 +19,13 @@ Home: https://github.com/gorhill/uBlock */ -/* global µBlock */ +'use strict'; /******************************************************************************/ /******************************************************************************/ µBlock.logger = (function() { -'use strict'; - /******************************************************************************/ /******************************************************************************/ diff --git a/src/js/messaging.js b/src/js/messaging.js index 360fc7e0f39fd..4cc466f7c4314 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -77,7 +77,7 @@ var onMessage = function(request, sender, callback) { return; case 'reloadAllFilters': - µb.reloadAllFilters(callback); + µb.loadFilterLists(); return; case 'scriptlet': @@ -121,7 +121,8 @@ var onMessage = function(request, sender, callback) { break; case 'forceUpdateAssets': - µb.assetUpdater.force(); + µb.scheduleAssetUpdater(0); + µb.assets.updateStart({ delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod || 2000 }); break; case 'getAppData': @@ -160,7 +161,7 @@ var onMessage = function(request, sender, callback) { break; case 'selectFilterLists': - µb.selectFilterLists(request.switches); + µb.saveSelectedFilterLists(request.keys, request.append); break; case 'setWhitelist': @@ -753,7 +754,7 @@ var backupUserData = function(callback) { timeStamp: Date.now(), version: vAPI.app.version, userSettings: µb.userSettings, - filterLists: {}, + selectedFilterLists: [], hiddenSettingsString: µb.stringFromHiddenSettings(), netWhitelist: µb.stringFromWhitelist(µb.netWhitelist), dynamicFilteringString: µb.permanentFirewall.toString(), @@ -762,8 +763,17 @@ var backupUserData = function(callback) { userFilters: '' }; - var onSelectedListsReady = function(filterLists) { - userData.filterLists = filterLists; + var onSelectedListsReady = function(selectedFilterLists) { + userData.selectedFilterLists = selectedFilterLists; + + // TODO(seamless migration): + // The following is strictly for convenience, to be minimally + // forward-compatible. This will definitely be removed in the + // short term, as I do not expect the need to install an older + // version of uBO to ever be needed beyond the short term. + // >>>>>>>> + userData.filterLists = µb.oldDataFromNewListKeys(selectedFilterLists); + // <<<<<<<< var filename = vAPI.i18n('aboutBackupFilename') .replace('{{datetime}}', µb.dateNowToSensibleString()) @@ -773,17 +783,15 @@ var backupUserData = function(callback) { 'url': 'data:text/plain;charset=utf-8,' + encodeURIComponent(JSON.stringify(userData, null, ' ')), 'filename': filename }); - µb.restoreBackupSettings.lastBackupFile = filename; µb.restoreBackupSettings.lastBackupTime = Date.now(); vAPI.storage.set(µb.restoreBackupSettings); - getLocalData(callback); }; var onUserFiltersReady = function(details) { userData.userFilters = details.content; - µb.extractSelectedFilterLists(onSelectedListsReady); + µb.loadSelectedFilterLists(onSelectedListsReady); }; µb.assets.get(µb.userFiltersPath, onUserFiltersReady); @@ -791,32 +799,32 @@ var backupUserData = function(callback) { var restoreUserData = function(request) { var userData = request.userData; - var countdown = 8; - var onCountdown = function() { - countdown -= 1; - if ( countdown === 0 ) { - vAPI.app.restart(); - } - }; var onAllRemoved = function() { - // Be sure to adjust `countdown` if adding/removing anything below - µb.keyvalSetOne('version', userData.version); µBlock.saveLocalSettings(); - vAPI.storage.set(userData.userSettings, onCountdown); - µb.keyvalSetOne('remoteBlacklists', userData.filterLists, onCountdown); + vAPI.storage.set(userData.userSettings); µb.hiddenSettingsFromString(userData.hiddenSettingsString || ''); - µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown); - µb.keyvalSetOne('dynamicFilteringString', userData.dynamicFilteringString || '', onCountdown); - µb.keyvalSetOne('urlFilteringString', userData.urlFilteringString || '', onCountdown); - µb.keyvalSetOne('hostnameSwitchesString', userData.hostnameSwitchesString || '', onCountdown); - µb.assets.put(µb.userFiltersPath, userData.userFilters, onCountdown); vAPI.storage.set({ + netWhitelist: userData.netWhitelist || '', + dynamicFilteringString: userData.dynamicFilteringString || '', + urlFilteringString: userData.urlFilteringString || '', + hostnameSwitchesString: userData.hostnameSwitchesString || '', lastRestoreFile: request.file || '', lastRestoreTime: Date.now(), lastBackupFile: '', lastBackupTime: 0 - }, onCountdown); + }); + µb.assets.put(µb.userFiltersPath, userData.userFilters); + + // 'filterLists' is available up to uBO v1.10.4, not beyond. + // 'selectedFilterLists' is available from uBO v1.11 and beyond. + if ( Array.isArray(userData.selectedFilterLists) ) { + µb.saveSelectedFilterLists(userData.selectedFilterLists); + } else if ( userData.filterLists instanceof Object ) { + µb.saveSelectedFilterLists(µb.newListKeysFromOldData(userData.filterLists)); + } + + vAPI.app.restart(); }; // https://github.com/chrisaljoudi/uBlock/issues/1102 @@ -848,9 +856,7 @@ var prepListEntries = function(entries) { var µburi = µb.URI; var entry, hn; for ( var k in entries ) { - if ( entries.hasOwnProperty(k) === false ) { - continue; - } + if ( entries.hasOwnProperty(k) === false ) { continue; } entry = entries[k]; if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) { entry.supportName = µburi.hostnameFromURI(entry.supportURL); @@ -869,16 +875,14 @@ var getLists = function(callback) { cache: null, parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters, cosmeticFilterCount: µb.cosmeticFilteringEngine.getFilterCount(), - current: µb.remoteBlacklists, + current: µb.availableFilterLists, ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, - manualUpdate: false, netFilterCount: µb.staticNetFilteringEngine.getFilterCount(), - userFiltersPath: µb.userFiltersPath + userFiltersPath: µb.userFiltersPath, + aliases: µb.assets.listKeyAliases }; var onMetadataReady = function(entries) { r.cache = entries; - r.manualUpdate = µb.assetUpdater.manualUpdate; - r.manualUpdateProgress = µb.assetUpdater.manualUpdateProgress; prepListEntries(r.cache); callback(r); }; @@ -952,9 +956,6 @@ var onMessage = function(request, sender, callback) { case 'getLocalData': return getLocalData(callback); - case 'purgeAllCaches': - return µb.assets.purgeAll(callback); - case 'readUserFilters': return µb.loadUserFilters(callback); @@ -973,8 +974,18 @@ var onMessage = function(request, sender, callback) { response = getRules(); break; + case 'purgeAllCaches': + if ( request.hard ) { + µb.assets.remove(/./); + } else { + µb.assets.remove(/compiled\//); + µb.assets.purge(/./); + } + break; + case 'purgeCache': - µb.assets.purgeCacheableAsset(request.path); + µb.assets.purge(request.assetKey); + µb.assets.remove('compiled/' + request.assetKey); break; case 'readHiddenSettings': diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index 473bffa79a372..dfae64ffd158d 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -402,27 +402,15 @@ RedirectEngine.prototype.resourceContentFromName = function(name, mime) { // TODO: combine same key-redirect pairs into a single regex. RedirectEngine.prototype.resourcesFromString = function(text) { - var textEnd = text.length; - var lineBeg = 0, lineEnd; - var line, fields, encoded; - var reNonEmptyLine = /\S/; + var line, fields, encoded, + reNonEmptyLine = /\S/, + lineIter = new µBlock.LineIterator(text); this.resources = new Map(); - while ( lineBeg < textEnd ) { - lineEnd = text.indexOf('\n', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = text.indexOf('\r', lineBeg); - if ( lineEnd < 0 ) { - lineEnd = textEnd; - } - } - line = text.slice(lineBeg, lineEnd); - lineBeg = lineEnd + 1; - - if ( line.startsWith('#') ) { - continue; - } + while ( lineIter.eot() === false ) { + line = lineIter.next(); + if ( line.startsWith('#') ) { continue; } if ( fields === undefined ) { fields = line.trim().split(/\s+/); diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js index f52002e676750..17e8f99151af5 100644 --- a/src/js/reverselookup-worker.js +++ b/src/js/reverselookup-worker.js @@ -1,7 +1,7 @@ /******************************************************************************* - uBlock - a browser extension to block requests. - Copyright (C) 2015 Raymond Hill + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -43,8 +43,8 @@ var fromNetFilter = function(details) { var lists = []; var compiledFilter = details.compiledFilter; var entry, content, pos, c; - for ( var path in listEntries ) { - entry = listEntries[path]; + for ( var assetKey in listEntries ) { + entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } @@ -173,11 +173,11 @@ var fromCosmeticFilter = function(details) { ); } - var re, path, entry; + var re, assetKey, entry; for ( var candidate in candidates ) { re = candidates[candidate]; - for ( path in listEntries ) { - entry = listEntries[path]; + for ( assetKey in listEntries ) { + entry = listEntries[assetKey]; if ( entry === undefined ) { continue; } @@ -206,7 +206,7 @@ var reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/; /******************************************************************************/ -onmessage = function(e) { +onmessage = function(e) { // jshint ignore:line var msg = e.data; switch ( msg.what ) { @@ -215,7 +215,7 @@ onmessage = function(e) { break; case 'setList': - listEntries[msg.details.path] = msg.details; + listEntries[msg.details.assetKey] = msg.details; break; case 'fromNetFilter': diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 79af14e7c571e..c18bfba6662b7 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -1,7 +1,7 @@ /******************************************************************************* - uBlock - a browser extension to block requests. - Copyright (C) 2015 Raymond Hill + uBlock Origin - a browser extension to block requests. + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,14 +19,12 @@ Home: https://github.com/gorhill/uBlock */ -/* global µBlock */ +'use strict'; /******************************************************************************/ µBlock.staticFilteringReverseLookup = (function() { -'use strict'; - /******************************************************************************/ var worker = null; @@ -77,16 +75,16 @@ var initWorker = function(callback) { var countdown = 0; var onListLoaded = function(details) { - var entry = entries[details.path]; + var entry = entries[details.assetKey]; // https://github.com/gorhill/uBlock/issues/536 - // Use path string when there is no filter list title. + // Use assetKey when there is no filter list title. worker.postMessage({ what: 'setList', details: { - path: details.path, - title: entry.title || details.path, + assetKey: details.assetKey, + title: entry.title || details.assetKey, supportURL: entry.supportURL, content: details.content } @@ -99,18 +97,18 @@ var initWorker = function(callback) { }; var µb = µBlock; - var path, entry; + var listKey, entry; - for ( path in µb.remoteBlacklists ) { - if ( µb.remoteBlacklists.hasOwnProperty(path) === false ) { - continue; - } - entry = µb.remoteBlacklists[path]; - if ( entry.off === true ) { + for ( listKey in µb.availableFilterLists ) { + if ( µb.availableFilterLists.hasOwnProperty(listKey) === false ) { continue; } - entries[path] = { - title: path !== µb.userFiltersPath ? entry.title : vAPI.i18n('1pPageName'), + entry = µb.availableFilterLists[listKey]; + if ( entry.off === true ) { continue; } + entries[listKey] = { + title: listKey !== µb.userFiltersPath ? + entry.title : + vAPI.i18n('1pPageName'), supportURL: entry.supportURL || '' }; countdown += 1; @@ -121,8 +119,8 @@ var initWorker = function(callback) { return; } - for ( path in entries ) { - µb.getCompiledFilterList(path, onListLoaded); + for ( listKey in entries ) { + µb.getCompiledFilterList(listKey, onListLoaded); } }; diff --git a/src/js/scriptlets/subscriber.js b/src/js/scriptlets/subscriber.js index 9c525de0d9b36..887e95db831f3 100644 --- a/src/js/scriptlets/subscriber.js +++ b/src/js/scriptlets/subscriber.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2015-2016 Raymond Hill + Copyright (C) 2015-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,6 +21,8 @@ /* global vAPI, HTMLDocument */ +'use strict'; + /******************************************************************************/ // Injected into specific web pages, those which have been pre-selected @@ -30,8 +32,6 @@ (function() { -'use strict'; - /******************************************************************************/ // https://github.com/chrisaljoudi/uBlock/issues/464 @@ -100,7 +100,8 @@ var onAbpLinkClicked = function(ev) { 'scriptlets', { what: 'selectFilterLists', - switches: [ { location: location, off: false } ] + keys: [ location ], + append: true }, onListsSelectionDone ); diff --git a/src/js/settings.js b/src/js/settings.js index 6e75aa1584e30..16582955916ed 100644 --- a/src/js/settings.js +++ b/src/js/settings.js @@ -56,7 +56,10 @@ var handleImportFilePicker = function() { if ( typeof userData.netWhitelist !== 'string' ) { throw 'Invalid'; } - if ( typeof userData.filterLists !== 'object' ) { + if ( + typeof userData.filterLists !== 'object' && + Array.isArray(userData.selectedFilterLists) === false + ) { throw 'Invalid'; } } diff --git a/src/js/start.js b/src/js/start.js index 12a583db48137..2ccd4fc8dc061 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ var µb = µBlock; vAPI.app.onShutdown = function() { µb.staticFilteringReverseLookup.shutdown(); - µb.assetUpdater.shutdown(); + µb.assets.updateStop(); µb.staticNetFilteringEngine.reset(); µb.cosmeticFilteringEngine.reset(); µb.sessionFirewall.reset(); @@ -58,14 +58,8 @@ vAPI.app.onShutdown = function() { var onAllReady = function() { // https://github.com/chrisaljoudi/uBlock/issues/184 // Check for updates not too far in the future. - µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb)); - µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb)); - µb.assetUpdater.onAssetUpdated.addEventListener(µb.assetUpdatedHandler.bind(µb)); - µb.assets.onAssetRemoved.addListener(µb.assetCacheRemovedHandler.bind(µb)); - - // Important: remove barrier to remote fetching, this was useful only - // for launch time. - µb.assets.remoteFetchBarrier -= 1; + µb.assets.addObserver(µb.assetObserver.bind(µb)); + µb.scheduleAssetUpdater(µb.userSettings.autoUpdate ? 7 * 60 * 1000 : 0); // vAPI.cloud is optional. if ( µb.cloudStorageSupported ) { @@ -129,7 +123,7 @@ var onSelfieReady = function(selfie) { return false; } - µb.remoteBlacklists = selfie.filterLists; + µb.availableFilterLists = selfie.availableFilterLists; µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); µb.redirectEngine.fromSelfie(selfie.redirectEngine); µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine); @@ -157,12 +151,6 @@ var onUserSettingsReady = function(fetched) { fromFetch(userSettings, fetched); - // https://github.com/chrisaljoudi/uBlock/issues/426 - // Important: block remote fetching for when loading assets at launch - // time. - µb.assets.autoUpdate = userSettings.autoUpdate; - µb.assets.autoUpdateDelay = µb.updateAssetsEvery; - if ( µb.privacySettingsSupported ) { vAPI.browserSettings.set({ 'hyperlinkAuditing': !userSettings.hyperlinkAuditingDisabled, @@ -192,7 +180,7 @@ var onUserSettingsReady = function(fetched) { var onSystemSettingsReady = function(fetched) { var mustSaveSystemSettings = false; if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) { - µb.assets.purge(/^cache:\/\/compiled-/); + µb.assets.remove(/^compiled\//); mustSaveSystemSettings = true; } if ( fetched.selfieMagic !== µb.systemSettings.selfieMagic ) { @@ -254,9 +242,6 @@ var fromFetch = function(to, fetched) { /******************************************************************************/ var onAdminSettingsRestored = function() { - // Forbid remote fetching of assets - µb.assets.remoteFetchBarrier += 1; - var fetchableProps = { 'compiledMagic': '', 'dynamicFilteringString': 'behind-the-scene * 3p noop\nbehind-the-scene * 3p-frame noop', diff --git a/src/js/storage.js b/src/js/storage.js index db079f5ac1b8f..c587919487353 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uBlock */ -/* global YaMD5, objectAssign, punycode, publicSuffixList */ +/* global objectAssign, punycode, publicSuffixList */ 'use strict'; @@ -104,6 +104,12 @@ case 'string': out[name] = value; break; + case 'number': + out[name] = parseInt(value, 10); + if ( isNaN(out[name]) ) { + out[name] = this.hiddenSettingsDefault[name]; + } + break; default: break; } @@ -151,55 +157,107 @@ this.netWhitelistModifyTime = Date.now(); }; -/******************************************************************************/ +/******************************************************************************* -// This will remove all unused filter list entries from -// µBlock.remoteBlacklists`. This helps reduce the size of backup files. + TODO(seamless migration): + The code related to 'remoteBlacklist' can be removed when I am confident + all users have moved to a version of uBO which no longer depends on + the property 'remoteBlacklists, i.e. v1.11 and beyond. -µBlock.extractSelectedFilterLists = function(callback) { - var µb = this; +**/ - var onBuiltinListsLoaded = function(details) { - var builtin; - try { - builtin = JSON.parse(details.content); - } catch (e) { - builtin = {}; +µBlock.loadSelectedFilterLists = function(callback) { + var µb = this; + vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists' ], function(bin) { + if ( !bin || !bin.selectedFilterLists && !bin.remoteBlacklists ) { + return callback(); } - - var result = JSON.parse(JSON.stringify(µb.remoteBlacklists)); - var entry, builtinPath, defaultState; - - for ( var path in result ) { - if ( result.hasOwnProperty(path) === false ) { - continue; - } - entry = result[path]; - // https://github.com/gorhill/uBlock/issues/277 - // uBlock's filter lists are always enabled by default, so we - // have to include in backup only those which are turned off. - if ( path.startsWith('assets/ublock/') ) { - if ( entry.off !== true ) { - delete result[path]; - } - continue; - } - builtinPath = path.replace(/^assets\/thirdparties\//, ''); - defaultState = builtin.hasOwnProperty(builtinPath) === false || - builtin[builtinPath].off === true; - if ( entry.off === true && entry.off === defaultState ) { - delete result[path]; + var listKeys = []; + if ( bin.selectedFilterLists ) { + listKeys = bin.selectedFilterLists; + } + if ( bin.remoteBlacklists ) { + var oldListKeys = µb.newListKeysFromOldData(bin.remoteBlacklists); + if ( oldListKeys.sort().join() !== listKeys.sort().join() ) { + listKeys = oldListKeys; + µb.saveSelectedFilterLists(listKeys); } + // TODO(seamless migration): + // Uncomment when all have moved to v1.11 and beyond. + //vAPI.storage.remove('remoteBlacklists'); } + callback(listKeys); + }); +}; - callback(result); +µBlock.saveSelectedFilterLists = function(listKeys, append) { + var µb = this; + var save = function(keys) { + var bin = { + selectedFilterLists: keys, + remoteBlacklists: µb.oldDataFromNewListKeys(keys) + }; + vAPI.storage.set(bin); }; + if ( append ) { + this.loadSelectedFilterLists(function(keys) { + listKeys = listKeys.concat(keys || []); + save(listKeys); + }); + } else { + save(listKeys); + } +}; + +// TODO(seamless migration): +// Remove when all have moved to v1.11 and beyond. +// >>>>>>>> +µBlock.newListKeysFromOldData = function(oldLists) { + var aliases = this.assets.listKeyAliases, + listKeys = [], newKey; + for ( var oldKey in oldLists ) { + if ( oldLists[oldKey].off !== true ) { + newKey = aliases[oldKey]; + listKeys.push(newKey ? newKey : oldKey); + } + } + return listKeys; +}; - // https://github.com/gorhill/uBlock/issues/63 - // Get built-in block lists: this will help us determine whether a - // specific list must be included in the result. - this.loadAndPatchStockFilterLists(onBuiltinListsLoaded); +µBlock.oldDataFromNewListKeys = function(selectedFilterLists) { + var µb = this, + remoteBlacklists = {}; + var reverseAliases = Object.keys(this.assets.listKeyAliases).reduce( + function(a, b) { + a[µb.assets.listKeyAliases[b]] = b; return a; + }, + {} + ); + remoteBlacklists = selectedFilterLists.reduce( + function(a, b) { + a[reverseAliases[b] || b] = { off: false }; + return a; + }, + {} + ); + remoteBlacklists = Object.keys(µb.assets.listKeyAliases).reduce( + function(a, b) { + var aliases = µb.assets.listKeyAliases; + if ( + b.startsWith('assets/') && + aliases[b] !== 'public_suffix_list.dat' && + aliases[b] !== 'ublock-resources' && + !a[b] + ) { + a[b] = { off: true }; + } + return a; + }, + remoteBlacklists + ); + return remoteBlacklists; }; +// <<<<<<<< /******************************************************************************/ @@ -207,14 +265,11 @@ // https://github.com/gorhill/uBlock/issues/1022 // Be sure to end with an empty line. content = content.trim(); - if ( content !== '' ) { - content += '\n'; - } + if ( content !== '' ) { content += '\n'; } this.assets.put(this.userFiltersPath, content, callback); + this.removeCompiledFilterList(this.userFiltersPath); }; -/******************************************************************************/ - µBlock.loadUserFilters = function(callback) { return this.assets.get(this.userFiltersPath, callback); }; @@ -222,25 +277,23 @@ /******************************************************************************/ µBlock.appendUserFilters = function(filters) { - if ( filters.length === 0 ) { - return; - } + if ( filters.length === 0 ) { return; } var µb = this; var onSaved = function() { - var compiledFilters = µb.compileFilters(filters); - var snfe = µb.staticNetFilteringEngine; - var cfe = µb.cosmeticFilteringEngine; - var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; - var discardedCount = snfe.discardedCount + cfe.discardedCount; + var compiledFilters = µb.compileFilters(filters), + snfe = µb.staticNetFilteringEngine, + cfe = µb.cosmeticFilteringEngine, + acceptedCount = snfe.acceptedCount + cfe.acceptedCount, + discardedCount = snfe.discardedCount + cfe.discardedCount; µb.applyCompiledFilters(compiledFilters, true); - var entry = µb.remoteBlacklists[µb.userFiltersPath]; - var deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; - var deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); + var entry = µb.availableFilterLists[µb.userFiltersPath], + deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount, + deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); entry.entryCount += deltaEntryCount; entry.entryUsedCount += deltaEntryUsedCount; - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); µb.staticNetFilteringEngine.freeze(); µb.redirectEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); @@ -248,9 +301,7 @@ }; var onLoaded = function(details) { - if ( details.error ) { - return; - } + if ( details.error ) { return; } // https://github.com/chrisaljoudi/uBlock/issues/976 // If we reached this point, the filter quite probably needs to be // added for sure: do not try to be too smart, trying to avoid @@ -263,166 +314,194 @@ /******************************************************************************/ -µBlock.getAvailableLists = function(callback) { - var availableLists = {}; - var relocationMap = {}; - - var fixLocation = function(location) { - // https://github.com/chrisaljoudi/uBlock/issues/418 - // We now support built-in external filter lists - if ( /^https?:/.test(location) === false ) { - location = 'assets/thirdparties/' + location; +µBlock.listKeysFromCustomFilterLists = function(raw) { + var out = {}; + var reIgnore = /^[!#]|[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/, + lineIter = new this.LineIterator(raw), + location; + while ( lineIter.eot() === false ) { + location = lineIter.next().trim(); + if ( location === '' || reIgnore.test(location) ) { continue; } + out[location] = true; + } + return Object.keys(out); +}; + +/******************************************************************************/ + +µBlock.autoSelectRegionalFilterLists = function(lists) { + var lang = self.navigator.language.slice(0, 2), + selectedListKeys = [], + list; + for ( var key in lists ) { + if ( lists.hasOwnProperty(key) === false ) { continue; } + list = lists[key]; + if ( list.off !== true ) { + selectedListKeys.push(key); + continue; } - return location; - }; + if ( list.lang === lang ) { + selectedListKeys.push(key); + list.off = false; + } + } + return selectedListKeys; +}; - // selected lists - var onSelectedListsLoaded = function(store) { - var µb = µBlock; - var lists = store.remoteBlacklists; - var locations = Object.keys(lists); - var location, availableEntry, storedEntry; - var off; - - while ( (location = locations.pop()) ) { - storedEntry = lists[location]; - off = storedEntry.off === true; - // New location? - if ( relocationMap.hasOwnProperty(location) ) { - µb.purgeFilterList(location); - location = relocationMap[location]; - if ( off && lists.hasOwnProperty(location) ) { - off = lists[location].off === true; - } +/******************************************************************************/ + +µBlock.changeExternalFilterLists = function(before, after) { + var µb = µBlock; + var onLoaded = function(keys) { + var fullDict = new Set(keys || []), + mustSave = false, + oldKeys = µb.listKeysFromCustomFilterLists(before), + oldDict = new Set(oldKeys), + newKeys = µb.listKeysFromCustomFilterLists(after), + newDict = new Set(newKeys), + i, key; + i = oldKeys.length; + while ( i-- ) { + key = oldKeys[i]; + if ( fullDict.has(key) && !newDict.has(key) ) { + fullDict.delete(key); + mustSave = true; } - availableEntry = availableLists[location]; - if ( availableEntry === undefined ) { - µb.purgeFilterList(location); - continue; + } + i = newKeys.length; + while ( i-- ) { + key = newKeys[i]; + if ( !fullDict.has(key) && !oldDict.has(key) ) { + fullDict.add(key); + mustSave = true; } - availableEntry.off = off; - if ( typeof availableEntry.homeURL === 'string' ) { - µb.assets.setHomeURL(location, availableEntry.homeURL); + } + if ( mustSave ) { + µb.saveSelectedFilterLists(µb.setToArray(fullDict)); + } + }; + this.loadSelectedFilterLists(onLoaded); +}; + +/******************************************************************************/ + +µBlock.getAvailableLists = function(callback) { + var µb = this, + oldAvailableLists = {}, + newAvailableLists = {}; + + // User filter list. + newAvailableLists[this.userFiltersPath] = { + group: 'default', + title: vAPI.i18n('1pPageName') + }; + + // Custom filter lists. + var importedListKeys = this.listKeysFromCustomFilterLists(µb.userSettings.externalLists), + i = importedListKeys.length, listKey, entry; + while ( i-- ) { + listKey = importedListKeys[i]; + entry = { + content: 'filters', + contentURL: importedListKeys[i], + external: true, + group: 'custom', + submitter: 'user', + title: '' + }; + newAvailableLists[listKey] = entry; + this.assets.registerAssetSource(listKey, entry); + } + + // Final steps: + // - reuse existing list metadata if any; + // - unregister unreferenced imported filter lists if any. + var finalize = function() { + var assetKey, newEntry, oldEntry; + + // Reuse existing metadata. + for ( assetKey in oldAvailableLists ) { + oldEntry = oldAvailableLists[assetKey]; + newEntry = newAvailableLists[assetKey]; + if ( newEntry === undefined ) { + µb.removeFilterList(assetKey); + continue; } - if ( storedEntry.entryCount !== undefined ) { - availableEntry.entryCount = storedEntry.entryCount; + if ( oldEntry.entryCount !== undefined ) { + newEntry.entryCount = oldEntry.entryCount; } - if ( storedEntry.entryUsedCount !== undefined ) { - availableEntry.entryUsedCount = storedEntry.entryUsedCount; + if ( oldEntry.entryUsedCount !== undefined ) { + newEntry.entryUsedCount = oldEntry.entryUsedCount; } // This may happen if the list name was pulled from the list // content. // https://github.com/chrisaljoudi/uBlock/issues/982 // There is no guarantee the title was successfully extracted from // the list content. - if ( availableEntry.title === '' && - typeof storedEntry.title === 'string' && - storedEntry.title !== '' + if ( + newEntry.title === '' && + typeof oldEntry.title === 'string' && + oldEntry.title !== '' ) { - availableEntry.title = storedEntry.title; + newEntry.title = oldEntry.title; } } - // https://github.com/gorhill/uBlock/issues/747 - if ( µb.firstInstall ) { - µb.autoSelectFilterLists(availableLists); + // Remove unreferenced imported filter lists. + var dict = new Set(importedListKeys); + for ( assetKey in newAvailableLists ) { + newEntry = newAvailableLists[assetKey]; + if ( newEntry.submitter !== 'user' ) { continue; } + if ( dict.has(assetKey) ) { continue; } + delete newAvailableLists[assetKey]; + µb.assets.unregisterAssetSource(assetKey); + µb.removeFilterList(assetKey); } - - callback(availableLists); }; - // built-in lists - var onBuiltinListsLoaded = function(details) { - var location, locations; - try { - locations = JSON.parse(details.content); - } catch (e) { - locations = {}; - } - var entry; - for ( location in locations ) { - if ( locations.hasOwnProperty(location) === false ) { - continue; - } - entry = locations[location]; - location = fixLocation(location); - // Migrate obsolete location to new location, if any - if ( typeof entry.oldLocation === 'string' ) { - entry.oldLocation = fixLocation(entry.oldLocation); - relocationMap[entry.oldLocation] = location; + // Selected lists. + var onSelectedListsLoaded = function(keys) { + var listKey; + // No user lists data means use default settings. + if ( Array.isArray(keys) ) { + var listKeySet = new Set(keys); + for ( listKey in newAvailableLists ) { + if ( newAvailableLists.hasOwnProperty(listKey) ) { + newAvailableLists[listKey].off = !listKeySet.has(listKey); + } } - availableLists[location] = entry; + } else if ( µb.firstInstall ) { + µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(newAvailableLists)); } - // Now get user's selection of lists - vAPI.storage.get( - { 'remoteBlacklists': availableLists }, - onSelectedListsLoaded - ); + finalize(); + callback(newAvailableLists); }; - // permanent lists - var location; - var lists = this.permanentLists; - for ( location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - availableLists[location] = lists[location]; - } - - // custom lists - var c; - var locations = this.userSettings.externalLists.split('\n'); - for ( var i = 0; i < locations.length; i++ ) { - location = locations[i].trim(); - c = location.charAt(0); - if ( location === '' || c === '!' || c === '#' ) { - continue; - } - // Coarse validation - if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) { - continue; + // Built-in filter lists. + var onBuiltinListsLoaded = function(entries) { + for ( var assetKey in entries ) { + if ( entries.hasOwnProperty(assetKey) === false ) { continue; } + entry = entries[assetKey]; + if ( entry.content !== 'filters' ) { continue; } + newAvailableLists[assetKey] = objectAssign({}, entry); } - availableLists[location] = { - title: '', - group: 'custom', - external: true - }; - } - // get built-in block lists. - this.loadAndPatchStockFilterLists(onBuiltinListsLoaded); -}; - -/******************************************************************************/ - -µBlock.autoSelectFilterLists = function(lists) { - var lang = self.navigator.language.slice(0, 2), - list; - for ( var path in lists ) { - if ( lists.hasOwnProperty(path) === false ) { - continue; - } - list = lists[path]; - if ( list.off !== true ) { - continue; - } - if ( list.lang === lang ) { - list.off = false; - } - } -}; + // Load set of currently selected filter lists. + µb.loadSelectedFilterLists(onSelectedListsLoaded); + }; -/******************************************************************************/ + // Available lists previously computed. + var onOldAvailableListsLoaded = function(bin) { + oldAvailableLists = bin && bin.availableFilterLists || {}; + µb.assets.metadata(onBuiltinListsLoaded); + }; -µBlock.createShortUniqueId = function(path) { - var md5 = YaMD5.hashStr(path); - return md5.slice(0, 4) + md5.slice(-4); + // Load previously saved available lists -- these contains data + // computed at run-time, we will reuse this data if possible. + vAPI.storage.get('availableFilterLists', onOldAvailableListsLoaded); }; -µBlock.createShortUniqueId.idLength = 8; - /******************************************************************************/ // This is used to be re-entrancy resistant. @@ -444,18 +523,11 @@ callback = this.noopFunc; } - // Never fetch from remote servers when we load filter lists: this has to - // be as fast as possible. - µb.assets.remoteFetchBarrier += 1; - var onDone = function() { - // Remove barrier to remote fetching - µb.assets.remoteFetchBarrier -= 1; - µb.staticNetFilteringEngine.freeze(); µb.cosmeticFilteringEngine.freeze(); µb.redirectEngine.freeze(); - vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists }); + vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists }); //quickProfiler.stop(0); @@ -473,15 +545,15 @@ var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; var discardedCount = snfe.discardedCount + cfe.discardedCount; µb.applyCompiledFilters(compiled, path === µb.userFiltersPath); - if ( µb.remoteBlacklists.hasOwnProperty(path) ) { - var entry = µb.remoteBlacklists[path]; + if ( µb.availableFilterLists.hasOwnProperty(path) ) { + var entry = µb.availableFilterLists[path]; entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; entry.entryUsedCount = entry.entryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); } }; var onCompiledListLoaded = function(details) { - applyCompiledFilters(details.path, details.content); + applyCompiledFilters(details.assetKey, details.content); filterlistsCount -= 1; if ( filterlistsCount === 0 ) { onDone(); @@ -489,7 +561,7 @@ }; var onFilterListsReady = function(lists) { - µb.remoteBlacklists = lists; + µb.availableFilterLists = lists; µb.redirectEngine.reset(); µb.cosmeticFilteringEngine.reset(); @@ -502,14 +574,10 @@ // This happens for assets which do not exist, ot assets with no // content. var toLoad = []; - for ( var path in lists ) { - if ( lists.hasOwnProperty(path) === false ) { - continue; - } - if ( lists[path].off ) { - continue; - } - toLoad.push(path); + for ( var assetKey in lists ) { + if ( lists.hasOwnProperty(assetKey) === false ) { continue; } + if ( lists[assetKey].off ) { continue; } + toLoad.push(assetKey); } filterlistsCount = toLoad.length; if ( filterlistsCount === 0 ) { @@ -528,32 +596,17 @@ /******************************************************************************/ -µBlock.getCompiledFilterListPath = function(path) { - return 'cache://compiled-filter-list:' + this.createShortUniqueId(path); -}; - -/******************************************************************************/ - -µBlock.getCompiledFilterList = function(path, callback) { - var compiledPath = this.getCompiledFilterListPath(path); - var µb = this; +µBlock.getCompiledFilterList = function(assetKey, callback) { + var µb = this, + compiledPath = 'compiled/' + assetKey; var onRawListLoaded = function(details) { + details.assetKey = assetKey; if ( details.content === '' ) { callback(details); return; } - var listMeta = µb.remoteBlacklists[path]; - // https://github.com/gorhill/uBlock/issues/313 - // Always try to fetch the name if this is an external filter list. - if ( listMeta && (listMeta.title === '' || listMeta.group === 'custom') ) { - var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i); - if ( matches !== null ) { - listMeta.title = matches[1].trim(); - } - } - - //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); + µb.extractFilterListMetadata(assetKey, details.content); details.content = µb.compileFilters(details.content); µb.assets.put(compiledPath, details.content); callback(details); @@ -561,12 +614,10 @@ var onCompiledListLoaded = function(details) { if ( details.content === '' ) { - //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: no compiled version for "%s"', path); - µb.assets.get(path, onRawListLoaded); + µb.assets.get(assetKey, onRawListLoaded); return; } - //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: using compiled version for "%s"', path); - details.path = path; + details.assetKey = assetKey; callback(details); }; @@ -575,61 +626,70 @@ /******************************************************************************/ -µBlock.purgeCompiledFilterList = function(path) { - this.assets.purge(this.getCompiledFilterListPath(path)); +µBlock.extractFilterListMetadata = function(assetKey, raw) { + var listEntry = this.availableFilterLists[assetKey]; + if ( listEntry === undefined ) { return; } + // Metadata expected to be found at the top of content. + var head = raw.slice(0, 1024), + matches, v; + // https://github.com/gorhill/uBlock/issues/313 + // Always try to fetch the name if this is an external filter list. + if ( listEntry.title === '' || listEntry.group === 'custom' ) { + matches = head.match(/(?:^|\n)!\s*Title:([^\n]+)/i); + if ( matches !== null ) { + listEntry.title = matches[1].trim(); + } + } + // Extract update frequency information + matches = head.match(/(?:^|\n)![\t ]*Expires:[\t ]*([\d]+)[\t ]*days?/i); + if ( matches !== null ) { + v = Math.max(parseInt(matches[1], 10), 2); + if ( v !== listEntry.updateAfter ) { + this.assets.registerAssetSource(assetKey, { updateAfter: v }); + } + } }; /******************************************************************************/ -µBlock.purgeFilterList = function(path) { - this.purgeCompiledFilterList(path); - this.assets.purge(path); +µBlock.removeCompiledFilterList = function(assetKey) { + this.assets.remove('compiled/' + assetKey); +}; + +µBlock.removeFilterList = function(assetKey) { + this.removeCompiledFilterList(assetKey); + this.assets.remove(assetKey); }; /******************************************************************************/ µBlock.compileFilters = function(rawText) { - var rawEnd = rawText.length; var compiledFilters = []; // Useful references: // https://adblockplus.org/en/filter-cheatsheet // https://adblockplus.org/en/filters - var staticNetFilteringEngine = this.staticNetFilteringEngine; - var cosmeticFilteringEngine = this.cosmeticFilteringEngine; - var reIsWhitespaceChar = /\s/; - var reMaybeLocalIp = /^[\d:f]/; - var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/; - var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/; - - var lineBeg = 0, lineEnd, currentLineBeg; - var line, lineRaw, c, pos; - - while ( lineBeg < rawEnd ) { - lineEnd = rawText.indexOf('\n', lineBeg); - if ( lineEnd === -1 ) { - lineEnd = rawText.indexOf('\r', lineBeg); - if ( lineEnd === -1 ) { - lineEnd = rawEnd; - } - } + var staticNetFilteringEngine = this.staticNetFilteringEngine, + cosmeticFilteringEngine = this.cosmeticFilteringEngine, + reIsWhitespaceChar = /\s/, + reMaybeLocalIp = /^[\d:f]/, + reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/, + reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/, + line, lineRaw, c, pos, + lineIter = new this.LineIterator(rawText); + + while ( lineIter.eot() === false ) { + line = lineRaw = lineIter.next().trim(); // rhill 2014-04-18: The trim is important here, as without it there // could be a lingering `\r` which would cause problems in the // following parsing code. - line = lineRaw = rawText.slice(lineBeg, lineEnd).trim(); - currentLineBeg = lineBeg; - lineBeg = lineEnd + 1; - if ( line.length === 0 ) { - continue; - } + if ( line.length === 0 ) { continue; } // Strip comments c = line.charAt(0); - if ( c === '!' || c === '[' ) { - continue; - } + if ( c === '!' || c === '[' ) { continue; } // Parse or skip cosmetic filters // All cosmetic filters are caught here @@ -640,9 +700,7 @@ // Whatever else is next can be assumed to not be a cosmetic filter // Most comments start in first column - if ( c === '#' ) { - continue; - } + if ( c === '#' ) { continue; } // Catch comments somewhere on the line // Remove: @@ -663,15 +721,11 @@ // Ignore hosts file redirect configuration // 127.0.0.1 localhost // 255.255.255.255 broadcasthost - if ( reIsLocalhostRedirect.test(line) ) { - continue; - } + if ( reIsLocalhostRedirect.test(line) ) { continue; } line = line.replace(reLocalIp, '').trim(); } - if ( line.length === 0 ) { - continue; - } + if ( line.length === 0 ) { continue; } staticNetFilteringEngine.compile(line, compiledFilters); } @@ -699,55 +753,6 @@ /******************************************************************************/ -// `switches` contains the filter lists for which the switch must be revisited. - -µBlock.selectFilterLists = function(switches) { - switches = switches || {}; - - // Only the lists referenced by the switches are touched. - var filterLists = this.remoteBlacklists; - var entry, state, location; - var i = switches.length; - while ( i-- ) { - entry = switches[i]; - state = entry.off === true; - location = entry.location; - if ( filterLists.hasOwnProperty(location) === false ) { - if ( state !== true ) { - filterLists[location] = { off: state }; - } - continue; - } - if ( filterLists[location].off === state ) { - continue; - } - filterLists[location].off = state; - } - - vAPI.storage.set({ 'remoteBlacklists': filterLists }); -}; - -/******************************************************************************/ - -// Plain reload of all filters. - -µBlock.reloadAllFilters = function() { - var µb = this; - - // We are just reloading the filter lists: we do not want assets to update. - // TODO: probably not needed anymore, since filter lists are now always - // loaded without update => see `µb.assets.remoteFetchBarrier`. - this.assets.autoUpdate = false; - - var onFiltersReady = function() { - µb.assets.autoUpdate = µb.userSettings.autoUpdate; - }; - - this.loadFilterLists(onFiltersReady); -}; - -/******************************************************************************/ - µBlock.loadRedirectResources = function(callback) { var µb = this; @@ -762,40 +767,46 @@ callback(); }; - this.assets.get('assets/ublock/resources.txt', onResourcesLoaded); + this.assets.get('ublock-resources', onResourcesLoaded); }; /******************************************************************************/ µBlock.loadPublicSuffixList = function(callback) { - var µb = this; - var path = µb.pslPath; - var compiledPath = 'cache://compiled-publicsuffixlist'; + var µb = this, + assetKey = µb.pslAssetKey, + compiledAssetKey = 'compiled/' + assetKey; if ( typeof callback !== 'function' ) { callback = this.noopFunc; } var onRawListLoaded = function(details) { if ( details.content !== '' ) { - //console.debug('µBlock.loadPublicSuffixList/onRawListLoaded: compiling "%s"', path); - publicSuffixList.parse(details.content, punycode.toASCII); - µb.assets.put(compiledPath, JSON.stringify(publicSuffixList.toSelfie())); + µb.compilePublicSuffixList(details.content); } callback(); }; var onCompiledListLoaded = function(details) { if ( details.content === '' ) { - //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: no compiled version for "%s"', path); - µb.assets.get(path, onRawListLoaded); + µb.assets.get(assetKey, onRawListLoaded); return; } - //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: using compiled version for "%s"', path); publicSuffixList.fromSelfie(JSON.parse(details.content)); callback(); }; - this.assets.get(compiledPath, onCompiledListLoaded); + this.assets.get(compiledAssetKey, onCompiledListLoaded); +}; + +/******************************************************************************/ + +µBlock.compilePublicSuffixList = function(content) { + publicSuffixList.parse(content, punycode.toASCII); + this.assets.put( + 'compiled/' + this.pslAssetKey, + JSON.stringify(publicSuffixList.toSelfie()) + ); }; /******************************************************************************/ @@ -814,7 +825,7 @@ var selfie = { magic: µb.systemSettings.selfieMagic, publicSuffixList: publicSuffixList.toSelfie(), - filterLists: µb.remoteBlacklists, + availableFilterLists: µb.availableFilterLists, staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), redirectEngine: µb.redirectEngine.toSelfie(), cosmeticFilteringEngine: µb.cosmeticFilteringEngine.toSelfie() @@ -885,6 +896,13 @@ var bin = {}; var binNotEmpty = false; + // Allows an admin to set their own 'assets.json' file, with their own + // set of stock assets. + if ( typeof data.assetsBootstrapLocation === 'string' ) { + bin.assetsBootstrapLocation = data.assetsBootstrapLocation; + binNotEmpty = true; + } + if ( typeof data.userSettings === 'object' ) { for ( var name in µb.userSettings ) { if ( µb.userSettings.hasOwnProperty(name) === false ) { @@ -898,8 +916,13 @@ } } - if ( typeof data.filterLists === 'object' ) { - bin.remoteBlacklists = data.filterLists; + // 'selectedFilterLists' is an array of filter list tokens. Each token + // is a reference to an asset in 'assets.json'. + if ( Array.isArray(data.selectedFilterLists) ) { + bin.selectedFilterLists = data.selectedFilterLists; + binNotEmpty = true; + } else if ( typeof data.filterLists === 'object' ) { + bin.selectedFilterLists = µb.newListKeysFromOldData(data.filterLists); binNotEmpty = true; } @@ -939,203 +962,95 @@ /******************************************************************************/ -µBlock.updateStartHandler = function(callback) { - var µb = this; - var onListsReady = function(lists) { - var assets = {}; - for ( var location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - if ( lists[location].off ) { - continue; - } - assets[location] = true; - } - assets[µb.pslPath] = true; - assets['assets/ublock/resources.txt'] = true; - callback(assets); - }; - - this.getAvailableLists(onListsReady); -}; - -/******************************************************************************/ - -µBlock.assetUpdatedHandler = function(details) { - var path = details.path || ''; - if ( this.remoteBlacklists.hasOwnProperty(path) === false ) { - return; - } - var entry = this.remoteBlacklists[path]; - if ( entry.off ) { - return; - } - // Compile the list while we have the raw version in memory - //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); - this.assets.put( - this.getCompiledFilterListPath(path), - this.compileFilters(details.content) - ); -}; - -/******************************************************************************/ - -µBlock.updateCompleteHandler = function(details) { - var µb = this; - var updatedCount = details.updatedCount; - - // Assets are supposed to have been all updated, prevent fetching from - // remote servers. - µb.assets.remoteFetchBarrier += 1; - - var onFiltersReady = function() { - µb.assets.remoteFetchBarrier -= 1; - }; - - var onPSLReady = function() { - if ( updatedCount !== 0 ) { - //console.debug('storage.js > µBlock.updateCompleteHandler: reloading filter lists'); - µb.loadFilterLists(onFiltersReady); - } else { - onFiltersReady(); +µBlock.scheduleAssetUpdater = (function() { + var timer, next = 0; + return function(updateDelay) { + if ( timer ) { + clearTimeout(timer); + timer = undefined; } - }; - - if ( details.hasOwnProperty(this.pslPath) ) { - //console.debug('storage.js > µBlock.updateCompleteHandler: reloading PSL'); - this.loadPublicSuffixList(onPSLReady); - updatedCount -= 1; - } else { - onPSLReady(); - } -}; - -/******************************************************************************/ - -µBlock.assetCacheRemovedHandler = (function() { - var barrier = false; - - var handler = function(paths) { - if ( barrier ) { + if ( updateDelay === 0 ) { + next = 0; return; } - barrier = true; - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( this.remoteBlacklists.hasOwnProperty(path) ) { - //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path); - this.purgeCompiledFilterList(path); - continue; - } - if ( path === this.pslPath ) { - //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path); - this.assets.purge('cache://compiled-publicsuffixlist'); - continue; - } + var now = Date.now(); + // Use the new schedule if and only if it is earlier than the previous + // one. + if ( next !== 0 ) { + updateDelay = Math.min(updateDelay, Math.max(next - now, 0)); } - this.selfieManager.destroy(); - barrier = false; + next = now + updateDelay; + timer = vAPI.setTimeout(function() { + timer = undefined; + next = 0; + var µb = µBlock; + µb.assets.updateStart({ + delay: µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 120000 + }); + }, updateDelay); }; - - return handler; })(); /******************************************************************************/ -// https://github.com/gorhill/uBlock/issues/602 -// - Load and patch `filter-list.json` -// - Load and patch user's `remoteBlacklists` -// - Load and patch cached filter lists -// - Load and patch compiled filter lists -// -// Once enough time has passed to safely assume all uBlock Origin -// installations have been converted to the new stock filter lists, this code -// can be removed. - -µBlock.patchFilterLists = function(filterLists) { - var modified = false; - var oldListKey, newListKey, listEntry; - for ( var listKey in filterLists ) { - if ( filterLists.hasOwnProperty(listKey) === false ) { - continue; - } - oldListKey = listKey; - if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - oldListKey = 'assets/thirdparties/' + listKey; - if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - continue; - } +µBlock.assetObserver = function(topic, details) { + // Do not update filter list if not in use. + if ( topic === 'before-asset-updated' ) { + if ( + this.availableFilterLists.hasOwnProperty(details.assetKey) && + this.availableFilterLists[details.assetKey].off === true + ) { + return false; } - newListKey = this.oldListToNewListMap[oldListKey]; - // https://github.com/gorhill/uBlock/issues/668 - // https://github.com/gorhill/uBlock/issues/669 - // Beware: an entry for the new list key may already exists. If it is - // the case, leave it as is. - if ( newListKey !== '' && filterLists.hasOwnProperty(newListKey) === false ) { - listEntry = filterLists[listKey]; - listEntry.homeURL = undefined; - filterLists[newListKey] = listEntry; - } - delete filterLists[listKey]; - modified = true; + return; } - return modified; -}; - -µBlock.loadAndPatchStockFilterLists = function(callback) { - var onStockListsLoaded = function(details) { - var µb = µBlock; - var stockLists; - try { - stockLists = JSON.parse(details.content); - } catch (e) { - stockLists = {}; - } - // Migrate assets affected by the change to their new name. - var reExternalURL = /^https?:\/\//; - var newListKey; - for ( var oldListKey in stockLists ) { - if ( stockLists.hasOwnProperty(oldListKey) === false ) { - continue; - } - // https://github.com/gorhill/uBlock/issues/708 - // Support migrating external stock filter lists as well. - if ( reExternalURL.test(oldListKey) === false ) { - oldListKey = 'assets/thirdparties/' + oldListKey; + // Compile the list while we have the raw version in memory + if ( topic === 'after-asset-updated' ) { + var cached = typeof details.content === 'string' && details.content !== ''; + if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) { + if ( cached ) { + if ( this.availableFilterLists[details.assetKey].off !== true ) { + this.extractFilterListMetadata( + details.assetKey, + details.content + ); + this.assets.put( + 'compiled/' + details.assetKey, + this.compileFilters(details.content) + ); + } + } else { + this.removeCompiledFilterList(details.assetKey); } - if ( µb.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) { - continue; + } else if ( details.assetKey === this.pslAssetKey ) { + if ( cached ) { + this.compilePublicSuffixList(details.content); } - newListKey = µb.oldListToNewListMap[oldListKey]; - if ( newListKey === '' ) { - continue; + } else if ( details.assetKey === 'ublock-resources' ) { + if ( cached ) { + this.redirectEngine.resourcesFromString(details.content); } - // Rename cached asset to preserve content -- so it does not - // need to be fetched from remote server. - µb.assets.rename(oldListKey, newListKey); - µb.assets.purge(µb.getCompiledFilterListPath(oldListKey)); } - µb.patchFilterLists(stockLists); - - // Stock lists information cascades into - // - In-memory user's selected filter lists, so we need to patch this. - µb.patchFilterLists(µb.remoteBlacklists); - - // Stock lists information cascades into - // - In-storage user's selected filter lists, so we need to patch this. - vAPI.storage.get('remoteBlacklists', function(bin) { - var userLists = bin.remoteBlacklists || {}; - if ( µb.patchFilterLists(userLists) ) { - µb.keyvalSetOne('remoteBlacklists', userLists); - } - details.content = JSON.stringify(stockLists); - callback(details); + vAPI.messaging.broadcast({ + what: 'assetUpdated', + key: details.assetKey, + cached: cached + }); - }; + return; + } - this.assets.get('assets/ublock/filter-lists.json', onStockListsLoaded); + // Reload all filter lists if needed. + if ( topic === 'after-assets-updated' ) { + if ( details.assetKeys.length !== 0 ) { + this.loadFilterLists(); + } + if ( this.userSettings.autoUpdate ) { + this.scheduleAssetUpdater(this.hiddenSettings.assetAutoUpdatePeriod * 3600000 || 25200000); + } else { + this.scheduleAssetUpdater(0); + } + return; + } }; diff --git a/src/js/ublock.js b/src/js/ublock.js index 528edb534897f..f765fbd74d4b8 100644 --- a/src/js/ublock.js +++ b/src/js/ublock.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -317,6 +317,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/, // Pre-change switch ( name ) { + case 'externalLists': + this.changeExternalFilterLists(us.externalLists, value); + break; case 'largeMediaSize': if ( typeof value !== 'number' ) { value = parseInt(value, 10) || 0; @@ -340,6 +343,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/, us.dynamicFilteringEnabled = true; } break; + case 'autoUpdate': + this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0); + break; case 'collapseBlocked': if ( value === false ) { this.cosmeticFilteringEngine.removeFromSelectorCache('*', 'net'); diff --git a/tools/make-assets.sh b/tools/make-assets.sh index fade452dcbafe..cebea3f454e8e 100755 --- a/tools/make-assets.sh +++ b/tools/make-assets.sh @@ -24,8 +24,6 @@ cp -R ../uAssets/thirdparties/www.malwaredomainlist.com $DES/thirdparti mkdir $DES/ublock cp -R ../uAssets/filters/* $DES/ublock/ -cp -R ./assets/ublock/filter-lists.json $DES/ublock/ - -cp ../uAssets/checksums/ublock0.txt $DES/checksums.txt +cp -R ./assets/assets.json $DES/ echo "done." diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh index 8d1f58911131a..eda30f2991edf 100755 --- a/tools/make-chromium.sh +++ b/tools/make-chromium.sh @@ -5,7 +5,11 @@ echo "*** uBlock0.chromium: Creating web store package" echo "*** uBlock0.chromium: Copying files" -DES=dist/build/uBlock0.chromium +if [ "$1" = experimental ]; then + DES=dist/build/experimental/uBlock0.chromium +else + DES=dist/build/uBlock0.chromium +fi rm -rf $DES mkdir -p $DES From ff64a8340cfd2d82ded628b7f904ca539e786025 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 13:35:10 -0500 Subject: [PATCH 05/45] code review: only built-in assets are candidates for removal when updating assets.json --- src/js/assets.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index f4daf8f28f7dc..42211c33a9fe0 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -352,9 +352,12 @@ var updateAssetSourceRegistry = function(json) { getAssetSourceRegistry(function(oldDict) { var assetKey; - // Remove obsolete entries + // Remove obsolete entries (only those which were built-in). for ( assetKey in oldDict ) { - if ( newDict[assetKey] === undefined ) { + if ( + newDict[assetKey] === undefined && + oldDict[assetKey].submitter === undefined + ) { unregisterAssetSource(assetKey); } } From 00b19515cf310c17eb68361d6db9b33a37781ac9 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 13:36:24 -0500 Subject: [PATCH 06/45] minor change --- tools/make-assets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make-assets.sh b/tools/make-assets.sh index cebea3f454e8e..ec3c5b38daca2 100755 --- a/tools/make-assets.sh +++ b/tools/make-assets.sh @@ -14,6 +14,7 @@ fi rm -rf $DES mkdir $DES +cp ./assets/assets.json $DES/ mkdir $DES/thirdparties cp -R ../uAssets/thirdparties/easylist-downloads.adblockplus.org $DES/thirdparties/ @@ -24,6 +25,5 @@ cp -R ../uAssets/thirdparties/www.malwaredomainlist.com $DES/thirdparti mkdir $DES/ublock cp -R ../uAssets/filters/* $DES/ublock/ -cp -R ./assets/assets.json $DES/ echo "done." From cc00dd3adcf0020b93419d5b76216ffa8d9fea09 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 13:47:21 -0500 Subject: [PATCH 07/45] new revision for dev build --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index e63d49aad9fb8..242e7740320d9 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.9", + "version": "1.10.5.10", "default_locale": "en", "description": "__MSG_extShortDesc__", From f4d2d6c8919f2046333436cc64a6227c6ffa4b4d Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 17:59:49 -0500 Subject: [PATCH 08/45] forgot to adjust alises after modifying assts.json --- src/js/assets.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index 42211c33a9fe0..b701316bb8734 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -180,9 +180,9 @@ api.listKeyAliases = { "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake", "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0", "http://margevicius.lt/easylistlithuania.txt": "LTU-0", - "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-0", - "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-1", - "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-2", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-0", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-1", + "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-2", "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0", "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0", @@ -356,7 +356,7 @@ var updateAssetSourceRegistry = function(json) { for ( assetKey in oldDict ) { if ( newDict[assetKey] === undefined && - oldDict[assetKey].submitter === undefined + newDict[assetKey].submitter === oldDict[assetKey].submitter ) { unregisterAssetSource(assetKey); } From 726f0d6e1f24128d5b11e4687bff4d9ad6f7b62a Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 18 Jan 2017 18:22:33 -0500 Subject: [PATCH 09/45] remove stray change mistakenly added to last commit --- src/js/assets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/assets.js b/src/js/assets.js index b701316bb8734..28acd991d7d40 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -356,7 +356,7 @@ var updateAssetSourceRegistry = function(json) { for ( assetKey in oldDict ) { if ( newDict[assetKey] === undefined && - newDict[assetKey].submitter === oldDict[assetKey].submitter + oldDict[assetKey].submitter === undefined ) { unregisterAssetSource(assetKey); } From 82155c09b63a364c6a214bf338c5e531ce3dabec Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 19 Jan 2017 08:35:08 -0500 Subject: [PATCH 10/45] fix #2323 --- src/js/background.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/js/background.js b/src/js/background.js index 791e56be94d6f..08c56ba508f10 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -108,8 +108,8 @@ return { // read-only systemSettings: { - compiledMagic: 'zelhzxrhkfjr', - selfieMagic: 'zelhzxrhkfjr' + compiledMagic: 'fxtcjjhbhyiw', + selfieMagic: 'fxtcjjhbhyiw' }, restoreBackupSettings: { From f2f7375e00fa5ca181cb20377b509abbc2ade17c Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 19 Jan 2017 08:35:37 -0500 Subject: [PATCH 11/45] new revision for dev build --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 242e7740320d9..ad12c746bb6e8 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.10", + "version": "1.10.5.11", "default_locale": "en", "description": "__MSG_extShortDesc__", From 29c7ba6a1a09a5cabb8d17ed22c9da2504821090 Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 19 Jan 2017 14:03:08 -0500 Subject: [PATCH 12/45] fix occasional sticky spinner --- src/js/3p-filters.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 4b7dfa1b9dfea..165dedcbfdc12 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -139,6 +139,7 @@ var renderFilterLists = function() { lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) ); } + li.classList.remove('updating'); li.classList.remove('discard'); return li; }; From 71a1e8c6940e974f2b6785138ef6d46a090ca461 Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 20 Jan 2017 08:40:19 -0500 Subject: [PATCH 13/45] code review of 3rd-party filters pane code --- src/js/3p-filters.js | 177 +++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 98 deletions(-) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 165dedcbfdc12..1e3ab20dc323a 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -30,9 +30,7 @@ /******************************************************************************/ var listDetails = {}; -var parseCosmeticFilters = true; -var ignoreGenericCosmeticFilters = false; -var selectedListsHashBefore = ''; +var filteringSettingsHash = ''; var externalLists = ''; /******************************************************************************/ @@ -61,7 +59,7 @@ var renderNumber = function(value) { /******************************************************************************/ -var renderFilterLists = function() { +var renderFilterLists = function(first) { var listGroupTemplate = uDom('#templates .groupEntry'), listEntryTemplate = uDom('#templates .listEntry'), listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'), @@ -72,40 +70,39 @@ var renderFilterLists = function() { var listNameFromListKey = function(listKey) { var list = listDetails.current[listKey] || listDetails.available[listKey]; var listTitle = list ? list.title : ''; - if ( listTitle === '' ) { - return listKey; - } + if ( listTitle === '' ) { return listKey; } return listTitle; }; var liFromListEntry = function(listKey, li) { - var entry = listDetails.available[listKey]; - li = li ? li : listEntryTemplate.clone().nodeAt(0); - li.setAttribute('data-listkey', listKey); - var elem = li.querySelector('input[type="checkbox"]'); - if ( entry.off !== true ) { - elem.setAttribute('checked', ''); - } else { - elem.removeAttribute('checked'); - } - elem = li.querySelector('a:nth-of-type(1)'); - elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); - elem.setAttribute('type', 'text/html'); - elem.textContent = listNameFromListKey(listKey) + '\u200E'; - elem = li.querySelector('a:nth-of-type(2)'); - if ( entry.instructionURL ) { - elem.setAttribute('href', entry.instructionURL); - elem.style.setProperty('display', ''); - } else { - elem.style.setProperty('display', 'none'); + var entry = listDetails.available[listKey], + elem; + if ( !li ) { + li = listEntryTemplate.clone().nodeAt(0); } - elem = li.querySelector('a:nth-of-type(3)'); - if ( entry.supportName ) { - elem.setAttribute('href', entry.supportURL); - elem.textContent = '(' + entry.supportName + ')'; - elem.style.setProperty('display', ''); - } else { - elem.style.setProperty('display', 'none'); + if ( li.getAttribute('data-listkey') !== listKey ) { + li.setAttribute('data-listkey', listKey); + elem = li.querySelector('input[type="checkbox"]'); + elem.checked = entry.off !== true; + elem = li.querySelector('a:nth-of-type(1)'); + elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); + elem.setAttribute('type', 'text/html'); + elem.textContent = listNameFromListKey(listKey) + '\u200E'; + elem = li.querySelector('a:nth-of-type(2)'); + if ( entry.instructionURL ) { + elem.setAttribute('href', entry.instructionURL); + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); + } + elem = li.querySelector('a:nth-of-type(3)'); + if ( entry.supportName ) { + elem.setAttribute('href', entry.supportURL); + elem.textContent = '(' + entry.supportName + ')'; + elem.style.setProperty('display', ''); + } else { + elem.style.setProperty('display', 'none'); + } } elem = li.querySelector('span.counts'); var text = listStatsTemplate @@ -124,15 +121,9 @@ var renderFilterLists = function() { typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 ); // Badge for update status - li.classList.toggle( - 'obsolete', - entry.off !== true && asset.obsolete === true - ); + li.classList.toggle('obsolete', entry.off !== true && asset.obsolete === true); // Badge for cache status - li.classList.toggle( - 'cached', - asset.cached === true && asset.writeTime > 0 - ); + li.classList.toggle('cached', asset.cached === true && asset.writeTime > 0); if ( asset.cached ) { li.querySelector('.status.purge').setAttribute( 'title', @@ -145,9 +136,7 @@ var renderFilterLists = function() { }; var listEntryCountFromGroup = function(listKeys) { - if ( Array.isArray(listKeys) === false ) { - return ''; - } + if ( Array.isArray(listKeys) === false ) { return ''; } var count = 0; var i = listKeys.length; while ( i-- ) { @@ -204,8 +193,6 @@ var renderFilterLists = function() { var onListsReceived = function(details) { // Before all, set context vars listDetails = details; - parseCosmeticFilters = details.parseCosmeticFilters; - ignoreGenericCosmeticFilters = details.ignoreGenericCosmeticFilters; // Incremental rendering: this will allow us to easily discard unused // DOM list entries. @@ -248,23 +235,19 @@ var renderFilterLists = function() { uDom('#lists .listEntries .listEntry.discard').remove(); uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); - uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); - uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); uDom('#listsOfBlockedHostsPrompt').text( vAPI.i18n('3pListsOfBlockedHostsPrompt') .replace('{{netFilterCount}}', renderNumber(details.netFilterCount)) .replace('{{cosmeticFilterCount}}', renderNumber(details.cosmeticFilterCount)) ); - // Compute a hash of the lists currently enabled in memory. - var selectedListsBefore = []; - for ( var key in listDetails.current ) { - if ( listDetails.current[key].off !== true ) { - selectedListsBefore.push(key); - } + // Compute a hash of the settings so that we can keep track of changes + // affecting the loading of filter lists. + if ( first ) { + uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true); + uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true); + filteringSettingsHash = hashFromCurrentFromSettings(); } - selectedListsHashBefore = selectedListsBefore.sort().join(); - renderWidgets(); }; @@ -276,9 +259,9 @@ var renderFilterLists = function() { // This is to give a visual hint that the selection of blacklists has changed. var renderWidgets = function() { - uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); + uDom('#buttonApply').toggleClass('disabled', filteringSettingsHash === hashFromCurrentFromSettings()); uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); - uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete > input[type="checkbox"]:checked') === null); }; /******************************************************************************/ @@ -291,30 +274,34 @@ var updateAssetStatus = function(details) { renderWidgets(); }; -/******************************************************************************/ +/******************************************************************************* -// Return whether selection of lists changed. + Compute a hash from all the settings affecting how filter lists are loaded + in memory. -var listsSelectionChanged = function() { - if ( - listDetails.parseCosmeticFilters !== parseCosmeticFilters || - listDetails.parseCosmeticFilters && - listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters - ) { - return true; - } - var selectedListsAfter = [], - listEntries = uDom('#lists .listEntry[data-listkey] > input[type="checkbox"]:checked'); - for ( var i = 0, n = listEntries.length; i < n; i++ ) { - selectedListsAfter.push(listEntries.at(i).ancestors('.listEntry[data-listkey]').attr('data-listkey')); - } +**/ - return selectedListsHashBefore !== selectedListsAfter.sort().join(); +var hashFromCurrentFromSettings = function() { + var hash = [ + document.getElementById('parseCosmeticFilters').checked, + document.getElementById('ignoreGenericCosmeticFilters').checked + ]; + var listHash = [], + listEntries = document.querySelectorAll('#lists .listEntry[data-listkey]'), + liEntry, + i = listEntries.length; + while ( i-- ) { + liEntry = listEntries[i]; + if ( liEntry.querySelector('input[type="checkbox"]:checked') !== null ) { + listHash.push(liEntry.getAttribute('data-listkey')); + } + } + return hash.concat(listHash.sort()).join(); }; /******************************************************************************/ -var onListCheckboxChanged = function() { +var onFilteringSettingsChanged = function() { renderWidgets(); }; @@ -352,33 +339,33 @@ var selectFilterLists = function(callback) { messaging.send('dashboard', { what: 'userSettings', name: 'parseAllABPHideFilters', - value: listDetails.parseCosmeticFilters + value: document.getElementById('parseCosmeticFilters').checked }); messaging.send('dashboard', { what: 'userSettings', name: 'ignoreGenericCosmeticFilters', - value: listDetails.ignoreGenericCosmeticFilters + value: document.getElementById('ignoreGenericCosmeticFilters').checked }); // Filter lists var listKeys = [], - liEntries = uDom('#lists .listEntry'), liEntry, - i = liEntries.length; + liEntries = document.querySelectorAll('#lists .listEntry[data-listkey]'), + i = liEntries.length, + liEntry; while ( i-- ) { - liEntry = liEntries.at(i); - if ( liEntry.descendants('input').first().prop('checked') ) { - listKeys.push(liEntry.attr('data-listkey')); + liEntry = liEntries[i]; + if ( liEntry.querySelector('input[type="checkbox"]:checked') !== null ) { + listKeys.push(liEntry.getAttribute('data-listkey')); } } messaging.send( 'dashboard', - { - what: 'selectFilterLists', - keys: listKeys - }, + { what: 'selectFilterLists', keys: listKeys }, callback ); + + filteringSettingsHash = hashFromCurrentFromSettings(); }; /******************************************************************************/ @@ -389,6 +376,7 @@ var buttonApplyHandler = function() { messaging.send('dashboard', { what: 'reloadAllFilters' }); }; selectFilterLists(onSelectionDone); + renderWidgets(); }; /******************************************************************************/ @@ -399,6 +387,7 @@ var buttonUpdateHandler = function() { messaging.send('dashboard', { what: 'forceUpdateAssets' }); }; selectFilterLists(onSelectionDone); + renderWidgets(); }; /******************************************************************************/ @@ -430,14 +419,6 @@ var autoUpdateCheckboxChanged = function() { /******************************************************************************/ -var cosmeticSwitchChanged = function() { - listDetails.parseCosmeticFilters = uDom.nodeFromId('parseCosmeticFilters').checked; - listDetails.ignoreGenericCosmeticFilters = uDom.nodeFromId('ignoreGenericCosmeticFilters').checked; - renderWidgets(); -}; - -/******************************************************************************/ - var renderExternalLists = function() { var onReceived = function(details) { uDom('#externalLists').val(details); @@ -553,18 +534,18 @@ self.cloud.onPull = fromCloudData; /******************************************************************************/ uDom('#autoUpdate').on('change', autoUpdateCheckboxChanged); -uDom('#parseCosmeticFilters').on('change', cosmeticSwitchChanged); -uDom('#ignoreGenericCosmeticFilters').on('change', cosmeticSwitchChanged); +uDom('#parseCosmeticFilters').on('change', onFilteringSettingsChanged); +uDom('#ignoreGenericCosmeticFilters').on('change', onFilteringSettingsChanged); uDom('#buttonApply').on('click', buttonApplyHandler); uDom('#buttonUpdate').on('click', buttonUpdateHandler); uDom('#buttonPurgeAll').on('click', buttonPurgeAllHandler); -uDom('#lists').on('change', '.listEntry > input', onListCheckboxChanged); +uDom('#lists').on('change', '.listEntry > input', onFilteringSettingsChanged); uDom('#lists').on('click', 'span.purge', onPurgeClicked); uDom('#externalLists').on('input', externalListsChangeHandler); uDom('#externalListsApply').on('click', externalListsApplyHandler); uDom('#lists').on('click', '.groupEntry > span', groupEntryClickHandler); -renderFilterLists(); +renderFilterLists(true); renderExternalLists(); /******************************************************************************/ From 4a8dd58f2d7b5b3d9014f09744b773844c3eaf5f Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 20 Jan 2017 11:24:18 -0500 Subject: [PATCH 14/45] new revision for dev build --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index ad12c746bb6e8..8712097d8fe03 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.11", + "version": "1.10.5.12", "default_locale": "en", "description": "__MSG_extShortDesc__", From 8f46662a24f2a73725e36368a19715e2b2a173ab Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 20 Jan 2017 12:48:42 -0500 Subject: [PATCH 15/45] added POL list compatible with uBO-specific syntax --- assets/assets.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/assets/assets.json b/assets/assets.json index 883b87ba01ee4..7b361c3cf52d3 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -512,6 +512,15 @@ "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt", "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" }, + "POL-1": { + "content": "filters", + "group": "regions", + "off": true, + "title": "POL: polskie filtry do uBlocka uzupelnienie", + "lang": "pl", + "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock_ublock.txt", + "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/" + }, "RUS-0": { "content": "filters", "group": "regions", From 6e48c74e4e06167f3ea806d28dfaaf3c2046418c Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 20 Jan 2017 15:17:11 -0500 Subject: [PATCH 16/45] code review: auto-select new built-in asset if it matches locale (https://github.com/uBlockOrigin/uAssets/issues/268#issuecomment-274146120) --- src/js/3p-filters.js | 10 ++++++--- src/js/assets.js | 8 +++++++- src/js/storage.js | 48 ++++++++++++++++++++++++++++++++------------ 3 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 1e3ab20dc323a..bd035874c7393 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -29,9 +29,9 @@ /******************************************************************************/ -var listDetails = {}; -var filteringSettingsHash = ''; -var externalLists = ''; +var listDetails = {}, + filteringSettingsHash = '', + externalLists = ''; /******************************************************************************/ @@ -41,6 +41,10 @@ var onMessage = function(msg) { updateAssetStatus(msg); break; case 'staticFilteringDataChanged': + filteringSettingsHash = [ + msg.parseCosmeticFilters, + msg.ignoreGenericCosmeticFilters + ].concat(msg.listKeys.sort()).join(); renderFilterLists(); break; default: diff --git a/src/js/assets.js b/src/js/assets.js index 28acd991d7d40..20f38aded8a22 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -361,8 +361,14 @@ var updateAssetSourceRegistry = function(json) { unregisterAssetSource(assetKey); } } - // Add/update existing entries + // Add/update existing entries. Notify of new asset sources. for ( assetKey in newDict ) { + if ( oldDict[assetKey] === undefined ) { + fireNotification( + 'builtin-asset-source-added', + { assetKey: assetKey, entry: newDict[assetKey] } + ); + } registerAssetSource(assetKey, newDict[assetKey]); } saveAssetSourceRegistry(); diff --git a/src/js/storage.js b/src/js/storage.js index c587919487353..6b07118532164 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -193,9 +193,10 @@ µBlock.saveSelectedFilterLists = function(listKeys, append) { var µb = this; var save = function(keys) { + var uniqueKeys = µb.setToArray(new Set(keys)); var bin = { - selectedFilterLists: keys, - remoteBlacklists: µb.oldDataFromNewListKeys(keys) + selectedFilterLists: uniqueKeys, + remoteBlacklists: µb.oldDataFromNewListKeys(uniqueKeys) }; vAPI.storage.set(bin); }; @@ -516,8 +517,9 @@ //quickProfiler.start('µBlock.loadFilterLists()'); - var µb = this; - var filterlistsCount = 0; + var µb = this, + filterlistsCount = 0, + loadedListKeys = []; if ( typeof callback !== 'function' ) { callback = this.noopFunc; @@ -531,7 +533,12 @@ //quickProfiler.stop(0); - vAPI.messaging.broadcast({ what: 'staticFilteringDataChanged' }); + vAPI.messaging.broadcast({ + what: 'staticFilteringDataChanged', + parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters, + ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters, + listKeys: loadedListKeys + }); callback(); @@ -539,17 +546,18 @@ µb.loadingFilterLists = false; }; - var applyCompiledFilters = function(path, compiled) { - var snfe = µb.staticNetFilteringEngine; - var cfe = µb.cosmeticFilteringEngine; - var acceptedCount = snfe.acceptedCount + cfe.acceptedCount; - var discardedCount = snfe.discardedCount + cfe.discardedCount; - µb.applyCompiledFilters(compiled, path === µb.userFiltersPath); - if ( µb.availableFilterLists.hasOwnProperty(path) ) { - var entry = µb.availableFilterLists[path]; + var applyCompiledFilters = function(assetKey, compiled) { + var snfe = µb.staticNetFilteringEngine, + cfe = µb.cosmeticFilteringEngine, + acceptedCount = snfe.acceptedCount + cfe.acceptedCount, + discardedCount = snfe.discardedCount + cfe.discardedCount; + µb.applyCompiledFilters(compiled, assetKey === µb.userFiltersPath); + if ( µb.availableFilterLists.hasOwnProperty(assetKey) ) { + var entry = µb.availableFilterLists[assetKey]; entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount; entry.entryUsedCount = entry.entryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount); } + loadedListKeys.push(assetKey); }; var onCompiledListLoaded = function(details) { @@ -1053,4 +1061,18 @@ } return; } + + // New asset source became available, if it's a filter list, should we + // auto-select it? + if ( topic === 'builtin-asset-source-added' ) { + if ( details.entry.content === 'filters' ) { + if ( + details.entry.off !== true || + self.navigator.language.startsWith(details.entry.lang) + ) { + this.saveSelectedFilterLists([ details.assetKey ], true); + } + } + return; + } }; From 20ebe6d18f54ba30bc5c554cb267812889e17a6c Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 20 Jan 2017 15:31:47 -0500 Subject: [PATCH 17/45] new revision for dev build --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 8712097d8fe03..1293bffe54c9e 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.12", + "version": "1.10.5.13", "default_locale": "en", "description": "__MSG_extShortDesc__", From 9309df4196cbe1a30b7b8f7cc88dd67aac3c151c Mon Sep 17 00:00:00 2001 From: gorhill Date: Sun, 22 Jan 2017 16:05:16 -0500 Subject: [PATCH 18/45] 3rd-party filters pane revisited --- assets/assets.json | 11 +- src/3p-filters.html | 37 +++--- src/css/3p-filters.css | 121 +++++++++++------- src/js/3p-filters.js | 217 ++++++++++++++++++-------------- src/js/assets.js | 10 +- src/js/background.js | 1 + src/js/messaging.js | 22 ++-- src/js/scriptlets/subscriber.js | 40 +----- src/js/storage.js | 204 ++++++++++++++++++++---------- src/js/ublock.js | 3 - 10 files changed, 389 insertions(+), 277 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index 7b361c3cf52d3..24bc1f4738996 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -30,7 +30,8 @@ "contentURL": [ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt", "assets/ublock/filters.txt" - ] + ], + "supportURL": "https://github.com/uBlockOrigin/uAssets/issues" }, "ublock-badware": { "content": "filters", @@ -196,7 +197,11 @@ "group": "social", "off": true, "title": "Anti-ThirdpartySocial (see warning inside list)", - "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt", + "contentURL": [ + "https://easylist.to/easylist/fanboy-social.txt", + "https://fanboy.co.nz/fanboy-antifacebook.txt", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt" + ], "supportURL": "https://forums.lanik.us/" }, "fanboy-annoyance": { @@ -206,6 +211,7 @@ "title": "Fanboy’s Annoyance List", "contentURL": [ "https://easylist.to/easylist/fanboy-annoyance.txt", + "https://fanboy.co.nz/fanboy-annoyance.txt", "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt" ], "supportURL": "https://forums.lanik.us/" @@ -217,6 +223,7 @@ "title": "Fanboy’s Social Blocking List", "contentURL": [ "https://easylist.to/easylist/fanboy-social.txt", + "https://fanboy.co.nz/fanboy-social.txt", "https://easylist-downloads.adblockplus.org/fanboy-social.txt" ], "supportURL": "https://forums.lanik.us/" diff --git a/src/3p-filters.html b/src/3p-filters.html index 48086f71fe271..d35910399759f 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -21,24 +21,19 @@
  • -
      -
    • -
      -
    -

  • +
  • +
    +

      -

      - - -

      -

      - -

      -
      +

      + + + +

      • diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index b3687bae1f4bd..07a549af91dc0 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -133,9 +133,6 @@ li.listEntry span.status { li.listEntry span.status:hover { opacity: 1; } -li.listEntry span.status.fa { - /* font-size: 110%; */ - } li.listEntry span.unsecure { color: darkred; } @@ -151,20 +148,19 @@ li.listEntry.failed span.failed { li.listEntry span.cache { cursor: pointer; } -li.listEntry.cached:not(.updating):not(.obsolete) > input[type="checkbox"]:checked ~ span.cache { +li.listEntry.cached:not(.obsolete) > input[type="checkbox"]:checked ~ span.cache { display: inline-block; } li.listEntry span.obsolete { color: hsl(36, 100%, 40%); } -li.listEntry.obsolete:not(.updating) > input[type="checkbox"]:checked ~ span.obsolete { +body:not(.updating) li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { display: inline-block; } li.listEntry span.updating { - border: none; - padding: 0; + transform-origin: 50% 46%; } -li.listEntry.updating span.updating { +body.updating li.listEntry.obsolete span.updating { animation: spin 2s linear infinite; display: inline-block; } diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 1b52d5276893b..f478da3ad9b6a 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -40,6 +40,9 @@ var onMessage = function(msg) { case 'assetUpdated': updateAssetStatus(msg); break; + case 'assetsUpdated': + document.body.classList.remove('updating'); + break; case 'staticFilteringDataChanged': renderFilterLists(); break; @@ -139,7 +142,6 @@ var renderFilterLists = function(soft) { lastUpdateTemplateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) ); } - li.classList.remove('updating'); li.classList.remove('discard'); return li; }; @@ -264,12 +266,10 @@ var renderFilterLists = function(soft) { /******************************************************************************/ -// This is to give a visual hint that the selection of blacklists has changed. - var renderWidgets = function() { uDom('#buttonApply').toggleClass('disabled', filteringSettingsHash === hashFromCurrentFromSettings()); uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); - uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete:not(.updating) > input[type="checkbox"]:checked') === null); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('body:not(.updating) #lists .listEntry.obsolete > input[type="checkbox"]:checked') === null); }; /******************************************************************************/ @@ -280,7 +280,6 @@ var updateAssetStatus = function(details) { li.classList.toggle('failed', !!details.failed); li.classList.toggle('obsolete', !details.cached); li.classList.toggle('cached', !!details.cached); - li.classList.remove('updating'); if ( details.cached ) { li.querySelector('.status.cache').setAttribute( 'title', @@ -429,9 +428,7 @@ var buttonApplyHandler = function() { var buttonUpdateHandler = function() { var onSelectionDone = function() { - uDom('#lists .listEntry.obsolete > input[type="checkbox"]:checked') - .ancestors('.listEntry[data-listkey]') - .addClass('updating'); + document.body.classList.add('updating'); messaging.send('dashboard', { what: 'forceUpdateAssets' }); renderWidgets(); }; diff --git a/src/js/assets.js b/src/js/assets.js index 9a29812d7a0eb..bf15d395c8f81 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -598,17 +598,25 @@ var assetCacheRemove = function(pattern, callback) { getAssetCacheRegistry(onReady); }; -var assetCacheMarkAsDirty = function(pattern, callback) { +var assetCacheMarkAsDirty = function(pattern, exclude, callback) { var onReady = function() { var cacheDict = assetCacheRegistry, cacheEntry, mustSave = false; for ( var assetKey in cacheDict ) { - if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { - continue; + if ( pattern instanceof RegExp ) { + if ( pattern.test(assetKey) === false ) { continue; } + } else if ( typeof pattern === 'string' ) { + if ( assetKey !== pattern ) { continue; } + } else if ( Array.isArray(pattern) ) { + if ( pattern.indexOf(assetKey) === -1 ) { continue; } } - if ( typeof pattern === 'string' && assetKey !== pattern ) { - continue; + if ( exclude instanceof RegExp ) { + if ( exclude.test(assetKey) ) { continue; } + } else if ( typeof exclude === 'string' ) { + if ( assetKey === exclude ) { continue; } + } else if ( Array.isArray(exclude) ) { + if ( exclude.indexOf(assetKey) !== -1 ) { continue; } } cacheEntry = cacheDict[assetKey]; if ( !cacheEntry.writeTime ) { continue; } @@ -623,7 +631,10 @@ var assetCacheMarkAsDirty = function(pattern, callback) { callback(); } }; - + if ( typeof exclude === 'function' ) { + callback = exclude; + exclude = undefined; + } getAssetCacheRegistry(onReady); }; @@ -887,9 +898,7 @@ api.metadata = function(callback) { /******************************************************************************/ -api.purge = function(pattern, callback) { - assetCacheMarkAsDirty(pattern, callback); -}; +api.purge = assetCacheMarkAsDirty; api.remove = function(pattern, callback) { assetCacheRemove(pattern, callback); diff --git a/src/js/messaging.js b/src/js/messaging.js index d5c780b6954ed..cab4802fe6737 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -979,8 +979,7 @@ var onMessage = function(request, sender, callback) { if ( request.hard ) { µb.assets.remove(/./); } else { - µb.assets.remove(/compiled\//); - µb.assets.purge(/./); + µb.assets.purge(/./, 'public_suffix_list.dat'); } break; diff --git a/src/js/storage.js b/src/js/storage.js index 782d2da24f416..7d394f52d7589 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1131,6 +1131,10 @@ } else { this.scheduleAssetUpdater(0); } + vAPI.messaging.broadcast({ + what: 'assetsUpdated', + assetKeys: details.assetKeys + }); return; } From 96df129ddb2571b06afed288c4715bee02b00a61 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 23 Jan 2017 10:13:07 -0500 Subject: [PATCH 23/45] code reivew: do not cache assets fetched for viewing purpose --- src/js/assets.js | 14 +++++++++++--- src/js/messaging.js | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index bf15d395c8f81..ba83f5d141a16 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -29,7 +29,8 @@ var reIsExternalPath = /^(?:[a-z-]+):\/\//, reIsUserAsset = /^user-/, - errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); + errorCantConnectTo = vAPI.i18n('errorCantConnectTo'), + noopfunc = function(){}; var api = { }; @@ -716,7 +717,14 @@ var saveUserAsset = function(assetKey, content, callback) { /******************************************************************************/ -api.get = function(assetKey, callback) { +api.get = function(assetKey, options, callback) { + if ( typeof options === 'function' ) { + callback = options; + options = {}; + } else if ( typeof callback !== 'function' ) { + callback = noopfunc; + } + if ( assetKey === µBlock.userFiltersPath ) { readUserAsset(assetKey, callback); return; @@ -755,7 +763,7 @@ api.get = function(assetKey, callback) { onContentNotLoaded(); return; } - if ( reIsExternalPath.test(contentURL) ) { + if ( reIsExternalPath.test(contentURL) && options.dontCache !== true ) { assetCacheWrite(assetKey, { content: this.responseText, url: contentURL diff --git a/src/js/messaging.js b/src/js/messaging.js index cab4802fe6737..201ddbf2878fc 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -57,7 +57,7 @@ var onMessage = function(request, sender, callback) { switch ( request.what ) { case 'getAssetContent': // https://github.com/chrisaljoudi/uBlock/issues/417 - µb.assets.get(request.url, callback); + µb.assets.get(request.url, { dontCache: true }, callback); return; case 'listsFromNetFilter': From 975b0d8280091905ed2a50b15e96198428779bc8 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 23 Jan 2017 11:08:01 -0500 Subject: [PATCH 24/45] new version for release candidate --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index f20c12d0e50b5..1f2e054509c3e 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.14", + "version": "1.10.5.100", "default_locale": "en", "description": "__MSG_extShortDesc__", From 8ab6c13167762b8d4572f457e48f3d0dc944f643 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 23 Jan 2017 14:31:43 -0500 Subject: [PATCH 25/45] code review: non-enabled lists must not be shown as updating --- src/css/3p-filters.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index 07a549af91dc0..a38b1ae60dff8 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -160,7 +160,7 @@ body:not(.updating) li.listEntry.obsolete > input[type="checkbox"]:checked ~ spa li.listEntry span.updating { transform-origin: 50% 46%; } -body.updating li.listEntry.obsolete span.updating { +body.updating li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.updating { animation: spin 2s linear infinite; display: inline-block; } From 97db7ba13e66fae7d4f2a61df836f20433d1af97 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 23 Jan 2017 17:16:37 -0500 Subject: [PATCH 26/45] fix #2332 --- src/js/3p-filters.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index f478da3ad9b6a..855f89eed835b 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -31,7 +31,8 @@ var listDetails = {}, filteringSettingsHash = '', - lastUpdateTemplateString = vAPI.i18n('3pLastUpdate'); + lastUpdateTemplateString = vAPI.i18n('3pLastUpdate'), + reValidExternalList = /[a-z-]+:\/\/\S*\/\S+/; /******************************************************************************/ @@ -111,6 +112,7 @@ var renderFilterLists = function(soft) { } else { li.classList.remove('mustread'); } + li.classList.remove('toRemove'); } // https://github.com/gorhill/uBlock/issues/1429 if ( !soft ) { @@ -314,8 +316,11 @@ var hashFromCurrentFromSettings = function() { listHash.push(liEntry.getAttribute('data-listkey')); } } - hash.push(listHash.sort().join()); - hash.push(document.getElementById('externalLists').value.trim()); + hash.push( + listHash.sort().join(), + reValidExternalList.test(document.getElementById('externalLists').value), + document.querySelector('#lists .listEntry.toRemove') !== null + ); return hash.join(); }; From 82b58664d4b431156feb63007373ab441198c097 Mon Sep 17 00:00:00 2001 From: gorhill Date: Mon, 23 Jan 2017 17:17:12 -0500 Subject: [PATCH 27/45] new revision for release candidate --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 1f2e054509c3e..6b233209d3043 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.100", + "version": "1.10.5.101", "default_locale": "en", "description": "__MSG_extShortDesc__", From 1a075bc6734eae4017762c4d1311e38e9bd14f52 Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 24 Jan 2017 08:23:52 -0500 Subject: [PATCH 28/45] code review: mobile-friendly changes --- src/1p-filters.html | 1 + src/3p-filters.html | 8 ++++---- src/about.html | 1 + src/css/3p-filters.css | 1 - src/css/dashboard-common.css | 5 +++++ src/css/dashboard.css | 5 +++++ src/dyna-rules.html | 1 + src/js/3p-filters.js | 2 +- src/whitelist.html | 1 + 9 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/1p-filters.html b/src/1p-filters.html index 37e81610eab3e..b1146a09d1d52 100644 --- a/src/1p-filters.html +++ b/src/1p-filters.html @@ -2,6 +2,7 @@ + uBlock — Your filters diff --git a/src/3p-filters.html b/src/3p-filters.html index 29ccb57365015..823dfff7e3754 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -3,7 +3,7 @@ -uBlock — Ubiquitous rules +uBlock — Filter lists @@ -15,11 +15,11 @@
      -
        -
      •   - +
      • + +
      •  
      • diff --git a/src/about.html b/src/about.html index 855170e966206..842088d956f13 100644 --- a/src/about.html +++ b/src/about.html @@ -2,6 +2,7 @@ + uBlock — About diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index a38b1ae60dff8..a5052ca45d4fa 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -1,7 +1,6 @@ @keyframes spin { 100% { transform: rotate(360deg); -webkit-transform: rotate(360deg); } } - ul { padding: 0; list-style-type: none; diff --git a/src/css/dashboard-common.css b/src/css/dashboard-common.css index dd0a8c19c75cf..bc8f3ddd28e3a 100644 --- a/src/css/dashboard-common.css +++ b/src/css/dashboard-common.css @@ -2,6 +2,11 @@ body { margin: 0; padding: 0 0.5em 0.5em 0.5em; } +@media screen and (max-device-width: 960px) { + body { + zoom: 1.4; + } + } h2, h3 { margin: 1em 0; font-family: sans-serif; diff --git a/src/css/dashboard.css b/src/css/dashboard.css index 1b34a7e7df50a..ea8f0573ecce1 100644 --- a/src/css/dashboard.css +++ b/src/css/dashboard.css @@ -7,6 +7,11 @@ html, body { height: 100%; overflow: hidden; } +@media screen and (max-device-width: 960px) { + #dashboard-nav { + zoom: 1.2; + } + } #dashboard-nav { border: 0; margin: 0; diff --git a/src/dyna-rules.html b/src/dyna-rules.html index cb81f99efc575..fac16dd28b652 100644 --- a/src/dyna-rules.html +++ b/src/dyna-rules.html @@ -2,6 +2,7 @@ + uBlock — Dynamic filtering rules diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index 855f89eed835b..f43cb8bd364c4 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -92,6 +92,7 @@ var renderFilterLists = function(soft) { elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); elem.setAttribute('type', 'text/html'); elem.textContent = listNameFromListKey(listKey) + '\u200E'; + li.classList.remove('toRemove'); if ( entry.supportName ) { li.classList.add('support'); elem = li.querySelector('a.support'); @@ -112,7 +113,6 @@ var renderFilterLists = function(soft) { } else { li.classList.remove('mustread'); } - li.classList.remove('toRemove'); } // https://github.com/gorhill/uBlock/issues/1429 if ( !soft ) { diff --git a/src/whitelist.html b/src/whitelist.html index 67a70ab92dabd..f8a45905292c5 100644 --- a/src/whitelist.html +++ b/src/whitelist.html @@ -2,6 +2,7 @@ + uBlock — Whitelist From 0e11d6e95ea83d48e23422f62cc8799fb6a61330 Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 24 Jan 2017 13:53:04 -0500 Subject: [PATCH 29/45] code review: fix rtl rendering of 3rd-party filters pane --- src/3p-filters.html | 18 +++++++++--------- src/css/3p-filters.css | 8 ++++---- src/js/3p-filters.js | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/3p-filters.html b/src/3p-filters.html index 823dfff7e3754..64931d5693134 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -43,15 +43,15 @@
      • ​​​​​​​​​
      diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css index a5052ca45d4fa..359ab709e5b61 100644 --- a/src/css/3p-filters.css +++ b/src/css/3p-filters.css @@ -23,7 +23,7 @@ body.hideUnused #listsOfBlockedHostsPrompt:before { padding-left: 0.5em; padding-right: 0em; } -body[dir=rtl] #lists { +body[dir="rtl"] #lists { padding-left: 0em; padding-right: 0.5em; } @@ -57,12 +57,12 @@ li.listEntry { line-height: 150%; margin: 0 auto 0 auto; margin-left: 2.5em; - margin-right: 0em; + margin-right: 0; text-indent: -2em; } -body[dir=rtl] li.listEntry { +body[dir="rtl"] li.listEntry { margin-left: 0em; - margin-right: 1em; + margin-right: 2.5em; } li.listEntry > * { margin-right: 0.5em; diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index f43cb8bd364c4..57cccf2f6ae16 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -91,7 +91,7 @@ var renderFilterLists = function(soft) { elem = li.querySelector('a:nth-of-type(1)'); elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); elem.setAttribute('type', 'text/html'); - elem.textContent = listNameFromListKey(listKey) + '\u200E'; + elem.textContent = listNameFromListKey(listKey); li.classList.remove('toRemove'); if ( entry.supportName ) { li.classList.add('support'); From c2a3ff141be9b5d5e85c55d5f54973c76e411518 Mon Sep 17 00:00:00 2001 From: gorhill Date: Tue, 24 Jan 2017 16:58:27 -0500 Subject: [PATCH 30/45] translation work from https://crowdin.com/project/ublock --- dist/description/description-hi.txt | 28 ++++++++++++++-------------- src/_locales/bg/messages.json | 2 +- src/_locales/el/messages.json | 2 +- src/_locales/et/messages.json | 2 +- src/_locales/fi/messages.json | 2 +- src/_locales/hi/messages.json | 2 +- src/_locales/id/messages.json | 2 +- src/_locales/it/messages.json | 2 +- src/_locales/ko/messages.json | 2 +- src/_locales/lv/messages.json | 8 ++++---- src/_locales/pt_BR/messages.json | 2 +- src/_locales/pt_PT/messages.json | 8 ++++---- src/_locales/sr/messages.json | 2 +- src/_locales/vi/messages.json | 2 +- src/_locales/zh_CN/messages.json | 2 +- 15 files changed, 34 insertions(+), 34 deletions(-) diff --git a/dist/description/description-hi.txt b/dist/description/description-hi.txt index 6bb418ca3f6f3..33caf12a383ed 100644 --- a/dist/description/description-hi.txt +++ b/dist/description/description-hi.txt @@ -1,28 +1,28 @@ -An efficient blocker: easy on memory and CPU footprint, and yet can load and enforce thousands more filters than other popular blockers out there. +एक कुशल अवरोधक: स्मृति और सीपीयू पदचिह्न पर आसान है, और अभी तक लोड और हजारों लागू वहाँ से बाहर अन्य लोकप्रिय ब्लॉकर्स और अधिक से अधिक फिल्टर कर सकते हैं। अपनी क्षमता का सचित्र अवलोकन: https://github.com/gorhill/uBlock/wiki/uBlock-vs.-ABP:-efficiency-compared -Usage: The big power button in the popup is to permanently disable/enable uBlock for the current web site. It applies to the current web site only, it is not a global power button. +उपयोग: पॉपअप में बड़ी शक्ति बटन स्थायी रूप से अक्षम / वर्तमान वेब साइट uBlock लिए सक्षम है। यह केवल मौजूदा वेब साइट पर लागू होता है, यह एक वैश्विक शक्ति बटन नहीं है। *** -Flexible, it's more than an "ad blocker": it can also read and create filters from hosts files. +लचीले, यह एक केवल "विज्ञापन अवरोधक" की तुलना से अधिक है: यह भी पढ़ सकता हैं और मेजबान फाइलों से फिल्टर बना सकते हैं। Out of the box, these lists of filters are loaded and enforced: -- EasyList -- Peter Lowe’s Ad server list -- EasyPrivacy -- Malware domains +- आसान सूची +- Peter Lowe's विज्ञापन सर्वर सूची +- आसान गुप्तता +- मैलवेयर डोमेन -More lists are available for you to select if you wish: +यदि आप चाहें तो आप का चयन करने के लिए और अधिक सूची उपलब्ध हैं -- Fanboy’s Enhanced Tracking List -- Dan Pollock’s hosts file -- hpHosts’s Ad and tracking servers -- MVPS HOSTS -- Spam404 -- And many others +- Fanboy's बढ़ी ट्रैकिंग सूची +- Dan Pollock's मेजबान फ़ाइल +- hpHosts's विज्ञापन और ट्रैकिंग सर्वर +- MVPS मेज़बान +-स्पैम404 +- और बहुत सारे Of course, the more filters enabled, the higher the memory footprint. Yet, even after adding Fanboy's two extra lists, hpHosts’s Ad and tracking servers, uBlock still has a lower memory footprint than other very popular blockers out there. diff --git a/src/_locales/bg/messages.json b/src/_locales/bg/messages.json index 782775f42dbc6..9a607eb4f3682 100644 --- a/src/_locales/bg/messages.json +++ b/src/_locales/bg/messages.json @@ -680,7 +680,7 @@ "description":"used as a prompt for the user to provide a custom device name" }, "advancedSettingsWarning":{ - "message":"Внимание! Променяте тези настройки на свой собствен риск.", + "message":"Внимание! Променяте настройките на свой собствен риск.", "description":"A warning to users at the top of 'Advanced settings' page" }, "genericSubmit":{ diff --git a/src/_locales/el/messages.json b/src/_locales/el/messages.json index ad9818a10af18..1cea6f934bc98 100644 --- a/src/_locales/el/messages.json +++ b/src/_locales/el/messages.json @@ -292,7 +292,7 @@ "description":"This will cause uBO to ignore all generic cosmetic filters." }, "3pIgnoreGenericCosmeticFiltersInfo":{ - "message":"

      Generic cosmetic filters are those cosmetic filters which are meant to apply on all web sites.

      Though handled efficiently by uBlock₀, generic cosmetic filters may still contribute measurable memory and CPU overhead on some web pages, especially for large and long-lived ones.

      Enabling this option will eliminate the memory and CPU overhead added to web pages as a result of handling generic cosmetic filters, and also lower the memory footprint of uBlock₀ itself.

      It is recommended to enable this option on less powerful devices.", + "message":"

      Τα γενικά κοσμητικά φίλτρα είναι εκείνα τα κοσμητικά φίλτρα που εφαρμόζονται σε όλες τις ιστοσελίδες.

      Αν και γίνεται αποτελεσματική διαχείρισή τους από το uBlock₀, τα γενικά κοσμητικά φίλτρα ενδέχεται να καταναλώσουν σημαντική μνήμη και να υπερφορτώσουν τη CPU σε μερικές ιστοσελίδες, ειδικά για μεγάλες μακροχρόνιες.

      Η ενεργοποίηση αυτής της επιλογής θα εξαλείψει την υπερφόρτωση μνήμης και CPU στις ιστοσελίδες ως αποτέλεσμα της διαχείρισης γενικών κοσμητικών φίλτρων, ενώ ενδέχεται να μειώσει την κατανάλωση μνήμης του ίδιου του uBlock₀.

      Προτείνεται η ενεργοποίηση αυτής της επιλογής στις λιγότερο ισχυρές συσκευές.", "description":"Describes the purpose of the 'Ignore generic cosmetic filters' feature." }, "3pListsOfBlockedHostsHeader":{ diff --git a/src/_locales/et/messages.json b/src/_locales/et/messages.json index edd521c2f94f1..de3eade6be014 100644 --- a/src/_locales/et/messages.json +++ b/src/_locales/et/messages.json @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"Nimekiri domeenidest, millel uBlock₀ keelatakse. Üks domeen rea kohta, vigased domeeninimed eiratakse vaikimisi ning kommenteeritakse välja.", + "message":"Valge nimekirja direktiivid määravad, millistel veebilehtedel peaks uBlock Origin keelatud olema. Üks domeen rea kohta. Sobimatuid direktiive vaikselt ignoreeritakse ja kommenteeritakse välja.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ diff --git a/src/_locales/fi/messages.json b/src/_locales/fi/messages.json index 6678f3b5e9c2f..efd1f28cc400a 100644 --- a/src/_locales/fi/messages.json +++ b/src/_locales/fi/messages.json @@ -616,7 +616,7 @@ "description":"Firefox\/Fennec-specific: Show Dashboard" }, "showNetworkLogButton":{ - "message":"Näytä Verkkopyyntöjen Loki", + "message":"Näytä loki", "description":"Firefox\/Fennec-specific: Show Logger" }, "fennecMenuItemBlockingOff":{ diff --git a/src/_locales/hi/messages.json b/src/_locales/hi/messages.json index 2777ccbb282bc..d52d1536616d9 100644 --- a/src/_locales/hi/messages.json +++ b/src/_locales/hi/messages.json @@ -40,7 +40,7 @@ "description":"appears as tab name in dashboard" }, "advancedSettingsPageName":{ - "message":"Advanced settings", + "message":"उन्नत सेटिंग्स", "description":"Title for the advanced settings page" }, "popupPowerSwitchInfo":{ diff --git a/src/_locales/id/messages.json b/src/_locales/id/messages.json index e17a97b95e0b2..6f9362d2940a0 100644 --- a/src/_locales/id/messages.json +++ b/src/_locales/id/messages.json @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"Daftar nama host yang mana uBlock₀ akan dinonfungsikan. Satu entri per baris. Nama host yang tidak valid akan diabaikan tanpa peringatan.", + "message":"Daftar putih domain atau halaman, uBlock Origin akan dinonaktifkan. Satu entri per baris. Entri yang tidak valid akan diabaikan dan dijadikan komentar.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ diff --git a/src/_locales/it/messages.json b/src/_locales/it/messages.json index 22b31136dd3a6..594851c70ddd8 100644 --- a/src/_locales/it/messages.json +++ b/src/_locales/it/messages.json @@ -680,7 +680,7 @@ "description":"used as a prompt for the user to provide a custom device name" }, "advancedSettingsWarning":{ - "message":"Attenzione! Cambia queste impostazioni a tuo rischio.", + "message":"Attenzione! Cambia queste impostazioni avanzate a tuo rischio e pericolo.", "description":"A warning to users at the top of 'Advanced settings' page" }, "genericSubmit":{ diff --git a/src/_locales/ko/messages.json b/src/_locales/ko/messages.json index f8466f176dd2f..83ed0caa5514f 100644 --- a/src/_locales/ko/messages.json +++ b/src/_locales/ko/messages.json @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"목록에 있는 호스트들은 uBlock₀에서 비활성화됩니다. 한 줄에 한 개씩 입력하세요. 존재하지 않는 호스트는 자동으로 무시됩니다.", + "message":"목록에 있는 호스트들은 uBlock₀에서 비활성화됩니다. 한 줄에 한 개씩 입력하세요. 존재하지 않는 호스트는 무시 및 주석 처리됩니다.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ diff --git a/src/_locales/lv/messages.json b/src/_locales/lv/messages.json index d137cd2decd3a..b7b96f754a7dd 100644 --- a/src/_locales/lv/messages.json +++ b/src/_locales/lv/messages.json @@ -40,7 +40,7 @@ "description":"appears as tab name in dashboard" }, "advancedSettingsPageName":{ - "message":"Advanced settings", + "message":"Papildu iestatījumi", "description":"Title for the advanced settings page" }, "popupPowerSwitchInfo":{ @@ -212,7 +212,7 @@ "description":"" }, "settingsAdvancedUserSettings":{ - "message":"advanced settings", + "message":"papildu iestatījumi", "description":"For the tooltip of a link which gives access to advanced settings" }, "settingsPrefetchingDisabledPrompt":{ @@ -288,7 +288,7 @@ "description":"Describes the purpose of the 'Parse and enforce cosmetic filters' feature." }, "3pIgnoreGenericCosmeticFilters":{ - "message":"Ignore generic cosmetic filters", + "message":"Ignorēt vispārīgos kosmētiskos filtrus", "description":"This will cause uBO to ignore all generic cosmetic filters." }, "3pIgnoreGenericCosmeticFiltersInfo":{ @@ -540,7 +540,7 @@ "description":"English: project' wiki on Github" }, "aboutSupport":{ - "message":"Support", + "message":"Atbalsts", "description":"A link for where to get support" }, "aboutCode":{ diff --git a/src/_locales/pt_BR/messages.json b/src/_locales/pt_BR/messages.json index a642797772981..ecdf542ae01a0 100644 --- a/src/_locales/pt_BR/messages.json +++ b/src/_locales/pt_BR/messages.json @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"Sua lista de exceções de servidores para qual o uBlock₀ será desativado. Uma entrada por linha. Servidores inválidos serão silenciosamente ignorados.", + "message":"Sua lista branca de servidores para qual o uBlock Origin será desativado. Uma regra por linha. Servidores inválidos serão silenciosamente ignorados.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ diff --git a/src/_locales/pt_PT/messages.json b/src/_locales/pt_PT/messages.json index b0bd15256d2e2..e2d63b466d109 100644 --- a/src/_locales/pt_PT/messages.json +++ b/src/_locales/pt_PT/messages.json @@ -84,7 +84,7 @@ "description":"Tooltip for the no-popups per-site switch" }, "popupTipNoLargeMedia":{ - "message":"Alternar o bloqueio de grandes elementos multimédia para este site", + "message":"Alternar o bloqueio de elementos multimédia grandes para este site", "description":"Tooltip for the no-large-media per-site switch" }, "popupTipNoCosmeticFiltering":{ @@ -144,7 +144,7 @@ "description":"" }, "popupHitDomainCountPrompt":{ - "message":"domínios conectados", + "message":"domínios ligados", "description":"appears in popup" }, "popupHitDomainCount":{ @@ -640,7 +640,7 @@ "description":"English: List of filter list names follows" }, "docblockedBack":{ - "message":"Recuar", + "message":"Retroceder", "description":"English: Go back" }, "docblockedClose":{ @@ -700,7 +700,7 @@ "description":"" }, "contextMenuTemporarilyAllowLargeMediaElements":{ - "message":"Permitir temporariamente grandes elementos multimédia", + "message":"Permitir temporariamente elementos multimédia grandes", "description":"A context menu entry, present when large media elements have been blocked on the current site" }, "dummy":{ diff --git a/src/_locales/sr/messages.json b/src/_locales/sr/messages.json index 9e060b20bb341..aef2d92d94501 100644 --- a/src/_locales/sr/messages.json +++ b/src/_locales/sr/messages.json @@ -152,7 +152,7 @@ "description":"appears in popup" }, "pickerCreate":{ - "message":"Направи", + "message":"Креирај", "description":"English: Create" }, "pickerPick":{ diff --git a/src/_locales/vi/messages.json b/src/_locales/vi/messages.json index f112d6f706eeb..b4d2248e7bb0e 100644 --- a/src/_locales/vi/messages.json +++ b/src/_locales/vi/messages.json @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"Danh sách tên các máy chủ mà uBlock₀ sẽ bị chặn. Một mục nhập trên mỗi dòng. Tên máy chủ không hợp lệ sẽ được tự động bỏ qua.", + "message":"Danh sách tên các máy chủ mà µBlock₀ sẽ bị vô hiệu. Một mục nhập trên mỗi dòng. Tên máy chủ không hợp lệ sẽ được tự động bỏ qua.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ diff --git a/src/_locales/zh_CN/messages.json b/src/_locales/zh_CN/messages.json index 9a753bad9c379..1d5fe1c75f8c7 100644 --- a/src/_locales/zh_CN/messages.json +++ b/src/_locales/zh_CN/messages.json @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"uBlock₀ 在列表里的城名将会停用。一行一条规则。无效的城名将直接被忽略。", + "message":"您的列表中针对 µBlock 的主机名将被禁用。每行一条规则。无效的主机名将直接被忽略。", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ From 18096366808f7b49153e92f544e81468aa8a9482 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 25 Jan 2017 08:05:41 -0500 Subject: [PATCH 31/45] fix #2337 --- platform/firefox/vapi-background.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platform/firefox/vapi-background.js b/platform/firefox/vapi-background.js index 742a99258f2ff..c9cc54b83c031 100644 --- a/platform/firefox/vapi-background.js +++ b/platform/firefox/vapi-background.js @@ -274,7 +274,11 @@ vAPI.browserSettings = { // has a `media.peerconnection.ice.default_address_only` pref which // purpose is to prevent local IP address leakage. case 'webrtcIPAddress': - if ( this.getValue('media.peerconnection', 'ice.default_address_only') !== undefined ) { + // https://github.com/gorhill/uBlock/issues/2337 + if ( this.getValue('media.peerconnection', 'ice.no_host') !== undefined ) { + prefName = 'ice.no_host'; + prefVal = true; + } else if ( this.getValue('media.peerconnection', 'ice.default_address_only') !== undefined ) { prefName = 'ice.default_address_only'; prefVal = true; } else { From d79a781bea6872c8a12a1b284de518d7762c4a99 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 25 Jan 2017 09:21:16 -0500 Subject: [PATCH 32/45] report blanket websocket blocking once only --- src/js/traffic.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/traffic.js b/src/js/traffic.js index 6e4587d0fca45..29918bfbe0073 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -457,7 +457,7 @@ var processCSP = function(pageStore, details) { ); } - if ( loggerEnabled ) { + if ( loggerEnabled && details.type !== 'script' ) { if ( blockInlineScript !== undefined ) { µb.logger.writeOne( tabId, From 393d0b2d0887812161fe7ede066a0766e9610bc3 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 25 Jan 2017 09:21:44 -0500 Subject: [PATCH 33/45] new revision or dev build --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 6b233209d3043..07989cebbac84 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.5.101", + "version": "1.10.7.100", "default_locale": "en", "description": "__MSG_extShortDesc__", From aadf4a6427fa2960b4fb9e20cfbaab66e2642d81 Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 26 Jan 2017 10:17:38 -0500 Subject: [PATCH 34/45] fix #2340 --- src/js/assets.js | 182 ++++++++++++++++++++++---------------------- src/js/messaging.js | 19 ++--- src/js/start.js | 14 +++- src/js/storage.js | 81 +++++++++----------- 4 files changed, 147 insertions(+), 149 deletions(-) diff --git a/src/js/assets.js b/src/js/assets.js index ba83f5d141a16..1e13848cfec66 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -137,72 +137,74 @@ var getTextFileFromURL = function(url, onLoad, onError) { **/ api.listKeyAliases = { - "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", - "assets/user/filters.txt": "user-filters", - "assets/ublock/resources.txt": "ublock-resources", - "assets/ublock/filters.txt": "ublock-filters", - "assets/ublock/privacy.txt": "ublock-privacy", - "assets/ublock/unbreak.txt": "ublock-unbreak", - "assets/ublock/badware.txt": "ublock-badware", - "assets/ublock/experimental.txt": "ublock-experimental", - "https://easylist-downloads.adblockplus.org/easylistchina.txt": "CHN-0", - "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "CHN-1", - "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt": "CHN-2", - "https://easylist-downloads.adblockplus.org/easylistgermany.txt": "DEU-0", - "https://adblock.dk/block.csv": "DNK-0", - "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt": "easylist", - "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "easylist-nocosmetic", - "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt": "easyprivacy", - "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "fanboy-annoyance", - "https://easylist-downloads.adblockplus.org/fanboy-social.txt": "fanboy-social", - "https://easylist-downloads.adblockplus.org/liste_fr.txt": "FRA-0", - "http://adblock.gardar.net/is.abp.txt": "ISL-0", - "https://easylist-downloads.adblockplus.org/easylistitaly.txt": "ITA-0", - "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "ITA-1", - "https://easylist-downloads.adblockplus.org/advblock.txt": "RUS-0", - "https://easylist-downloads.adblockplus.org/bitblock.txt": "RUS-1", - "https://filters.adtidy.org/extension/chromium/filters/1.txt": "RUS-2", - "https://adguard.com/en/filter-rules.html?id=1": "RUS-2", - "https://easylist-downloads.adblockplus.org/easylistdutch.txt": "NLD-0", - "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt": "LVA-0", - "http://hosts-file.net/.%5Cad_servers.txt": "hphosts", - "http://adblock.ee/list.php": "EST-0", - "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt": "disconnect-malvertising", - "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt": "disconnect-malware", - "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt": "disconnect-tracking", - "https://www.certyficate.it/adblock/adblock.txt": "POL-0", - "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt": "awrl-0", - "http://adb.juvander.net/Finland_adb.txt": "FIN-0", - "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt": "KOR-0", - "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt": "KOR-1", - "https://www.fanboy.co.nz/fanboy-korean.txt": "KOR-2", - "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0", - "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt": "JPN-0", - "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake", - "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0", - "http://margevicius.lt/easylistlithuania.txt": "LTU-0", - "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-0", - "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-1", - "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-2", - "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", - "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0", - "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0", - "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "HUN-0", - "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "CZE-0", - "http://someonewhocares.org/hosts/hosts": "dpollock-0", - "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt": "spam404-0", - "http://stanev.org/abp/adblock_bg.txt": "BGR-0", - "http://winhelp2002.mvps.org/hosts.txt": "mvps-0", - "https://www.fanboy.co.nz/enhancedstats.txt": "fanboy-enhanced", - "https://www.fanboy.co.nz/fanboy-antifacebook.txt": "fanboy-thirdparty_social", - "https://easylist-downloads.adblockplus.org/easylistspanish.txt": "spa-0", - "https://www.fanboy.co.nz/fanboy-swedish.txt": "SWE-0", - "https://www.fanboy.co.nz/r/fanboy-ultimate.txt": "fanboy-ultimate", - "https://filters.adtidy.org/extension/chromium/filters/13.txt": "TUR-0", - "https://adguard.com/filter-rules.html?id=13": "TUR-0", - "https://www.fanboy.co.nz/fanboy-vietnam.txt": "VIE-0", - "https://www.void.gr/kargig/void-gr-filters.txt": "GRC-0", - "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt": "SVN-0" + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", + "assets/user/filters.txt": "user-filters", + "assets/ublock/resources.txt": "ublock-resources", + "assets/ublock/filters.txt": "ublock-filters", + "assets/ublock/privacy.txt": "ublock-privacy", + "assets/ublock/unbreak.txt": "ublock-unbreak", + "assets/ublock/badware.txt": "ublock-badware", + "assets/ublock/experimental.txt": "ublock-experimental", + "https://easylist-downloads.adblockplus.org/easylistchina.txt": "CHN-0", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "CHN-1", + "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt": "CHN-2", + "https://easylist-downloads.adblockplus.org/easylistgermany.txt": "DEU-0", + "https://adblock.dk/block.csv": "DNK-0", + "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt": "easylist", + "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "easylist-nocosmetic", + "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt": "easyprivacy", + "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "fanboy-annoyance", + "https://easylist-downloads.adblockplus.org/fanboy-social.txt": "fanboy-social", + "https://easylist-downloads.adblockplus.org/liste_fr.txt": "FRA-0", + "http://adblock.gardar.net/is.abp.txt": "ISL-0", + "https://easylist-downloads.adblockplus.org/easylistitaly.txt": "ITA-0", + "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "ITA-1", + "https://easylist-downloads.adblockplus.org/advblock.txt": "RUS-0", + "https://easylist-downloads.adblockplus.org/bitblock.txt": "RUS-1", + "https://filters.adtidy.org/extension/chromium/filters/1.txt": "RUS-2", + "https://adguard.com/en/filter-rules.html?id=1": "RUS-2", + "https://easylist-downloads.adblockplus.org/easylistdutch.txt": "NLD-0", + "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt": "LVA-0", + "http://hosts-file.net/.%5Cad_servers.txt": "hphosts", + "http://adblock.ee/list.php": "EST-0", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt": "disconnect-malvertising", + "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt": "disconnect-malware", + "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt": "disconnect-tracking", + "https://www.certyficate.it/adblock/adblock.txt": "POL-0", + "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt": "POL-0", + "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt": "awrl-0", + "http://adb.juvander.net/Finland_adb.txt": "FIN-0", + "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt": "KOR-0", + "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt": "KOR-1", + "https://www.fanboy.co.nz/fanboy-korean.txt": "KOR-2", + "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0", + "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0", + "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt": "JPN-0", + "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake", + "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0", + "http://margevicius.lt/easylistlithuania.txt": "LTU-0", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-0", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-1", + "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-2", + "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", + "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0", + "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0", + "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "HUN-0", + "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "CZE-0", + "http://someonewhocares.org/hosts/hosts": "dpollock-0", + "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt": "spam404-0", + "http://stanev.org/abp/adblock_bg.txt": "BGR-0", + "http://winhelp2002.mvps.org/hosts.txt": "mvps-0", + "https://www.fanboy.co.nz/enhancedstats.txt": "fanboy-enhanced", + "https://www.fanboy.co.nz/fanboy-antifacebook.txt": "fanboy-thirdparty_social", + "https://easylist-downloads.adblockplus.org/easylistspanish.txt": "spa-0", + "https://www.fanboy.co.nz/fanboy-swedish.txt": "SWE-0", + "https://www.fanboy.co.nz/r/fanboy-ultimate.txt": "fanboy-ultimate", + "https://filters.adtidy.org/extension/chromium/filters/13.txt": "TUR-0", + "https://adguard.com/filter-rules.html?id=13": "TUR-0", + "https://www.fanboy.co.nz/fanboy-vietnam.txt": "VIE-0", + "https://www.void.gr/kargig/void-gr-filters.txt": "GRC-0", + "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt": "SVN-0" }; var migrate = function(callback) { @@ -343,7 +345,7 @@ var saveAssetSourceRegistry = (function() { }; })(); -var updateAssetSourceRegistry = function(json) { +var updateAssetSourceRegistry = function(json, silent) { var newDict; try { newDict = JSON.parse(json); @@ -351,29 +353,29 @@ var updateAssetSourceRegistry = function(json) { } if ( newDict instanceof Object === false ) { return; } - getAssetSourceRegistry(function(oldDict) { - var assetKey; - // Remove obsolete entries (only those which were built-in). - for ( assetKey in oldDict ) { - if ( - newDict[assetKey] === undefined && - oldDict[assetKey].submitter === undefined - ) { - unregisterAssetSource(assetKey); - } + var oldDict = assetSourceRegistry, + assetKey; + + // Remove obsolete entries (only those which were built-in). + for ( assetKey in oldDict ) { + if ( + newDict[assetKey] === undefined && + oldDict[assetKey].submitter === undefined + ) { + unregisterAssetSource(assetKey); } - // Add/update existing entries. Notify of new asset sources. - for ( assetKey in newDict ) { - if ( oldDict[assetKey] === undefined ) { - fireNotification( - 'builtin-asset-source-added', - { assetKey: assetKey, entry: newDict[assetKey] } - ); - } - registerAssetSource(assetKey, newDict[assetKey]); + } + // Add/update existing entries. Notify of new asset sources. + for ( assetKey in newDict ) { + if ( oldDict[assetKey] === undefined && !silent ) { + fireNotification( + 'builtin-asset-source-added', + { assetKey: assetKey, entry: newDict[assetKey] } + ); } - saveAssetSourceRegistry(); - }); + registerAssetSource(assetKey, newDict[assetKey]); + } + saveAssetSourceRegistry(); }; var getAssetSourceRegistry = function(callback) { @@ -406,7 +408,7 @@ var getAssetSourceRegistry = function(callback) { getTextFileFromURL( µBlock.assetsBootstrapLocation || 'assets/assets.json', function() { - updateAssetSourceRegistry(this.responseText); + updateAssetSourceRegistry(this.responseText, true); registryReady(); } ); @@ -899,7 +901,7 @@ api.metadata = function(callback) { }); getAssetCacheRegistry(function() { - cacheRegistryReady = assetCacheRegistry; + cacheRegistryReady = true; if ( assetRegistryReady ) { onReady(); } }); }; diff --git a/src/js/messaging.js b/src/js/messaging.js index 201ddbf2878fc..4549d6c5ba18b 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -754,27 +754,25 @@ var backupUserData = function(callback) { timeStamp: Date.now(), version: vAPI.app.version, userSettings: µb.userSettings, - selectedFilterLists: [], + selectedFilterLists: µb.selectedFilterLists, hiddenSettingsString: µb.stringFromHiddenSettings(), netWhitelist: µb.stringFromWhitelist(µb.netWhitelist), dynamicFilteringString: µb.permanentFirewall.toString(), urlFilteringString: µb.permanentURLFiltering.toString(), hostnameSwitchesString: µb.hnSwitches.toString(), - userFilters: '' - }; - - var onSelectedListsReady = function(selectedFilterLists) { - userData.selectedFilterLists = selectedFilterLists; - + userFilters: '', // TODO(seamless migration): // The following is strictly for convenience, to be minimally // forward-compatible. This will definitely be removed in the // short term, as I do not expect the need to install an older // version of uBO to ever be needed beyond the short term. // >>>>>>>> - userData.filterLists = µb.oldDataFromNewListKeys(selectedFilterLists); + filterLists: µb.oldDataFromNewListKeys(µb.selectedFilterLists) // <<<<<<<< + }; + var onUserFiltersReady = function(details) { + userData.userFilters = details.content; var filename = vAPI.i18n('aboutBackupFilename') .replace('{{datetime}}', µb.dateNowToSensibleString()) .replace(/ +/g, '_'); @@ -789,11 +787,6 @@ var backupUserData = function(callback) { getLocalData(callback); }; - var onUserFiltersReady = function(details) { - userData.userFilters = details.content; - µb.loadSelectedFilterLists(onSelectedListsReady); - }; - µb.assets.get(µb.userFiltersPath, onUserFiltersReady); }; diff --git a/src/js/start.js b/src/js/start.js index 2ccd4fc8dc061..8c37217f9b439 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -241,7 +241,7 @@ var fromFetch = function(to, fetched) { /******************************************************************************/ -var onAdminSettingsRestored = function() { +var onSelectedFilterListsLoaded = function() { var fetchableProps = { 'compiledMagic': '', 'dynamicFilteringString': 'behind-the-scene * 3p noop\nbehind-the-scene * 3p-frame noop', @@ -266,6 +266,18 @@ var onAdminSettingsRestored = function() { /******************************************************************************/ +// TODO(seamless migration): +// Eventually selected filter list keys will be loaded as a fetchable +// property. Until then we need to handle backward and forward +// compatibility, this means a special asynchronous call to load selected +// filter lists. + +var onAdminSettingsRestored = function() { + µb.loadSelectedFilterLists(onSelectedFilterListsLoaded); +}; + +/******************************************************************************/ + µb.hiddenSettings = (function() { var out = objectAssign({}, µb.hiddenSettingsDefault), json = vAPI.localStorage.getItem('hiddenSettings'); diff --git a/src/js/storage.js b/src/js/storage.js index 7d394f52d7589..f914c045237ac 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -170,7 +170,12 @@ var µb = this; vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists' ], function(bin) { if ( !bin || !bin.selectedFilterLists && !bin.remoteBlacklists ) { - return callback(); + // Select default filter lists if first-time launch. + µb.assets.metadata(function(availableLists) { + µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(availableLists)); + callback(); + }); + return; } var listKeys = []; if ( bin.selectedFilterLists ) { @@ -186,33 +191,30 @@ // Uncomment when all have moved to v1.11 and beyond. //vAPI.storage.remove('remoteBlacklists'); } - µb.selectedFilterLists = listKeys.slice(); - callback(listKeys); + µb.selectedFilterLists = listKeys; + callback(); }); }; µBlock.saveSelectedFilterLists = function(newKeys, append) { - var µb = this; - this.loadSelectedFilterLists(function(oldKeys) { - oldKeys = oldKeys || []; - if ( append ) { - newKeys = newKeys.concat(oldKeys); - } - var newSet = new Set(newKeys); - // Purge unused filter lists from cache. - for ( var i = 0, n = oldKeys.length; i < n; i++ ) { - if ( newSet.has(oldKeys[i]) === false ) { - µb.removeFilterList(oldKeys[i]); - } + var oldKeys = this.selectedFilterLists.slice(); + if ( append ) { + newKeys = newKeys.concat(oldKeys); + } + var newSet = new Set(newKeys); + // Purge unused filter lists from cache. + for ( var i = 0, n = oldKeys.length; i < n; i++ ) { + if ( newSet.has(oldKeys[i]) === false ) { + this.removeFilterList(oldKeys[i]); } - newKeys = µb.setToArray(newSet); - var bin = { - selectedFilterLists: newKeys, - remoteBlacklists: µb.oldDataFromNewListKeys(newKeys) - }; - µb.selectedFilterLists = newKeys; - vAPI.storage.set(bin); - }); + } + newKeys = this.setToArray(newSet); + var bin = { + selectedFilterLists: newKeys, + remoteBlacklists: this.oldDataFromNewListKeys(newKeys) + }; + this.selectedFilterLists = newKeys; + vAPI.storage.set(bin); }; // TODO(seamless migration): @@ -429,7 +431,7 @@ µBlock.autoSelectRegionalFilterLists = function(lists) { var lang = self.navigator.language.slice(0, 2), - selectedListKeys = [], + selectedListKeys = [ this.userFiltersPath ], list; for ( var key in lists ) { if ( lists.hasOwnProperty(key) === false ) { continue; } @@ -522,26 +524,7 @@ } }; - // Selected lists. - var onSelectedListsLoaded = function(keys) { - var listKey; - // No user lists data means use default settings. - if ( Array.isArray(keys) ) { - var listKeySet = new Set(keys); - for ( listKey in newAvailableLists ) { - if ( newAvailableLists.hasOwnProperty(listKey) ) { - newAvailableLists[listKey].off = !listKeySet.has(listKey); - } - } - } else if ( µb.firstInstall ) { - µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(newAvailableLists)); - } - - finalize(); - callback(newAvailableLists); - }; - - // Built-in filter lists. + // Built-in filter lists loaded. var onBuiltinListsLoaded = function(entries) { for ( var assetKey in entries ) { if ( entries.hasOwnProperty(assetKey) === false ) { continue; } @@ -551,7 +534,15 @@ } // Load set of currently selected filter lists. - µb.loadSelectedFilterLists(onSelectedListsLoaded); + var listKeySet = new Set(µb.selectedFilterLists); + for ( listKey in newAvailableLists ) { + if ( newAvailableLists.hasOwnProperty(listKey) ) { + newAvailableLists[listKey].off = !listKeySet.has(listKey); + } + } + + finalize(); + callback(newAvailableLists); }; // Available lists previously computed. From 597583265315763347c47b59fb2ff7e9815a5ec7 Mon Sep 17 00:00:00 2001 From: gorhill Date: Thu, 26 Jan 2017 10:24:28 -0500 Subject: [PATCH 35/45] new release candidate --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 07989cebbac84..b04c85130c35b 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.7.100", + "version": "1.10.7.101", "default_locale": "en", "description": "__MSG_extShortDesc__", From 0b4f31bd8aba77c5ee84ce46a095480bb821e9bf Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 27 Jan 2017 13:44:52 -0500 Subject: [PATCH 36/45] fix #2344 --- assets/assets.json | 2 +- src/js/storage.js | 7 +++---- src/js/utils.js | 12 ++++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/assets/assets.json b/assets/assets.json index e8dc9cd56a40c..305d1b6cdd658 100644 --- a/assets/assets.json +++ b/assets/assets.json @@ -529,7 +529,7 @@ "group": "regions", "off": true, "title": "RUS: RU AdList (Дополнительная региональная подписка)", - "lang": "ru", + "lang": "be ru uk", "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt", "supportURL": "https://forums.lanik.us/viewforum.php?f=102" }, diff --git a/src/js/storage.js b/src/js/storage.js index f914c045237ac..3467282088fa7 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -430,8 +430,7 @@ /******************************************************************************/ µBlock.autoSelectRegionalFilterLists = function(lists) { - var lang = self.navigator.language.slice(0, 2), - selectedListKeys = [ this.userFiltersPath ], + var selectedListKeys = [ this.userFiltersPath ], list; for ( var key in lists ) { if ( lists.hasOwnProperty(key) === false ) { continue; } @@ -440,7 +439,7 @@ selectedListKeys.push(key); continue; } - if ( list.lang === lang ) { + if ( this.matchCurrentLanguage(list.lang) ) { selectedListKeys.push(key); list.off = false; } @@ -1135,7 +1134,7 @@ if ( details.entry.content === 'filters' ) { if ( details.entry.off !== true || - self.navigator.language.startsWith(details.entry.lang) + this.matchCurrentLanguage(details.entry.lang) ) { this.saveSelectedFilterLists([ details.assetKey ], true); } diff --git a/src/js/utils.js b/src/js/utils.js index 04c9a3e41000c..e7834c3de6a6f 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -245,3 +245,15 @@ }; /******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/2344 + +µBlock.matchCurrentLanguage = function(s) { + if ( typeof s !== 'string' ) { return false; } + if ( this.matchCurrentLanguage.reLang === undefined ) { + this.matchCurrentLanguage.reLang = new RegExp('\\b' + self.navigator.language.slice(0, 2) + '\\b'); + } + return this.matchCurrentLanguage.reLang.test(s); +}; + +/******************************************************************************/ From 0d2e4f56218e77168fbb98ce476359f489db8f61 Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 27 Jan 2017 13:46:33 -0500 Subject: [PATCH 37/45] new release candidate --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index b04c85130c35b..5c2540e7a23da 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.7.101", + "version": "1.10.7.102", "default_locale": "en", "description": "__MSG_extShortDesc__", From a0c172c13e689c9df2a7f57ada1e115fa4d3db25 Mon Sep 17 00:00:00 2001 From: gorhill Date: Sat, 28 Jan 2017 13:42:33 -0500 Subject: [PATCH 38/45] to mitigate https://github.com/gorhill/uBO-Extra/issues/7 --- platform/chromium/vapi-background.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index db6b0305ec29c..63290d47b6ab0 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -978,15 +978,20 @@ vAPI.net.registerListeners = function() { // search for "https://github.com/gorhill/uBlock/issues/1497". var onBeforeWebsocketRequest = function(details) { details.type = 'websocket'; - var matches = /url=([^&]+)/.exec(details.url); + var requestURL = details.url; + var matches = /[?&]url=([^&]+)/.exec(requestURL); details.url = decodeURIComponent(matches[1]); var r = onBeforeRequestClient(details); // Blocked? - if ( r && r.cancel ) { - return r; - } - // Returning a 1x1 transparent pixel means "not blocked". - return { redirectUrl: 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' }; + if ( r && r.cancel ) { return r; } + // Try to redirect to the URL of an image already present in the + // document, or a 1x1 data: URL if none is present. + matches = /[?&]r=([^&]+)/.exec(requestURL); + return { + redirectUrl: matches !== null ? + decodeURIComponent(matches[1]) : + 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==' + }; }; var onBeforeRequestClient = this.onBeforeRequest.callback; From ba3127a59b9c46b6c93ee1befd826f3333a8c21f Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 1 Feb 2017 08:59:35 -0500 Subject: [PATCH 39/45] new revision for release candidate --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 5c2540e7a23da..335f79ab51736 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.7.102", + "version": "1.10.7.103", "default_locale": "en", "description": "__MSG_extShortDesc__", From 1ba853df8cabc43d143535b4b65ed63a4fe992cc Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 1 Feb 2017 09:05:41 -0500 Subject: [PATCH 40/45] all releases are pre-release by default --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5f15a9782e3b0..a46f7f9c2113a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ env: script: ./tools/make-${BROWSER}.sh all deploy: provider: releases + prerelease: true # https://github.com/travis-ci/travis-ci/issues/6772 edge: branch: releases-fix From 60605033bfd252f6b13966b70b3ce1f45849af79 Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 3 Feb 2017 08:12:26 -0500 Subject: [PATCH 41/45] fix #1871? ("blind" fix, need confirmation) --- src/js/messaging.js | 16 ++++++++++++---- src/js/storage.js | 8 ++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/js/messaging.js b/src/js/messaging.js index 4549d6c5ba18b..20ec0e3ca11c4 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -793,6 +793,10 @@ var backupUserData = function(callback) { var restoreUserData = function(request) { var userData = request.userData; + var restart = function() { + vAPI.app.restart(); + }; + var onAllRemoved = function() { µBlock.saveLocalSettings(); vAPI.storage.set(userData.userSettings); @@ -811,13 +815,17 @@ var restoreUserData = function(request) { // 'filterLists' is available up to uBO v1.10.4, not beyond. // 'selectedFilterLists' is available from uBO v1.11 and beyond. + var listKeys; if ( Array.isArray(userData.selectedFilterLists) ) { - µb.saveSelectedFilterLists(userData.selectedFilterLists); + listKeys = userData.selectedFilterLists; } else if ( userData.filterLists instanceof Object ) { - µb.saveSelectedFilterLists(µb.newListKeysFromOldData(userData.filterLists)); + listKeys = µb.newListKeysFromOldData(userData.filterLists); + } + if ( listKeys !== undefined ) { + µb.saveSelectedFilterLists(listKeys, restart); + } else { + restart(); } - - vAPI.app.restart(); }; // https://github.com/chrisaljoudi/uBlock/issues/1102 diff --git a/src/js/storage.js b/src/js/storage.js index 3467282088fa7..66203f733d2fb 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -196,7 +196,11 @@ }); }; -µBlock.saveSelectedFilterLists = function(newKeys, append) { +µBlock.saveSelectedFilterLists = function(newKeys, append, callback) { + if ( typeof append === 'function' ) { + callback = append; + append = false; + } var oldKeys = this.selectedFilterLists.slice(); if ( append ) { newKeys = newKeys.concat(oldKeys); @@ -214,7 +218,7 @@ remoteBlacklists: this.oldDataFromNewListKeys(newKeys) }; this.selectedFilterLists = newKeys; - vAPI.storage.set(bin); + vAPI.storage.set(bin, callback); }; // TODO(seamless migration): From f321ea31c1b6ab6771bf059cbde2673a9cf0f32a Mon Sep 17 00:00:00 2001 From: gorhill Date: Fri, 3 Feb 2017 08:13:08 -0500 Subject: [PATCH 42/45] new revision for release candidate --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 335f79ab51736..9b2af15813fd9 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.7.103", + "version": "1.10.7.104", "default_locale": "en", "description": "__MSG_extShortDesc__", From 1bceca9cbbddc23e8f02b6f6c81a1cf17939355e Mon Sep 17 00:00:00 2001 From: gorhill Date: Sun, 5 Feb 2017 07:43:28 -0500 Subject: [PATCH 43/45] fix var name (https://github.com/el1t/uBlock-Safari/issues/25#issuecomment-277506943)) --- src/js/storage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/storage.js b/src/js/storage.js index 66203f733d2fb..bfba9d2ea560c 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1121,7 +1121,7 @@ this.loadFilterLists(); } if ( this.userSettings.autoUpdate ) { - this.scheduleAssetUpdater(this.hiddenSettings.assetAutoUpdatePeriod * 3600000 || 25200000); + this.scheduleAssetUpdater(this.hiddenSettings.autoUpdatePeriod * 3600000 || 25200000); } else { this.scheduleAssetUpdater(0); } From e4e6d8d8c3dc3daf1481f5f8043926e5a549315f Mon Sep 17 00:00:00 2001 From: gorhill Date: Sun, 5 Feb 2017 15:25:00 -0500 Subject: [PATCH 44/45] new stable version --- platform/chromium/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 9b2af15813fd9..b17c8120e87d7 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "uBlock Origin", - "version": "1.10.7.104", + "version": "1.11.0", "default_locale": "en", "description": "__MSG_extShortDesc__", From 2852f9be197ec20387a669df9ce01ef7d6df1c84 Mon Sep 17 00:00:00 2001 From: gorhill Date: Sun, 5 Feb 2017 15:27:21 -0500 Subject: [PATCH 45/45] translation work from https://crowdin.com/project/ublock --- src/_locales/fy/messages.json | 10 +++++----- src/_locales/it/messages.json | 2 +- src/_locales/pt_BR/messages.json | 6 +++--- src/_locales/uk/messages.json | 6 +++--- src/_locales/zh_CN/messages.json | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/_locales/fy/messages.json b/src/_locales/fy/messages.json index 059edde5c7c16..d4cdd3a6bf802 100644 --- a/src/_locales/fy/messages.json +++ b/src/_locales/fy/messages.json @@ -40,7 +40,7 @@ "description":"appears as tab name in dashboard" }, "advancedSettingsPageName":{ - "message":"Advanced settings", + "message":"Avansearre ynstellingen", "description":"Title for the advanced settings page" }, "popupPowerSwitchInfo":{ @@ -212,7 +212,7 @@ "description":"" }, "settingsAdvancedUserSettings":{ - "message":"advanced settings", + "message":"avansearre ynstellingen", "description":"For the tooltip of a link which gives access to advanced settings" }, "settingsPrefetchingDisabledPrompt":{ @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"Jo list fan hostnammen wêrop uBlock₀ útskeakele is. Ien per rigel ynjaan. Unjildige hostnammen wurde stil negearre.", + "message":"De whitelist-ynstruksjes skriuwe foar op hokker websiden uBlock Origin útskeakele wurde moat. Ien fermelding per rigel. Unjildige ynstruksjes wurde sûnder meidieling negearre en útskeakele.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ @@ -680,7 +680,7 @@ "description":"used as a prompt for the user to provide a custom device name" }, "advancedSettingsWarning":{ - "message":"Warning! Change these advanced settings at your own risk.", + "message":"Warskôging! Wizigje dizze avansearre ynstellingen op eigen risiko.", "description":"A warning to users at the top of 'Advanced settings' page" }, "genericSubmit":{ @@ -688,7 +688,7 @@ "description":"for generic 'Submit' buttons" }, "genericApplyChanges":{ - "message":"Apply changes", + "message":"Wizigingen tapasse", "description":"for generic 'Apply changes' buttons" }, "genericRevert":{ diff --git a/src/_locales/it/messages.json b/src/_locales/it/messages.json index 594851c70ddd8..a8bf2552f51a9 100644 --- a/src/_locales/it/messages.json +++ b/src/_locales/it/messages.json @@ -680,7 +680,7 @@ "description":"used as a prompt for the user to provide a custom device name" }, "advancedSettingsWarning":{ - "message":"Attenzione! Cambia queste impostazioni avanzate a tuo rischio e pericolo.", + "message":"Attenzione! Modifica queste impostazioni avanzate a tuo rischio e pericolo.", "description":"A warning to users at the top of 'Advanced settings' page" }, "genericSubmit":{ diff --git a/src/_locales/pt_BR/messages.json b/src/_locales/pt_BR/messages.json index ecdf542ae01a0..42fcae42696b7 100644 --- a/src/_locales/pt_BR/messages.json +++ b/src/_locales/pt_BR/messages.json @@ -8,7 +8,7 @@ "description":"this will be in the chrome web store: must be 132 characters or less" }, "dashboardName":{ - "message":"uBlock₀ — Painel", + "message":"uBlock₀ — Painel de controle", "description":"English: uBlock₀ — Dashboard" }, "settingsPageName":{ @@ -200,7 +200,7 @@ "description":"English: Make use of context menu where appropriate" }, "settingsColorBlindPrompt":{ - "message":"Cores amigáveis para daltônicos", + "message":"Modo Daltonismo", "description":"English: Color-blind friendly" }, "settingsCloudStorageEnabledPrompt":{ @@ -612,7 +612,7 @@ "description":"English: {{value}} days ago" }, "showDashboardButton":{ - "message":"Mostrar Painel", + "message":"Mostrar Painel de Controle", "description":"Firefox\/Fennec-specific: Show Dashboard" }, "showNetworkLogButton":{ diff --git a/src/_locales/uk/messages.json b/src/_locales/uk/messages.json index 5757874f75717..7f0ae5f73f0fb 100644 --- a/src/_locales/uk/messages.json +++ b/src/_locales/uk/messages.json @@ -84,7 +84,7 @@ "description":"Tooltip for the no-popups per-site switch" }, "popupTipNoLargeMedia":{ - "message":"Перемкнути блокування великих медіа елементів на цьому сайті", + "message":"Увімк\/Вимк блокування великих медіа елементів на цьому сайті", "description":"Tooltip for the no-large-media per-site switch" }, "popupTipNoCosmeticFiltering":{ @@ -292,7 +292,7 @@ "description":"This will cause uBO to ignore all generic cosmetic filters." }, "3pIgnoreGenericCosmeticFiltersInfo":{ - "message":"

      Загальні косметичні фільтри — це косметичні фільтри, які повинні застосовуватися до всіх веб-сторінок.

      Хоча uBlock і звертається з ними ефективно, вони все одно можуть вимагати значну кількість ресурсів на деяких, особливо великих, сторінках.

      Увімкнення цього параметра знизить споживання ресурсів таких сторінок від застосування загальних косметичних фільтрів, а також знизить споживання пам'яті самого uBlock.

      Рекомендується увімкнути цей параметр на слабких пристроях.", + "message":"

      Загальні косметичні фільтри — це косметичні фільтри, які застосовуються до всіх веб-сторінок.

      Хоча й uBlock обробує фільтри ефективно, вони все одно можуть вимагати значну кількість ресурсів на деяких, особливо навантаженних сторінках.

      Увімкнення цього параметра знизить споживання ресурсів на таких сторінках від застосування загальних косметичних фільтрів, а також знизить споживання пам'яті самого uBlock.

      Рекомендується увімкнути цей параметр на слабких пристроях.", "description":"Describes the purpose of the 'Ignore generic cosmetic filters' feature." }, "3pListsOfBlockedHostsHeader":{ @@ -424,7 +424,7 @@ "description":"English: dynamic rule syntax and full documentation." }, "whitelistPrompt":{ - "message":"Список тих доменів, для яких µBlock буде вимикатись. Один запис на рядок. Недопустимі назви будуть ігноруватись.", + "message":"Ваш список адрес сайтів, для яких µBlock буде неактивним. Додайте по одному запису на рядок. Невірні адреси будуть проігноровані без зауважень.", "description":"English: An overview of the content of the dashboard's Whitelist pane." }, "whitelistImport":{ diff --git a/src/_locales/zh_CN/messages.json b/src/_locales/zh_CN/messages.json index 1d5fe1c75f8c7..7cd9edb6802fc 100644 --- a/src/_locales/zh_CN/messages.json +++ b/src/_locales/zh_CN/messages.json @@ -224,7 +224,7 @@ "description":"English: " }, "settingsWebRTCIPAddressHiddenPrompt":{ - "message":"防止 WebRTC 泄露本地IP地址", + "message":"避免 WebRTC 泄露本地IP地址", "description":"English: " }, "settingPerSiteSwitchGroup":{