summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRaymond Hill <gorhill@users.noreply.github.com>2017-01-18 13:17:47 -0500
committerGitHub <noreply@github.com>2017-01-18 13:17:47 -0500
commit3b9fd49c508812dfe2545836c63bd5d8cfd2ead4 (patch)
tree6f36522a12e49d15ca3db55db863b70aec302617
parent69fc59305ec60607593474b238e476f808ec7294 (diff)
Assets management refactored (#2314)
* refactoring assets management code * finalizing refactoring of assets management * various code review of new assets management code * fix #2281 * fix #1961 * fix #1293 * fix #1275 * fix update scheduler timing logic * forward compatibility (to be removed once 1.11+ is widespread) * more codereview; give admins ability to specify own assets.json * "assetKey" is more accurate than "path" * fix group count update when building dom incrementally * reorganize content (order, added URLs, etc.) * ability to customize updater through advanced settings * better spinner icon
Diffstat
-rw-r--r--assets/assets.json585
-rw-r--r--src/3p-filters.html16
-rw-r--r--src/_locales/en/messages.json4
-rw-r--r--src/background.html1
-rw-r--r--src/css/3p-filters.css91
-rw-r--r--src/js/3p-filters.js433
-rw-r--r--src/js/assets.js2078
-rw-r--r--src/js/background.js105
-rw-r--r--src/js/logger.js4
-rw-r--r--src/js/messaging.js89
-rw-r--r--src/js/redirect-engine.js24
-rw-r--r--src/js/reverselookup-worker.js18
-rw-r--r--src/js/reverselookup.js38
-rw-r--r--src/js/scriptlets/subscriber.js9
-rw-r--r--src/js/settings.js5
-rw-r--r--src/js/start.js27
-rw-r--r--src/js/storage.js957
-rw-r--r--src/js/ublock.js8
-rwxr-xr-xtools/make-assets.sh4
-rwxr-xr-xtools/make-chromium.sh6
20 files changed, 2074 insertions, 2428 deletions
diff --git a/assets/assets.json b/assets/assets.json
new file mode 100644
index 00000000..883b87ba
--- /dev/null
+++ b/assets/assets.json
@@ -0,0 +1,585 @@
+{
+ "assets.json": {
+ "content": "internal",
+ "updateAfter": 13,
+ "contentURL": [
+ "https://raw.githubusercontent.com/gorhill/uBlock/master/assets/assets.json",
+ "assets/assets.json"
+ ]
+ },
+ "public_suffix_list.dat": {
+ "content": "internal",
+ "updateAfter": 19,
+ "contentURL": [
+ "https://publicsuffix.org/list/public_suffix_list.dat",
+ "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat"
+ ]
+ },
+ "ublock-resources": {
+ "content": "internal",
+ "updateAfter": 7,
+ "contentURL": [
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/resources.txt",
+ "assets/ublock/resources.txt"
+ ]
+ },
+ "ublock-filters": {
+ "content": "filters",
+ "group": "default",
+ "title": "uBlock filters",
+ "contentURL": [
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt",
+ "assets/ublock/filters.txt"
+ ]
+ },
+ "ublock-badware": {
+ "content": "filters",
+ "group": "default",
+ "title": "uBlock filters – Badware risks",
+ "contentURL": [
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/badware.txt",
+ "assets/ublock/badware.txt"
+ ],
+ "supportURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks",
+ "instructionURL": "https://github.com/gorhill/uBlock/wiki/Badware-risks"
+ },
+ "ublock-experimental": {
+ "content": "filters",
+ "group": "default",
+ "title": "uBlock filters – Experimental",
+ "off": true,
+ "contentURL": [
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/experimental.txt",
+ "assets/ublock/experimental.txt"
+ ],
+ "supportURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters",
+ "instructionURL": "https://github.com/gorhill/uBlock/wiki/Experimental-filters"
+ },
+ "ublock-privacy": {
+ "content": "filters",
+ "group": "default",
+ "title": "uBlock filters – Privacy",
+ "contentURL": [
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/privacy.txt",
+ "assets/ublock/privacy.txt"
+ ]
+ },
+ "ublock-unbreak": {
+ "content": "filters",
+ "group": "default",
+ "title": "uBlock filters – Unbreak",
+ "contentURL": [
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/unbreak.txt",
+ "assets/ublock/unbreak.txt"
+ ]
+ },
+ "awrl-0": {
+ "content": "filters",
+ "group": "ads",
+ "off": true,
+ "title": "Adblock Warning Removal List",
+ "contentURL": "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "reek-0": {
+ "content": "filters",
+ "group": "ads",
+ "off": true,
+ "title": "Anti-Adblock Killer | Reek",
+ "contentURL": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt",
+ "supportURL": "https://github.com/reek/anti-adblock-killer",
+ "instructionURL": "https://github.com/reek/anti-adblock-killer#instruction"
+ },
+ "easylist": {
+ "content": "filters",
+ "group": "ads",
+ "title": "EasyList",
+ "contentURL": [
+ "https://easylist.to/easylist/easylist.txt",
+ "https://easylist-downloads.adblockplus.org/easylist.txt",
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easylist.txt",
+ "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt"
+ ],
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "easylist-nocosmetic": {
+ "content": "filters",
+ "group": "ads",
+ "off": true,
+ "title": "EasyList without element hiding rules",
+ "contentURL": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "disconnect-tracking": {
+ "content": "filters",
+ "group": "privacy",
+ "off": true,
+ "title": "Basic tracking list by Disconnect",
+ "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt"
+ },
+ "easyprivacy": {
+ "content": "filters",
+ "group": "privacy",
+ "title": "EasyPrivacy",
+ "contentURL": [
+ "https://easylist.to/easylist/easyprivacy.txt",
+ "https://easylist-downloads.adblockplus.org/easyprivacy.txt",
+ "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt",
+ "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt"
+ ],
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "fanboy-enhanced": {
+ "content": "filters",
+ "group": "privacy",
+ "off": true,
+ "title": "Fanboy’s Enhanced Tracking List",
+ "contentURL": "https://www.fanboy.co.nz/enhancedstats.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "disconnect-malvertising": {
+ "content": "filters",
+ "group": "malware",
+ "off": true,
+ "title": "Malvertising filter list by Disconnect",
+ "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt"
+ },
+ "malware-0": {
+ "content": "filters",
+ "group": "malware",
+ "title": "Malware Domain List",
+ "contentURL": [
+ "https://www.malwaredomainlist.com/hostslist/hosts.txt",
+ "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt"
+ ]
+ },
+ "malware-1": {
+ "content": "filters",
+ "group": "malware",
+ "title": "Malware domains",
+ "contentURL": [
+ "https://mirror.cedia.org.ec/malwaredomains/justdomains",
+ "https://mirror1.malwaredomains.com/files/justdomains",
+ "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains",
+ "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt"
+ ],
+ "supportURL": "http://www.malwaredomains.com/"
+ },
+ "malware-2": {
+ "content": "filters",
+ "group": "malware",
+ "off": true,
+ "title": "Malware domains (long-lived)",
+ "contentURL": [
+ "https://mirror1.malwaredomains.com/files/immortal_domains.txt",
+ "https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt"
+ ],
+ "supportURL": "http://www.malwaredomains.com/"
+ },
+ "disconnect-malware": {
+ "content": "filters",
+ "group": "malware",
+ "off": true,
+ "title": "Malware filter list by Disconnect",
+ "contentURL": "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt"
+ },
+ "spam404-0": {
+ "content": "filters",
+ "group": "malware",
+ "off": true,
+ "title": "Spam404",
+ "contentURL": "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt",
+ "supportURL": "http://www.spam404.com/"
+ },
+ "fanboy-thirdparty_social": {
+ "content": "filters",
+ "group": "social",
+ "off": true,
+ "title": "Anti-ThirdpartySocial (see warning inside list)",
+ "contentURL": "https://www.fanboy.co.nz/fanboy-antifacebook.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "fanboy-annoyance": {
+ "content": "filters",
+ "group": "social",
+ "off": true,
+ "title": "Fanboy’s Annoyance List",
+ "contentURL": [
+ "https://easylist.to/easylist/fanboy-annoyance.txt",
+ "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt"
+ ],
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "fanboy-social": {
+ "content": "filters",
+ "group": "social",
+ "off": true,
+ "title": "Fanboy’s Social Blocking List",
+ "contentURL": [
+ "https://easylist.to/easylist/fanboy-social.txt",
+ "https://easylist-downloads.adblockplus.org/fanboy-social.txt"
+ ],
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "dpollock-0": {
+ "content": "filters",
+ "group": "multipurpose",
+ "updateAfter": 11,
+ "off": true,
+ "title": "Dan Pollock’s hosts file",
+ "contentURL": "http://someonewhocares.org/hosts/hosts",
+ "supportURL": "http://someonewhocares.org/hosts/"
+ },
+ "fanboy-ultimate": {
+ "content": "filters",
+ "group": "multipurpose",
+ "off": true,
+ "title": "Fanboy+Easylist-Merged Ultimate List",
+ "contentURL": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "hphosts": {
+ "content": "filters",
+ "group": "multipurpose",
+ "updateAfter": 11,
+ "off": true,
+ "title": "hpHosts’ Ad and tracking servers",
+ "contentURL": "https://hosts-file.net/.%5Cad_servers.txt",
+ "supportURL": "https://hosts-file.net/"
+ },
+ "mvps-0": {
+ "content": "filters",
+ "group": "multipurpose",
+ "updateAfter": 11,
+ "off": true,
+ "title": "MVPS HOSTS",
+ "contentURL": "http://winhelp2002.mvps.org/hosts.txt",
+ "supportURL": "http://winhelp2002.mvps.org/"
+ },
+ "plowe-0": {
+ "content": "filters",
+ "group": "multipurpose",
+ "updateAfter": 13,
+ "title": "Peter Lowe’s Ad and tracking server list",
+ "contentURL": [
+ "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext",
+ "assets/thirdparties/pgl.yoyo.org/as/serverlist",
+ "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt"
+ ],
+ "supportURL": "https://pgl.yoyo.org/adservers/"
+ },
+ "ara-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "ara: Liste AR",
+ "lang": "ar",
+ "contentURL": "https://easylist-downloads.adblockplus.org/Liste_AR.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=98"
+ },
+ "BGR-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "BGR: Bulgarian Adblock list",
+ "lang": "bg",
+ "contentURL": "https://stanev.org/abp/adblock_bg.txt",
+ "supportURL": "https://stanev.org/abp/"
+ },
+ "CHN-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "CHN: EasyList China (中文)",
+ "lang": "zh",
+ "contentURL": "https://easylist-downloads.adblockplus.org/easylistchina.txt",
+ "supportURL": "http://abpchina.org/forum/forum.php"
+ },
+ "CHN-1": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "CHN: CJX's EasyList Lite (main focus on Chinese sites)",
+ "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt",
+ "supportURL": "https://github.com/cjx82630/cjxlist"
+ },
+ "CHN-2": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "CHN: CJX's Annoyance List",
+ "contentURL": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt",
+ "supportURL": "https://github.com/cjx82630/cjxlist"
+ },
+ "CZE-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "CZE, SVK: EasyList Czech and Slovak",
+ "lang": "cs",
+ "contentURL": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt",
+ "supportURL": "https://github.com/tomasko126/easylistczechandslovak"
+ },
+ "DEU-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "DEU: EasyList Germany",
+ "lang": "de",
+ "contentURL": [
+ "https://easylist.to/easylistgermany/easylistgermany.txt",
+ "https://easylist-downloads.adblockplus.org/easylistgermany.txt"
+ ],
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=90"
+ },
+ "DNK-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "DNK: Schacks Adblock Plus liste",
+ "lang": "da",
+ "contentURL": "https://adblock.dk/block.csv",
+ "supportURL": "https://henrik.schack.dk/adblock/"
+ },
+ "EST-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "EST: Eesti saitidele kohandatud filter",
+ "lang": "et",
+ "contentURL": "http://adblock.ee/list.php",
+ "supportURL": "http://adblock.ee/"
+ },
+ "EU-prebake": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "EU: Prebake - Filter Obtrusive Cookie Notices",
+ "contentURL": "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt",
+ "supportURL": "https://github.com/liamja/Prebake"
+ },
+ "FIN-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "FIN: Finnish Addition to Easylist",
+ "lang": "fi",
+ "contentURL": "http://adb.juvander.net/Finland_adb.txt",
+ "supportURL": "http://www.juvander.fi/AdblockFinland"
+ },
+ "FRA-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "FRA: EasyList Liste FR",
+ "lang": "fr",
+ "contentURL": "https://easylist-downloads.adblockplus.org/liste_fr.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=91"
+ },
+ "GRC-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "GRC: Greek AdBlock Filter",
+ "lang": "el",
+ "contentURL": "https://www.void.gr/kargig/void-gr-filters.txt",
+ "supportURL": "https://github.com/kargig/greek-adblockplus-filter"
+ },
+ "HUN-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "HUN: hufilter",
+ "lang": "hu",
+ "contentURL": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt",
+ "supportURL": "https://github.com/szpeter80/hufilter"
+ },
+ "IDN-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "IDN: ABPindo",
+ "lang": "id",
+ "contentURL": [
+ "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt",
+ "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt"
+ ],
+ "supportURL": "https://github.com/ABPindo/indonesianadblockrules"
+ },
+ "ISL-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "ISL: Icelandic ABP List",
+ "lang": "is",
+ "contentURL": "http://adblock.gardar.net/is.abp.txt",
+ "supportURL": "http://adblock.gardar.net/"
+ },
+ "ISR-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "ISR: EasyList Hebrew",
+ "lang": "he",
+ "contentURL": "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt",
+ "supportURL": "https://github.com/easylist/EasyListHebrew"
+ },
+ "ITA-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "ITA: EasyList Italy",
+ "lang": "it",
+ "contentURL": "https://easylist-downloads.adblockplus.org/easylistitaly.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=96"
+ },
+ "ITA-1": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "ITA: ABP X Files",
+ "contentURL": "https://raw.githubusercontent.com/gioxx/xfiles/master/filtri.txt",
+ "supportURL": "http://noads.it/"
+ },
+ "JPN-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "JPN: ABP Japanese filters (日本用フィルタ)",
+ "lang": "ja",
+ "contentURL": "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt",
+ "supportURL": "https://github.com/k2jp/abp-japanese-filters/wiki/Support_Policy"
+ },
+ "KOR-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "KOR: Korean Adblock List",
+ "lang": "ko",
+ "contentURL": "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt",
+ "supportURL": "https://github.com/gfmaster/adblock-korea-contrib"
+ },
+ "KOR-1": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "KOR: YousList",
+ "lang": "ko",
+ "contentURL": "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt",
+ "supportURL": "https://github.com/yous/YousList"
+ },
+ "KOR-2": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "KOR: Fanboy's Korean",
+ "contentURL": "https://www.fanboy.co.nz/fanboy-korean.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "LTU-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "LTU: Adblock Plus Lithuania",
+ "lang": "lt",
+ "contentURL": "http://margevicius.lt/easylistlithuania.txt",
+ "supportURL": "http://margevicius.lt/easylist_lithuania/"
+ },
+ "LVA-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "LVA: Latvian List",
+ "lang": "lv",
+ "contentURL": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt",
+ "supportURL": "https://notabug.org/latvian-list/adblock-latvian"
+ },
+ "NLD-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "NLD: EasyList Dutch",
+ "lang": "nl",
+ "contentURL": "https://easylist-downloads.adblockplus.org/easylistdutch.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=100"
+ },
+ "POL-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "POL: polskie filtry do Adblocka i uBlocka",
+ "lang": "pl",
+ "contentURL": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt",
+ "supportURL": "https://www.certyficate.it/adblock-ublock-polish-filters/"
+ },
+ "RUS-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "RUS: RU AdList (Дополнительная региональная подписка)",
+ "lang": "ru",
+ "contentURL": "https://easylist-downloads.adblockplus.org/advblock.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=102"
+ },
+ "RUS-1": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "RUS: BitBlock List (Дополнительная подписка фильтров)",
+ "contentURL": "https://easylist-downloads.adblockplus.org/bitblock.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=102"
+ },
+ "RUS-2": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "RUS: Adguard Russian Filter",
+ "contentURL": "https://filters.adtidy.org/extension/chromium/filters/1.txt",
+ "supportURL": "https://forum.adguard.com/forumdisplay.php?69-%D0%A4%D0%B8%D0%BB%D1%8C%D1%82%D1%80%D1%8B-Adguard"
+ },
+ "spa-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "spa: EasyList Spanish",
+ "lang": "es",
+ "contentURL": "https://easylist-downloads.adblockplus.org/easylistspanish.txt",
+ "supportURL": "https://forums.lanik.us/viewforum.php?f=103"
+ },
+ "SVN-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "SVN: Slovenian List",
+ "lang": "sl",
+ "contentURL": "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt",
+ "supportURL": "https://github.com/betterwebleon/slovenian-list"
+ },
+ "SWE-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "SWE: Fanboy's Swedish",
+ "lang": "sv",
+ "contentURL": "https://www.fanboy.co.nz/fanboy-swedish.txt",
+ "supportURL": "https://forums.lanik.us/"
+ },
+ "TUR-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "TUR: Adguard Turkish Filter",
+ "lang": "tr",
+ "contentURL": "https://filters.adtidy.org/extension/chromium/filters/13.txt",
+ "supportURL": "https://forum.adguard.com/forumdisplay.php?51-Filter-Rules"
+ },
+ "VIE-0": {
+ "content": "filters",
+ "group": "regions",
+ "off": true,
+ "title": "VIE: Fanboy's Vietnamese",
+ "lang": "vi",
+ "contentURL": "https://www.fanboy.co.nz/fanboy-vietnam.txt",
+ "supportURL": "https://forums.lanik.us/"
+ }
+}
diff --git a/src/3p-filters.html b/src/3p-filters.html
index 8bce9d01..48086f71 100644
--- a/src/3p-filters.html
+++ b/src/3p-filters.html
@@ -40,16 +40,10 @@
<button id="externalListsApply" class="custom important" disabled="true" data-i18n="3pExternalListsApply"></button></p>
</div>
-<div id="busyOverlay">
- <div></div>
- <!-- progress bar widget -->
- <div><div></div><div></div></div>
-</div>
-
<div id="templates" style="display: none;">
<ul>
<li class="groupEntry"><span class="geName"></span> <span class="geCount dim"></span>
- <ul></ul>
+ <ul class="listEntries"></ul>
</li>
<li class="listEntry">
<input type="checkbox">
@@ -57,10 +51,10 @@
<a class="fa" style="display: none;" target="_blank">&#xf05a;</a>
<a href="" style="display: none;" target="_blank"></a>: <!--
--><span class="counts dim"></span><!--
-  --><span class="status unsecure" style="display: none;">http</span><!--
-  --><span class="status new" style="display: none;" data-i18n="3pExternalListNew"></span><!--
-  --><span class="status obsolete" style="display: none;" data-i18n="3pExternalListObsolete"></span><!--
-  --><span class="status purge" style="display: none;" data-i18n="3pExternalListPurge"></span>
+  --><span class="status unsecure">http</span><!--
+  --><span class="status obsolete" data-i18n="3pExternalListObsolete"></span><!--
+  --><span class="status purge" data-i18n="3pExternalListPurge"></span><!--
+  --><span class="fa status updating">&#xf110;</span>
</li>
</ul>
</div>
diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json
index b31059c9..653cf75c 100644
--- a/src/_locales/en/messages.json
+++ b/src/_locales/en/messages.json
@@ -580,8 +580,8 @@
"description": "Message asking user to confirm reset"
},
"errorCantConnectTo":{
- "message":"Unable to connect to {{url}}",
- "description":"English: Network error: unable to connect to {{url}}"
+ "message":"Network error: {{msg}}",
+ "description":"English: Network error: {{msg}}"
},
"subscriberConfirm":{
"message":"uBlock₀: Add the following URL to your custom filter lists?\n\nTitle: \"{{title}}\"\nURL: {{url}}",
diff --git a/src/background.html b/src/background.html
index a8463068..0d56942f 100644
--- a/src/background.html
+++ b/src/background.html
@@ -8,7 +8,6 @@
<script src="js/polyfill.js"></script>
<script src="lib/punycode.js"></script>
<script src="lib/publicsuffixlist.js"></script>
-<script src="lib/yamd5.js"></script>
<script src="js/vapi-common.js"></script>
<script src="js/vapi-background.js"></script>
<script src="js/background.js"></script>
diff --git a/src/css/3p-filters.css b/src/css/3p-filters.css
index 99f87cfd..8aa6760f 100644
--- a/src/css/3p-filters.css
+++ b/src/css/3p-filters.css
@@ -1,3 +1,7 @@
+@keyframes spin {
+ 100% { transform: rotate(360deg); -webkit-transform: rotate(360deg); }
+ }
+
ul {
padding: 0;
list-style-type: none;
@@ -88,7 +92,7 @@ body[dir=rtl] #buttonApply {
span.status {
border: 1px solid transparent;
color: #444;
- display: inline-block;
+ display: none;
font-size: smaller;
line-height: 1;
margin: 0 0 0 0.5em;
@@ -99,6 +103,16 @@ span.unsecure {
background-color: hsl(0, 100%, 88%);
border-color: hsl(0, 100%, 83%);
}
+li.listEntry.unsecure span.unsecure {
+ display: inline;
+ }
+span.obsolete {
+ background-color: hsl(36, 100%, 80%);
+ border-color: hsl(36, 100%, 75%);
+ }
+li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete {
+ display: inline;
+ }
span.purge {
border-color: #ddd;
background-color: #eee;
@@ -107,10 +121,16 @@ span.purge {
span.purge:hover {
opacity: 1;
}
-span.obsolete,
-span.new {
- background-color: hsl(36, 100%, 80%);
- border-color: hsl(36, 100%, 75%);
+li.listEntry.cached span.purge {
+ display: inline;
+ }
+span.updating {
+ border: none;
+ padding: 0;
+ }
+li.listEntry.updating span.updating {
+ animation: spin 2s linear infinite;
+ display: inline-block;
}
#externalListsDiv {
margin: 2em auto 0 2em;
@@ -125,64 +145,3 @@ body[dir=rtl] #externalListsDiv {
width: 100%;
word-wrap: normal;
}
-body #busyOverlay {
- background-color: transparent;
- bottom: 0;
- cursor: wait;
- display: none;
- left: 0;
- position: fixed;
- right: 0;
- top: 0;
- z-index: 1000;
- }
-body.busy #busyOverlay {
- display: block;
- }
-#busyOverlay > div:nth-of-type(1) {
- background-color: white;
- bottom: 0;
- left: 0;
- opacity: 0.75;
- position: absolute;
- right: 0;
- top: 0;
- }
-#busyOverlay > div:nth-of-type(2) {
- background-color: #eee;
- border: 1px solid transparent;
- border-color: #80b3ff #80b3ff hsl(216, 100%, 75%);
- border-radius: 3px;
- box-sizing: border-box;
- height: 3em;
- left: 10%;
- position: absolute;
- bottom: 75%;
- width: 80%;
- }
-#busyOverlay > div:nth-of-type(2) > div:nth-of-type(1) {
- background-color: hsl(216, 100%, 75%);
- background-image: linear-gradient(#a8cbff, #80b3ff);
- background-repeat: repeat-x;
- border: 0;
- box-sizing: border-box;
- color: #222;
- height: 100%;
- left: 0;
- padding: 0;
- position: absolute;
- width: 25%;
- }
-#busyOverlay > div:nth-of-type(2) > div:nth-of-type(2) {
- background-color: transparent;
- border: 0;
- box-sizing: border-box;
- height: 100%;
- left: 0;
- line-height: 3em;
- overflow: hidden;
- position: absolute;
- text-align: center;
- top: 0;
- width: 100%;
- }
diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js
index 0b960e81..4b7dfa1b 100644
--- a/src/js/3p-filters.js
+++ b/src/js/3p-filters.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -21,38 +21,30 @@
/* global uDom */
+'use strict';
+
/******************************************************************************/
(function() {
-'use strict';
-
/******************************************************************************/
-var userListName = vAPI.i18n('1pPageName');
var listDetails = {};
var parseCosmeticFilters = true;
var ignoreGenericCosmeticFilters = false;
+var selectedListsHashBefore = '';
var externalLists = '';
-var cacheWasPurged = false;
-var needUpdate = false;
-var hasCachedContent = false;
/******************************************************************************/
var onMessage = function(msg) {
switch ( msg.what ) {
+ case 'assetUpdated':
+ updateAssetStatus(msg);
+ break;
case 'staticFilteringDataChanged':
renderFilterLists();
break;
-
- case 'forceUpdateAssetsProgress':
- renderBusyOverlay(true, msg.progress);
- if ( msg.done ) {
- messaging.send('dashboard', { what: 'reloadAllFilters' });
- }
- break;
-
default:
break;
}
@@ -69,20 +61,15 @@ var renderNumber = function(value) {
/******************************************************************************/
-// TODO: get rid of background page dependencies
-
var renderFilterLists = function() {
- var listGroupTemplate = uDom('#templates .groupEntry');
- var listEntryTemplate = uDom('#templates .listEntry');
- var listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats');
- var renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString;
- var lastUpdateString = vAPI.i18n('3pLastUpdate');
+ var listGroupTemplate = uDom('#templates .groupEntry'),
+ listEntryTemplate = uDom('#templates .listEntry'),
+ listStatsTemplate = vAPI.i18n('3pListsOfBlockedHostsPerListStats'),
+ renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString,
+ lastUpdateString = vAPI.i18n('3pLastUpdate');
- // Assemble a pretty blacklist name if possible
+ // Assemble a pretty list name if possible
var listNameFromListKey = function(listKey) {
- if ( listKey === listDetails.userFiltersPath ) {
- return userListName;
- }
var list = listDetails.current[listKey] || listDetails.available[listKey];
var listTitle = list ? list.title : '';
if ( listTitle === '' ) {
@@ -91,73 +78,68 @@ var renderFilterLists = function() {
return listTitle;
};
- var liFromListEntry = function(listKey) {
+ var liFromListEntry = function(listKey, li) {
var entry = listDetails.available[listKey];
- var li = listEntryTemplate.clone();
-
+ li = li ? li : listEntryTemplate.clone().nodeAt(0);
+ li.setAttribute('data-listkey', listKey);
+ var elem = li.querySelector('input[type="checkbox"]');
if ( entry.off !== true ) {
- li.descendants('input').attr('checked', '');
+ elem.setAttribute('checked', '');
+ } else {
+ elem.removeAttribute('checked');
}
-
- var elem = li.descendants('a:nth-of-type(1)');
- elem.attr('href', 'asset-viewer.html?url=' + encodeURI(listKey));
- elem.attr('type', 'text/html');
- elem.attr('data-listkey', listKey);
- elem.text(listNameFromListKey(listKey) + '\u200E');
-
+ elem = li.querySelector('a:nth-of-type(1)');
+ elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey));
+ elem.setAttribute('type', 'text/html');
+ elem.textContent = listNameFromListKey(listKey) + '\u200E';
+ elem = li.querySelector('a:nth-of-type(2)');
if ( entry.instructionURL ) {
- elem = li.descendants('a:nth-of-type(2)');
- elem.attr('href', entry.instructionURL);
- elem.css('display', '');
+ elem.setAttribute('href', entry.instructionURL);
+ elem.style.setProperty('display', '');
+ } else {
+ elem.style.setProperty('display', 'none');
}
-
+ elem = li.querySelector('a:nth-of-type(3)');
if ( entry.supportName ) {
- elem = li.descendants('a:nth-of-type(3)');
- elem.attr('href', entry.supportURL);
- elem.text('(' + entry.supportName + ')');
- elem.css('display', '');
+ elem.setAttribute('href', entry.supportURL);
+ elem.textContent = '(' + entry.supportName + ')';
+ elem.style.setProperty('display', '');
+ } else {
+ elem.style.setProperty('display', 'none');
}
-
- elem = li.descendants('span.counts');
+ elem = li.querySelector('span.counts');
var text = listStatsTemplate
.replace('{{used}}', renderNumber(!entry.off && !isNaN(+entry.entryUsedCount) ? entry.entryUsedCount : 0))
.replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?');
- elem.text(text);
-
- // https://github.com/gorhill/uBlock/issues/78
- // Badge for non-secure connection
- var remoteURL = listKey;
- if ( remoteURL.lastIndexOf('http:', 0) !== 0 ) {
- remoteURL = entry.homeURL || '';
- }
- if ( remoteURL.lastIndexOf('http:', 0) === 0 ) {
- li.descendants('span.status.unsecure').css('display', '');
- }
+ elem.textContent = text;
// https://github.com/chrisaljoudi/uBlock/issues/104
var asset = listDetails.cache[listKey] || {};
+ // https://github.com/gorhill/uBlock/issues/78
+ // Badge for non-secure connection
+ var remoteURL = asset.remoteURL;
+ li.classList.toggle(
+ 'unsecure',
+ typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0
+ );
// Badge for update status
- if ( entry.off !== true ) {
- if ( asset.repoObsolete ) {
- li.descendants('span.status.new').css('display', '');
- needUpdate = true;
- } else if ( asset.cacheObsolete ) {
- li.descendants('span.status.obsolete').css('display', '');
- needUpdate = true;
- } else if ( entry.external && !asset.cached ) {
- li.descendants('span.status.obsolete').css('display', '');
- needUpdate = true;
- }
- }
-
- // In cache
+ li.classList.toggle(
+ 'obsolete',
+ entry.off !== true && asset.obsolete === true
+ );
+ // Badge for cache status
+ li.classList.toggle(
+ 'cached',
+ asset.cached === true && asset.writeTime > 0
+ );
if ( asset.cached ) {
- elem = li.descendants('span.status.purge');
- elem.css('display', '');
- elem.attr('title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.lastModified)));
- hasCachedContent = true;
+ li.querySelector('.status.purge').setAttribute(
+ 'title',
+ lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime))
+ );
}
+ li.classList.remove('discard');
return li;
};
@@ -176,27 +158,31 @@ var renderFilterLists = function() {
};
var liFromListGroup = function(groupKey, listKeys) {
- var liGroup = listGroupTemplate.clone();
- var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1));
- if ( groupName !== '' ) {
- liGroup.descendants('span.geName').text(groupName);
- liGroup.descendants('span.geCount').text(listEntryCountFromGroup(listKeys));
+ var liGroup = document.querySelector('#lists > .groupEntry[data-groupkey="' + groupKey + '"]');
+ if ( liGroup === null ) {
+ liGroup = listGroupTemplate.clone().nodeAt(0);
+ var groupName = vAPI.i18n('3pGroup' + groupKey.charAt(0).toUpperCase() + groupKey.slice(1));
+ if ( groupName !== '' ) {
+ liGroup.querySelector('.geName').textContent = groupName;
+ }
}
- var ulGroup = liGroup.descendants('ul');
- if ( !listKeys ) {
- return liGroup;
+ if ( liGroup.querySelector('.geName:empty') === null ) {
+ liGroup.querySelector('.geCount').textContent = listEntryCountFromGroup(listKeys);
}
+ var ulGroup = liGroup.querySelector('.listEntries');
+ if ( !listKeys ) { return liGroup; }
listKeys.sort(function(a, b) {
return (listDetails.available[a].title || '').localeCompare(listDetails.available[b].title || '');
});
for ( var i = 0; i < listKeys.length; i++ ) {
- ulGroup.append(liFromListEntry(listKeys[i]));
+ var liEntry = liFromListEntry(listKeys[i], ulGroup.children[i]);
+ if ( liEntry.parentElement === null ) {
+ ulGroup.appendChild(liEntry);
+ }
}
return liGroup;
};
- // https://www.youtube.com/watch?v=unCVi4hYRlY#t=30m18s
-
var groupsFromLists = function(lists) {
var groups = {};
var listKeys = Object.keys(lists);
@@ -219,14 +205,16 @@ var renderFilterLists = function() {
listDetails = details;
parseCosmeticFilters = details.parseCosmeticFilters;
ignoreGenericCosmeticFilters = details.ignoreGenericCosmeticFilters;
- needUpdate = false;
- hasCachedContent = false;
+
+ // Incremental rendering: this will allow us to easily discard unused
+ // DOM list entries.
+ uDom('#lists .listEntries .listEntry').addClass('discard');
// Visually split the filter lists in purpose-based groups
- var ulLists = uDom('#lists').empty(), liGroup;
- var groups = groupsFromLists(details.available);
- var groupKey, i;
- var groupKeys = [
+ var ulLists = document.querySelector('#lists'),
+ groups = groupsFromLists(details.available),
+ liGroup, i, groupKey,
+ groupKeys = [
'default',
'ads',
'privacy',
@@ -239,31 +227,44 @@ var renderFilterLists = function() {
for ( i = 0; i < groupKeys.length; i++ ) {
groupKey = groupKeys[i];
liGroup = liFromListGroup(groupKey, groups[groupKey]);
- liGroup.toggleClass(
+ liGroup.setAttribute('data-groupkey', groupKey);
+ liGroup.classList.toggle(
'collapsed',
vAPI.localStorage.getItem('collapseGroup' + (i + 1)) === 'y'
);
- ulLists.append(liGroup);
+ if ( liGroup.parentElement === null ) {
+ ulLists.appendChild(liGroup);
+ }
delete groups[groupKey];
}
// For all groups not covered above (if any left)
groupKeys = Object.keys(groups);
for ( i = 0; i < groupKeys.length; i++ ) {
groupKey = groupKeys[i];
- ulLists.append(liFromListGroup(groupKey, groups[groupKey]));
+ ulLists.appendChild(liFromListGroup(groupKey, groups[groupKey]));
}
+ uDom('#lists .listEntries .listEntry.discard').remove();
+ uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null);
+ uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true);
+ uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true);
+ uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true);
uDom('#listsOfBlockedHostsPrompt').text(
vAPI.i18n('3pListsOfBlockedHostsPrompt')
.replace('{{netFilterCount}}', renderNumber(details.netFilterCount))
.replace('{{cosmeticFilterCount}}', renderNumber(details.cosmeticFilterCount))
);
- uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true);
- uDom('#parseCosmeticFilters').prop('checked', listDetails.parseCosmeticFilters === true);
- uDom('#ignoreGenericCosmeticFilters').prop('checked', listDetails.ignoreGenericCosmeticFilters === true);
+
+ // Compute a hash of the lists currently enabled in memory.
+ var selectedListsBefore = [];
+ for ( var key in listDetails.current ) {
+ if ( listDetails.current[key].off !== true ) {
+ selectedListsBefore.push(key);
+ }
+ }
+ selectedListsHashBefore = selectedListsBefore.sort().join();
renderWidgets();
- renderBusyOverlay(details.manualUpdate, details.manualUpdateProgress);
};
messaging.send('dashboard', { what: 'getLists' }, onListsReceived);
@@ -271,33 +272,22 @@ var renderFilterLists = function() {
/******************************************************************************/
-// Progress must be normalized to [0, 1], or can be undefined.
+// This is to give a visual hint that the selection of blacklists has changed.
-var renderBusyOverlay = function(state, progress) {
- progress = progress || {};
- var showProgress = typeof progress.value === 'number';
- if ( showProgress ) {
- uDom('#busyOverlay > div:nth-of-type(2) > div:first-child').css(
- 'width',
- (progress.value * 100).toFixed(1) + '%'
- );
- var text = progress.text || '';
- if ( text !== '' ) {
- uDom('#busyOverlay > div:nth-of-type(2) > div:last-child').text(text);
- }
- }
- uDom('#busyOverlay > div:nth-of-type(2)').css('display', showProgress ? '' : 'none');
- uDom('body').toggleClass('busy', !!state);
+var renderWidgets = function() {
+ uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged());
+ uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null);
+ uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('#lists .listEntry.obsolete') === null);
};
/******************************************************************************/
-// This is to give a visual hint that the selection of blacklists has changed.
-
-var renderWidgets = function() {
- uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged());
- uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged());
- uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent);
+var updateAssetStatus = function(details) {
+ var li = uDom('#lists .listEntry[data-listkey="' + details.key + '"]');
+ li.toggleClass('obsolete', !details.cached);
+ li.toggleClass('cached', details.cached);
+ li.removeClass('updating');
+ renderWidgets();
};
/******************************************************************************/
@@ -307,98 +297,49 @@ var renderWidgets = function() {
var listsSelectionChanged = function() {
if (
listDetails.parseCosmeticFilters !== parseCosmeticFilters ||
- listDetails.parseCosmeticFilters && listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters
+ listDetails.parseCosmeticFilters &&
+ listDetails.ignoreGenericCosmeticFilters !== ignoreGenericCosmeticFilters
) {
return true;
}
-
- if ( cacheWasPurged ) {
- return true;
- }
-
- var availableLists = listDetails.available;
- var currentLists = listDetails.current;
- var location, availableOff, currentOff;
-
- // This check existing entries
- for ( location in availableLists ) {
- if ( availableLists.hasOwnProperty(location) === false ) {
- continue;
- }
- availableOff = availableLists[location].off === true;
- currentOff = currentLists[location] === undefined || currentLists[location].off === true;
- if ( availableOff !== currentOff ) {
- return true;
- }
+ var selectedListsAfter = [],
+ listEntries = uDom('#lists .listEntry[data-listkey] > input[type="checkbox"]:checked');
+ for ( var i = 0, n = listEntries.length; i < n; i++ ) {
+ selectedListsAfter.push(listEntries.at(i).ancestors('.listEntry[data-listkey]').attr('data-listkey'));
}
- // This check removed entries
- for ( location in currentLists ) {
- if ( currentLists.hasOwnProperty(location) === false ) {
- continue;
- }
- currentOff = currentLists[location].off === true;
- availableOff = availableLists[location] === undefined || availableLists[location].off === true;
- if ( availableOff !== currentOff ) {
- return true;
- }
- }
-
- return false;
-};
-
-/******************************************************************************/
-
-// Return whether content need update.
-
-var listsContentChanged = function() {
- return needUpdate;
+ return selectedListsHashBefore !== selectedListsAfter.sort().join();
};
/******************************************************************************/
var onListCheckboxChanged = function() {
- var href = uDom(this).parent().descendants('a').first().attr('data-listkey');
- if ( typeof href !== 'string' ) {
- return;
- }
- if ( listDetails.available[href] === undefined ) {
- return;
- }
- listDetails.available[href].off = !this.checked;
renderWidgets();
};
/******************************************************************************/
var onPurgeClicked = function() {
- var button = uDom(this);
- var li = button.parent();
- var href = li.descendants('a').first().attr('data-listkey');
- if ( !href ) {
- return;
- }
+ var button = uDom(this),
+ liEntry = button.ancestors('[data-listkey]'),
+ listKey = liEntry.attr('data-listkey');
+ if ( !listKey ) { return; }
- messaging.send('dashboard', { what: 'purgeCache', path: href });
- button.remove();
+ messaging.send('dashboard', { what: 'purgeCache', assetKey: listKey });
// If the cached version is purged, the installed version must be assumed
// to be obsolete.
// https://github.com/gorhill/uBlock/issues/1733
// An external filter list must not be marked as obsolete, they will always
// be fetched anyways if there is no cached copy.
- var entry = listDetails.current && listDetails.current[href];
- if ( entry && entry.off !== true && /^[a-z]+:\/\//.test(href) === false ) {
- if ( typeof entry.homeURL !== 'string' || entry.homeURL === '' ) {
- li.descendants('span.status.new').css('display', '');
- } else {
- li.descendants('span.status.obsolete').css('display', '');
- }
- needUpdate = true;
+ var entry = listDetails.current && listDetails.current[listKey];
+ if ( entry && entry.off !== true ) {
+ liEntry.addClass('obsolete');
+ uDom('#buttonUpdate').removeClass('disabled');
}
+ liEntry.removeClass('cached');
- if ( li.descendants('input').first().prop('checked') ) {
- cacheWasPurged = true;
+ if ( liEntry.descendants('input').first().prop('checked') ) {
renderWidgets();
}
};
@@ -419,22 +360,21 @@ var selectFilterLists = function(callback) {
});
// Filter lists
- var switches = [];
- var lis = uDom('#lists .listEntry'), li;
- var i = lis.length;
+ var listKeys = [],
+ liEntries = uDom('#lists .listEntry'), liEntry,
+ i = liEntries.length;
while ( i-- ) {
- li = lis.at(i);
- switches.push({
- location: li.descendants('a').attr('data-listkey'),
- off: li.descendants('input').prop('checked') === false
- });
+ liEntry = liEntries.at(i);
+ if ( liEntry.descendants('input').first().prop('checked') ) {
+ listKeys.push(liEntry.attr('data-listkey'));
+ }
}
messaging.send(
'dashboard',
{
what: 'selectFilterLists',
- switches: switches
+ keys: listKeys
},
callback
);
@@ -444,49 +384,34 @@ var selectFilterLists = function(callback) {
var buttonApplyHandler = function() {
uDom('#buttonApply').removeClass('enabled');
-
- renderBusyOverlay(true);
-
var onSelectionDone = function() {
messaging.send('dashboard', { what: 'reloadAllFilters' });
};
-
selectFilterLists(onSelectionDone);
-
- cacheWasPurged = false;
};
/******************************************************************************/
var buttonUpdateHandler = function() {
- uDom('#buttonUpdate').removeClass('enabled');
-
- if ( needUpdate ) {
- renderBusyOverlay(true);
-
- var onSelectionDone = function() {
- messaging.send('dashboard', { what: 'forceUpdateAssets' });
- };
-
- selectFilterLists(onSelectionDone);
-
- cacheWasPurged = false;
- }
+ var onSelectionDone = function() {
+ uDom('#lists .listEntry.obsolete').addClass('updating');
+ messaging.send('dashboard', { what: 'forceUpdateAssets' });
+ };
+ selectFilterLists(onSelectionDone);
};
/******************************************************************************/
-var buttonPurgeAllHandler = function() {
+var buttonPurgeAllHandler = function(ev) {
uDom('#buttonPurgeAll').removeClass('enabled');
-
- renderBusyOverlay(true);
-
- var onCompleted = function() {
- cacheWasPurged = true;
- renderFilterLists();
- };
-
- messaging.send('dashboard', { what: 'purgeAllCaches' }, onCompleted);
+ messaging.send(
+ 'dashboard',
+ {
+ what: 'purgeAllCaches',
+ hard: ev.ctrlKey && ev.shiftKey
+ },
+ renderFilterLists
+ );
};
/******************************************************************************/
@@ -562,7 +487,7 @@ var groupEntryClickHandler = function() {
/******************************************************************************/
-var getCloudData = function() {
+var toCloudData = function() {
var bin = {
parseCosmeticFilters: uDom.nodeFromId('parseCosmeticFilters').checked,
ignoreGenericCosmeticFilters: uDom.nodeFromId('ignoreGenericCosmeticFilters').checked,
@@ -570,24 +495,22 @@ var getCloudData = function() {
externalLists: externalLists
};
- var lis = uDom('#lists .listEntry'), li;
- var i = lis.length;
+ var liEntries = uDom('#lists .listEntry'), liEntry;
+ var i = liEntries.length;
while ( i-- ) {
- li = lis.at(i);
- if ( li.descendants('input').prop('checked') ) {
- bin.selectedLists.push(li.descendants('a').attr('data-listkey'));
+ liEntry = liEntries.at(i);
+ if ( liEntry.descendants('input').prop('checked') ) {
+ bin.selectedLists.push(liEntry.attr('data-listkey'));
}
}
return bin;
};
-var setCloudData = function(data, append) {
- if ( typeof data !== 'object' || data === null ) {
- return;
- }
+var fromCloudData = function(data, append) {
+ if ( typeof data !== 'object' || data === null ) { return; }
- var elem, checked;
+ var elem, checked, i, n;
elem = uDom.nodeFromId('parseCosmeticFilters');
checked = data.parseCosmeticFilters === true || append && elem.checked;
@@ -597,30 +520,34 @@ var setCloudData = function(data, append) {
checked = data.ignoreGenericCosmeticFilters === true || append && elem.checked;
elem.checked = listDetails.ignoreGenericCosmeticFilters = checked;
- var lis = uDom('#lists .listEntry'), li, listKey;
- var i = lis.length;
- while ( i-- ) {
- li = lis.at(i);
- elem = li.descendants('input');
- listKey = li.descendants('a').attr('data-listkey');
- checked = data.selectedLists.indexOf(listKey) !== -1 ||
- append && elem.prop('checked');
- elem.prop('checked', checked);
- listDetails.available[listKey].off = !checked;
+ var listKey;
+ for ( i = 0, n = data.selectedLists.length; i < n; i++ ) {
+ listKey = data.selectedLists[i];
+ if ( listDetails.aliases[listKey] ) {
+ data.selectedLists[i] = listDetails.aliases[listKey];
+ }
+ }
+ var selectedSet = new Set(data.selectedLists),
+ listEntries = uDom('#lists .listEntry'),
+ listEntry, input;
+ for ( i = 0, n = listEntries.length; i < n; i++ ) {
+ listEntry = listEntries.at(i);
+ listKey = listEntry.attr('data-listkey');
+ input = listEntry.descendants('input').first();
+ if ( append && input.prop('checked') ) { continue; }
+ input.prop('checked', selectedSet.has(listKey) );
}
elem = uDom.nodeFromId('externalLists');
- if ( !append ) {
- elem.value = '';
- }
+ if ( !append ) { elem.value = ''; }
elem.value += data.externalLists || '';
renderWidgets();
externalListsChangeHandler();
};
-self.cloud.onPush = getCloudData;
-self.cloud.onPull = setCloudData;
+self.cloud.onPush = toCloudData;
+self.cloud.onPull = fromCloudData;
/******************************************************************************/
diff --git a/src/js/assets.js b/src/js/assets.js
index 88909948..f4daf8f2 100644
--- a/src/js/assets.js
+++ b/src/js/assets.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -19,311 +19,54 @@
Home: https://github.com/gorhill/uBlock
*/
-/* global YaMD5 */
-
'use strict';
-/*******************************************************************************
-
-File system structure:
- assets
- ublock
- ...
- thirdparties
- ...
- user
- filters.txt
- ...
-
-*/
-
/******************************************************************************/
-// Low-level asset files manager
-
µBlock.assets = (function() {
/******************************************************************************/
-var oneSecond = 1000;
-var oneMinute = 60 * oneSecond;
-var oneHour = 60 * oneMinute;
-var oneDay = 24 * oneHour;
-
-/******************************************************************************/
+var reIsExternalPath = /^(?:[a-z-]+):\/\//,
+ reIsUserAsset = /^user-/,
+ errorCantConnectTo = vAPI.i18n('errorCantConnectTo');
-var projectRepositoryRoot = µBlock.projectServerRoot;
-var assetsRepositoryRoot = 'https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/';
-var nullFunc = function() {};
-var reIsExternalPath = /^(file|ftps?|https?|resource):\/\//;
-var reIsUserPath = /^assets\/user\//;
-var reIsCachePath = /^cache:\/\//;
-var lastRepoMetaTimestamp = 0;
-var lastRepoMetaIsRemote = false;
-var refreshRepoMetaPeriod = 5 * oneHour;
-var errorCantConnectTo = vAPI.i18n('errorCantConnectTo');
-var xhrTimeout = vAPI.localStorage.getItem('xhrTimeout') || 30000;
-var onAssetRemovedListener = null;
-
-var exports = {
- autoUpdate: true,
- autoUpdateDelay: 4 * oneDay,
-
- // https://github.com/chrisaljoudi/uBlock/issues/426
- remoteFetchBarrier: 0
+var api = {
};
/******************************************************************************/
-var AssetEntry = function() {
- this.localChecksum = '';
- this.repoChecksum = '';
- this.expireTimestamp = 0;
-};
-
-var RepoMetadata = function() {
- this.entries = {};
- this.waiting = [];
-};
-
-var repoMetadata = null;
+var observers = [];
-// We need these to persist beyond repoMetaData
-var homeURLs = {};
-
-/******************************************************************************/
-
-var stringIsNotEmpty = function(s) {
- return typeof s === 'string' && s !== '';
+api.addObserver = function(observer) {
+ if ( observers.indexOf(observer) === -1 ) {
+ observers.push(observer);
+ }
};
-/******************************************************************************/
-
-var cacheIsObsolete = function(t) {
- return typeof t !== 'number' || (Date.now() - t) >= exports.autoUpdateDelay;
+api.removeObserver = function(observer) {
+ var pos;
+ while ( (pos = observers.indexOf(observer)) !== -1 ) {
+ observers.splice(pos, 1);
+ }
};
-/******************************************************************************/
-
-var cachedAssetsManager = (function() {
- var exports = {};
- var entries = null;
- var cachedAssetPathPrefix = 'cached_asset_content://';
-
- var getEntries = function(callback) {
- if ( entries !== null ) {
- callback(entries);
- return;
+var fireNotification = function(topic, details) {
+ var result;
+ for ( var i = 0; i < observers.length; i++ ) {
+ if ( observers[i](topic, details) === false ) {
+ result = false;
}
- // Flush cached non-user assets if these are from a prior version.
- // https://github.com/gorhill/httpswitchboard/issues/212
- var onLastVersionRead = function(store) {
- var currentVersion = vAPI.app.version;
- var lastVersion = store.extensionLastVersion || '0.0.0.0';
- if ( currentVersion !== lastVersion ) {
- vAPI.cacheStorage.set({ 'extensionLastVersion': currentVersion });
- }
- callback(entries);
- };
- var onLoaded = function(bin) {
- // https://github.com/gorhill/httpswitchboard/issues/381
- // Maybe the index was requested multiple times and already
- // fetched by one of the occurrences.
- if ( entries === null ) {
- var lastError = vAPI.lastError();
- if ( lastError ) {
- console.error(
- 'µBlock> cachedAssetsManager> getEntries():',
- lastError.message
- );
- }
- entries = bin.cached_asset_entries || {};
- }
- vAPI.cacheStorage.get('extensionLastVersion', onLastVersionRead);
- };
- vAPI.cacheStorage.get('cached_asset_entries', onLoaded);
- };
- exports.entries = getEntries;
-
- exports.load = function(path, cbSuccess, cbError) {
- cbSuccess = cbSuccess || nullFunc;
- cbError = cbError || cbSuccess;
- var details = {
- 'path': path,
- 'content': ''
- };
- var cachedContentPath = cachedAssetPathPrefix + path;
- var onLoaded = function(bin) {
- var lastError = vAPI.lastError();
- if ( lastError ) {
- details.error = 'Error: ' + lastError.message;
- console.error('µBlock> cachedAssetsManager.load():', details.error);
- cbError(details);
- return;
- }
- // Not sure how this can happen, but I've seen it happen. It could
- // be because the save occurred while I was stepping in the code
- // though, which means it would not occur during normal operation.
- // Still, just to be safe.
- if ( stringIsNotEmpty(bin[cachedContentPath]) === false ) {
- exports.remove(path);
- details.error = 'Error: not found';
- cbError(details);
- return;
- }
- details.content = bin[cachedContentPath];
- cbSuccess(details);
- };
- var onEntries = function(entries) {
- if ( entries[path] === undefined ) {
- details.error = 'Error: not found';
- cbError(details);
- return;
- }
- vAPI.cacheStorage.get(cachedContentPath, onLoaded);
- };
- getEntries(onEntries);
- };
-
- exports.save = function(path, content, cbSuccess, cbError) {
- cbSuccess = cbSuccess || nullFunc;
- cbError = cbError || cbSuccess;
- var details = {
- path: path,
- content: content
- };
- if ( content === '' ) {
- exports.remove(path);
- cbSuccess(details);
- return;
- }
- var cachedContentPath = cachedAssetPathPrefix + path;
- var bin = {};
- bin[cachedContentPath] = content;
- var removedItems = [];
- var onSaved = function() {
- var lastError = vAPI.lastError();
- if ( lastError ) {
- details.error = 'Error: ' + lastError.message;
- console.error('µBlock> cachedAssetsManager.save():', details.error);
- cbError(details);
- return;
- }
- // Saving over an existing item must be seen as removing an
- // existing item and adding a new one.
- if ( onAssetRemovedListener instanceof Function ) {
- onAssetRemovedListener(removedItems);
- }
- cbSuccess(details);
- };
- var onEntries = function(entries) {
- if ( entries.hasOwnProperty(path) ) {
- removedItems.push(path);
- }
- entries[path] = Date.now();
- bin.cached_asset_entries = entries;
- vAPI.cacheStorage.set(bin, onSaved);
- };
- getEntries(onEntries);
- };
-
- exports.remove = function(pattern, before) {
- var onEntries = function(entries) {
- var keystoRemove = [];
- var removedItems = [];
- var paths = Object.keys(entries);
- var i = paths.length;
- var path;
- while ( i-- ) {
- path = paths[i];
- if ( typeof pattern === 'string' && path !== pattern ) {
- continue;
- }
- if ( pattern instanceof RegExp && !pattern.test(path) ) {
- continue;
- }
- if ( typeof before === 'number' && entries[path] >= before ) {
- continue;
- }
- removedItems.push(path);
- keystoRemove.push(cachedAssetPathPrefix + path);
- delete entries[path];
- }
- if ( keystoRemove.length ) {
- vAPI.cacheStorage.remove(keystoRemove);
- vAPI.cacheStorage.set({ 'cached_asset_entries': entries });
- if ( onAssetRemovedListener instanceof Function ) {
- onAssetRemovedListener(removedItems);
- }
- }
- };
- getEntries(onEntries);
- };
-
- exports.removeAll = function(callback) {
- var onEntries = function() {
- // Careful! do not remove 'assets/user/'
- exports.remove(/^https?:\/\/[a-z0-9]+/);
- exports.remove(/^assets\/(ublock|thirdparties)\//);
- exports.remove(/^cache:\/\//);
- exports.remove('assets/checksums.txt');
- if ( typeof callback === 'function' ) {
- callback(null);
- }
- };
- getEntries(onEntries);
- };
-
- exports.rmrf = function() {
- exports.remove(/./);
- };
-
- exports.exists = function(path) {
- return entries !== null && entries.hasOwnProperty(path);
- };
-
- getEntries(function(){});
-
- return exports;
-})();
-
-/******************************************************************************/
-
-var toRepoURL = function(path) {
- if ( path.startsWith('assets/ublock/filter-lists.json') ) {
- return projectRepositoryRoot + path;
}
-
- if ( path.startsWith('assets/checksums.txt') ) {
- return path.replace(
- /^assets\/checksums.txt/,
- assetsRepositoryRoot + 'checksums/ublock0.txt'
- );
- }
-
- if ( path.startsWith('assets/thirdparties/') ) {
- return path.replace(
- /^assets\/thirdparties\//,
- assetsRepositoryRoot + 'thirdparties/'
- );
- }
-
- if ( path.startsWith('assets/ublock/') ) {
- return path.replace(
- /^assets\/ublock\//,
- assetsRepositoryRoot + 'filters/'
- );
- }
-
- // At this point, `path` is assumed to point to a resource specific to
- // this project.
- return projectRepositoryRoot + path;
+ return result;
};
/******************************************************************************/
var getTextFileFromURL = function(url, onLoad, onError) {
- // console.log('µBlock.assets/getTextFileFromURL("%s"):', url);
+ if ( reIsExternalPath.test(url) === false ) {
+ url = vAPI.getURL(url);
+ }
if ( typeof onError !== 'function' ) {
onError = onLoad;
@@ -353,6 +96,7 @@ var getTextFileFromURL = function(url, onLoad, onError) {
var onErrorReceived = function() {
this.onload = this.onerror = this.ontimeout = null;
+ µBlock.logger.writeOne('', 'error', errorCantConnectTo.replace('{{msg}}', ''));
onError.call(this);
};
@@ -362,7 +106,7 @@ var getTextFileFromURL = function(url, onLoad, onError) {
var xhr = new XMLHttpRequest();
try {
xhr.open('get', url, true);
- xhr.timeout = xhrTimeout;
+ xhr.timeout = µBlock.hiddenSettings.assetFetchTimeout * 1000 || 30000;
xhr.onload = onResponseReceived;
xhr.onerror = onErrorReceived;
xhr.ontimeout = onErrorReceived;
@@ -373,1289 +117,897 @@ var getTextFileFromURL = function(url, onLoad, onError) {
}
};
-/******************************************************************************/
+/*******************************************************************************
-var updateLocalChecksums = function() {
- var localChecksums = [];
- var entries = repoMetadata.entries;
- var entry;
- for ( var path in entries ) {
- if ( entries.hasOwnProperty(path) === false ) {
- continue;
- }
- entry = entries[path];
- if ( entry.localChecksum !== '' ) {
- localChecksums.push(entry.localChecksum + ' ' + path);
+ TODO(seamless migration):
+ This block of code will be removed when I am confident all users have
+ moved to a version of uBO which does not require the old way of caching
+ assets.
+
+ api.listKeyAliases: a map of old asset keys to new asset keys.
+
+ migrate(): to seamlessly migrate the old cache manager to the new one:
+ - attempt to preserve and move content of cached assets to new locations;
+ - removes all traces of now obsolete cache manager entries in cacheStorage.
+
+ This code will typically execute only once, when the newer version of uBO
+ is first installed and executed.
+
+**/
+
+api.listKeyAliases = {
+ "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat",
+ "assets/user/filters.txt": "user-filters",
+ "assets/ublock/resources.txt": "ublock-resources",
+ "assets/ublock/filters.txt": "ublock-filters",
+ "assets/ublock/privacy.txt": "ublock-privacy",
+ "assets/ublock/unbreak.txt": "ublock-unbreak",
+ "assets/ublock/badware.txt": "ublock-badware",
+ "assets/ublock/experimental.txt": "ublock-experimental",
+ "https://easylist-downloads.adblockplus.org/easylistchina.txt": "CHN-0",
+ "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "CHN-1",
+ "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjx-annoyance.txt": "CHN-2",
+ "https://easylist-downloads.adblockplus.org/easylistgermany.txt": "DEU-0",
+ "https://adblock.dk/block.csv": "DNK-0",
+ "assets/thirdparties/easylist-downloads.adblockplus.org/easylist.txt": "easylist",
+ "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "easylist-nocosmetic",
+ "assets/thirdparties/easylist-downloads.adblockplus.org/easyprivacy.txt": "easyprivacy",
+ "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "fanboy-annoyance",
+ "https://easylist-downloads.adblockplus.org/fanboy-social.txt": "fanboy-social",
+ "https://easylist-downloads.adblockplus.org/liste_fr.txt": "FRA-0",
+ "http://adblock.gardar.net/is.abp.txt": "ISL-0",
+ "https://easylist-downloads.adblockplus.org/easylistitaly.txt": "ITA-0",
+ "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "ITA-1",
+ "https://easylist-downloads.adblockplus.org/advblock.txt": "RUS-0",
+ "https://easylist-downloads.adblockplus.org/bitblock.txt": "RUS-1",
+ "https://filters.adtidy.org/extension/chromium/filters/1.txt": "RUS-2",
+ "https://adguard.com/en/filter-rules.html?id=1": "RUS-2",
+ "https://easylist-downloads.adblockplus.org/easylistdutch.txt": "NLD-0",
+ "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt": "LVA-0",
+ "http://hosts-file.net/.%5Cad_servers.txt": "hphosts",
+ "http://adblock.ee/list.php": "EST-0",
+ "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt": "disconnect-malvertising",
+ "https://s3.amazonaws.com/lists.disconnect.me/simple_malware.txt": "disconnect-malware",
+ "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt": "disconnect-tracking",
+ "https://www.certyficate.it/adblock/adblock.txt": "POL-0",
+ "https://easylist-downloads.adblockplus.org/antiadblockfilters.txt": "awrl-0",
+ "http://adb.juvander.net/Finland_adb.txt": "FIN-0",
+ "https://raw.githubusercontent.com/gfmaster/adblock-korea-contrib/master/filter.txt": "KOR-0",
+ "https://raw.githubusercontent.com/yous/YousList/master/youslist.txt": "KOR-1",
+ "https://www.fanboy.co.nz/fanboy-korean.txt": "KOR-2",
+ "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "IDN-0",
+ "https://raw.githubusercontent.com/k2jp/abp-japanese-filters/master/abpjf.txt": "JPN-0",
+ "https://raw.githubusercontent.com/liamja/Prebake/master/obtrusive.txt": "EU-prebake",
+ "https://easylist-downloads.adblockplus.org/Liste_AR.txt": "ara-0",
+ "http://margevicius.lt/easylistlithuania.txt": "LTU-0",
+ "http://malwaredomains.lehigh.edu/files/immortal_domains.txt": "malware-0",
+ "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-1",
+ "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-2",
+ "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0",
+ "https://raw.githubusercontent.com/easylist/EasyListHebrew/master/EasyListHebrew.txt": "ISR-0",
+ "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "reek-0",
+ "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "HUN-0",
+ "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "CZE-0",
+ "http://someonewhocares.org/hosts/hosts": "dpollock-0",
+ "https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt": "spam404-0",
+ "http://stanev.org/abp/adblock_bg.txt": "BGR-0",
+ "http://winhelp2002.mvps.org/hosts.txt": "mvps-0",
+ "https://www.fanboy.co.nz/enhancedstats.txt": "fanboy-enhanced",
+ "https://www.fanboy.co.nz/fanboy-antifacebook.txt": "fanboy-thirdparty_social",
+ "https://easylist-downloads.adblockplus.org/easylistspanish.txt": "spa-0",
+ "https://www.fanboy.co.nz/fanboy-swedish.txt": "SWE-0",
+ "https://www.fanboy.co.nz/r/fanboy-ultimate.txt": "fanboy-ultimate",
+ "https://filters.adtidy.org/extension/chromium/filters/13.txt": "TUR-0",
+ "https://adguard.com/filter-rules.html?id=13": "TUR-0",
+ "https://www.fanboy.co.nz/fanboy-vietnam.txt": "VIE-0",
+ "https://www.void.gr/kargig/void-gr-filters.txt": "GRC-0",
+ "https://raw.githubusercontent.com/betterwebleon/slovenian-list/master/filters.txt": "SVN-0"
+};
+
+var migrate = function(callback) {
+ var entries,
+ moveCount = 0,
+ toRemove = [];
+
+ var countdown = function(change) {
+ moveCount -= (change || 0);
+ if ( moveCount !== 0 ) { return; }
+ vAPI.cacheStorage.remove(toRemove);
+ saveAssetCacheRegistry();
+ callback();
+ };
+
+ var onContentRead = function(oldKey, newKey, bin) {
+ var content = bin && bin['cached_asset_content://' + oldKey] || undefined;
+ if ( content ) {
+ assetCacheRegistry[newKey] = {
+ readTime: Date.now(),
+ writeTime: entries[oldKey]
+ };
+ if ( reIsExternalPath.test(oldKey) ) {
+ assetCacheRegistry[newKey].remoteURL = oldKey;
+ }
+ bin = {};
+ bin['cache/' + newKey] = content;
+ vAPI.cacheStorage.set(bin);
}
- }
- cachedAssetsManager.save('assets/checksums.txt', localChecksums.join('\n'));
-};
+ countdown(1);
+ };
-/******************************************************************************/
+ var onEntries = function(bin) {
+ entries = bin && bin['cached_asset_entries'];
+ if ( !entries ) { return callback(); }
+ if ( bin && bin['assetCacheRegistry'] ) {
+ assetCacheRegistry = bin['assetCacheRegistry'];
+ }
+ var aliases = api.listKeyAliases;
+ for ( var oldKey in entries ) {
+ if ( oldKey.endsWith('assets/user/filters.txt') ) { continue; }
+ var newKey = aliases[oldKey];
+ if ( !newKey && /^https?:\/\//.test(oldKey) ) {
+ newKey = oldKey;
+ }
+ if ( newKey ) {
+ vAPI.cacheStorage.get(
+ 'cached_asset_content://' + oldKey,
+ onContentRead.bind(null, oldKey, newKey)
+ );
+ moveCount += 1;
+ }
+ toRemove.push('cached_asset_content://' + oldKey);
+ }
+ toRemove.push('cached_asset_entries', 'extensionLastVersion');
+ countdown();
+ };
-// Gather meta data of all assets.
+ vAPI.cacheStorage.get(
+ [ 'cached_asset_entries', 'assetCacheRegistry' ],
+ onEntries
+ );
+};
-var getRepoMetadata = function(callback) {
- callback = callback || nullFunc;
+/*******************************************************************************
- // https://github.com/chrisaljoudi/uBlock/issues/515
- // Handle re-entrancy here, i.e. we MUST NOT tamper with the waiting list
- // of callers, if any, except to add one at the end of the list.
- if ( repoMetadata !== null && repoMetadata.waiting.length !== 0 ) {
- repoMetadata.waiting.push(callback);
- return;
+ The purpose of the asset source registry is to keep key detail information
+ about an asset:
+ - Where to load it from: this may consist of one or more URLs, either local
+ or remote.
+ - After how many days an asset should be deemed obsolete -- i.e. in need of
+ an update.
+ - The origin and type of an asset.
+ - The last time an asset was registered.
+
+**/
+
+var assetSourceRegistryStatus,
+ assetSourceRegistry = Object.create(null);
+
+var registerAssetSource = function(assetKey, dict) {
+ var entry = assetSourceRegistry[assetKey] || {};
+ for ( var prop in dict ) {
+ if ( dict.hasOwnProperty(prop) === false ) { continue; }
+ if ( dict[prop] === undefined ) {
+ delete entry[prop];
+ } else {
+ entry[prop] = dict[prop];
+ }
}
-
- if ( exports.remoteFetchBarrier === 0 && lastRepoMetaIsRemote === false ) {
- lastRepoMetaTimestamp = 0;
+ var contentURL = dict.contentURL;
+ if ( contentURL !== undefined ) {
+ if ( typeof contentURL === 'string' ) {
+ contentURL = entry.contentURL = [ contentURL ];
+ } else if ( Array.isArray(contentURL) === false ) {
+ contentURL = entry.contentURL = [];
+ }
+ var remoteURLCount = 0;
+ for ( var i = 0; i < contentURL.length; i++ ) {
+ if ( reIsExternalPath.test(contentURL[i]) ) {
+ remoteURLCount += 1;
+ }
+ }
+ entry.hasLocalURL = remoteURLCount !== contentURL.length;
+ entry.hasRemoteURL = remoteURLCount !== 0;
+ } else if ( entry.contentURL === undefined ) {
+ entry.contentURL = [];
}
- if ( (Date.now() - lastRepoMetaTimestamp) >= refreshRepoMetaPeriod ) {
- repoMetadata = null;
+ if ( typeof entry.updateAfter !== 'number' ) {
+ entry.updateAfter = 5;
}
- if ( repoMetadata !== null ) {
- callback(repoMetadata);
- return;
+ if ( entry.submitter ) {
+ entry.submitTime = Date.now(); // To detect stale entries
}
+ assetSourceRegistry[assetKey] = entry;
+};
- lastRepoMetaTimestamp = Date.now();
- lastRepoMetaIsRemote = exports.remoteFetchBarrier === 0;
-
- var defaultChecksums;
- var localChecksums;
- var repoChecksums;
+var unregisterAssetSource = function(assetKey) {
+ assetCacheRemove(assetKey);
+ delete assetSourceRegistry[assetKey];
+};
- var checksumsReceived = function() {
- if (
- defaultChecksums === undefined ||
- localChecksums === undefined ||
- repoChecksums === undefined
- ) {
- return;
- }
- // Remove from cache assets which no longer exist in the repo
- var entries = repoMetadata.entries;
- var checksumsChanged = false;
- var entry;
- for ( var path in entries ) {
- if ( entries.hasOwnProperty(path) === false ) {
- continue;
- }
- entry = entries[path];
- // https://github.com/gorhill/uBlock/issues/760
- // If the resource does not have a cached instance, we must reset
- // the checksum to its value at install time.
- if (
- stringIsNotEmpty(defaultChecksums[path]) &&
- entry.localChecksum !== defaultChecksums[path] &&
- cachedAssetsManager.exists(path) === false
- ) {
- entry.localChecksum = defaultChecksums[path];
- checksumsChanged = true;
- }
- // If repo checksums could not be fetched, assume no change.
- // https://github.com/gorhill/uBlock/issues/602
- // Added: if repo checksum is that of the empty string,
- // assume no change
- if (
- repoChecksums === '' ||
- entry.repoChecksum === 'd41d8cd98f00b204e9800998ecf8427e'
- ) {
- entry.repoChecksum = entry.localChecksum;
- }
- if ( entry.repoChecksum !== '' || entry.localChecksum === '' ) {
- continue;
- }
- checksumsChanged = true;
- cachedAssetsManager.remove(path);
- entry.localChecksum = '';
- }
- if ( checksumsChanged ) {
- updateLocalChecksums();
- }
- // Notify all waiting callers
- // https://github.com/chrisaljoudi/uBlock/issues/515
- // VERY IMPORTANT: because of re-entrancy, we MUST:
- // - process the waiting callers in a FIFO manner
- // - not cache repoMetadata.waiting.length, we MUST use the live
- // value, because it can change while looping
- // - not change the waiting list until they are all processed
- for ( var i = 0; i < repoMetadata.waiting.length; i++ ) {
- repoMetadata.waiting[i](repoMetadata);
- }
- repoMetadata.waiting.length = 0;
+var saveAssetSourceRegistry = (function() {
+ var timer;
+ var save = function() {
+ timer = undefined;
+ vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry });
};
-
- var validateChecksums = function(details) {
- if ( details.error || details.content === '' ) {
- return '';
+ return function(lazily) {
+ if ( timer !== undefined ) {
+ clearTimeout(timer);
}
- if ( /^(?:[0-9a-f]{32}\s+\S+(?:\s+|$))+/.test(details.content) === false ) {
- return '';
- }
- // https://github.com/gorhill/uBlock/issues/602
- // External filter lists are not meant to appear in checksums.txt.
- // TODO: remove this code once v1.1.0.0 is everywhere.
- var out = [];
- var listMap = µBlock.oldListToNewListMap;
- var lines = details.content.split(/\s*\n\s*/);
- var line, matches;
- for ( var i = 0; i < lines.length; i++ ) {
- line = lines[i];
- matches = line.match(/^[0-9a-f]+ (.+)$/);
- if ( matches === null || listMap.hasOwnProperty(matches[1]) ) {
- continue;
- }
- out.push(line);
+ if ( lazily ) {
+ timer = vAPI.setTimeout(save, 500);
+ } else {
+ save();
}
- return out.join('\n');
};
+})();
- var parseChecksums = function(text, eachFn) {
- var lines = text.split(/\n+/);
- var i = lines.length;
- var fields;
- while ( i-- ) {
- fields = lines[i].trim().split(/\s+/);
- if ( fields.length !== 2 ) {
- continue;
- }
- eachFn(fields[1], fields[0]);
- }
- };
+var updateAssetSourceRegistry = function(json) {
+ var newDict;
+ try {
+ newDict = JSON.parse(json);
+ } catch (ex) {
+ }
+ if ( newDict instanceof Object === false ) { return; }
- var onLocalChecksumsLoaded = function(details) {
- var entries = repoMetadata.entries;
- var processChecksum = function(path, checksum) {
- if ( entries.hasOwnProperty(path) === false ) {
- entries[path] = new AssetEntry();
+ getAssetSourceRegistry(function(oldDict) {
+ var assetKey;
+ // Remove obsolete entries
+ for ( assetKey in oldDict ) {
+ if ( newDict[assetKey] === undefined ) {
+ unregisterAssetSource(assetKey);
}
- entries[path].localChecksum = checksum;
- };
- if ( (localChecksums = validateChecksums(details)) ) {
- parseChecksums(localChecksums, processChecksum);
}
- checksumsReceived();
- };
-
- var onRepoChecksumsLoaded = function(details) {
- var entries = repoMetadata.entries;
- var processChecksum = function(path, checksum) {
- if ( entries.hasOwnProperty(path) === false ) {
- entries[path] = new AssetEntry();
- }
- entries[path].repoChecksum = checksum;
- };
- if ( (repoChecksums = validateChecksums(details)) ) {
- parseChecksums(repoChecksums, processChecksum);
+ // Add/update existing entries
+ for ( assetKey in newDict ) {
+ registerAssetSource(assetKey, newDict[assetKey]);
}
- checksumsReceived();
- };
-
- // https://github.com/gorhill/uBlock/issues/760
- // We need the checksum values at install time, because some resources
- // may have been purged, in which case the checksum must be reset to the
- // value at install time.
- var onDefaultChecksumsLoaded = function() {
- defaultChecksums = Object.create(null);
- var processChecksum = function(path, checksum) {
- defaultChecksums[path] = checksum;
- };
- parseChecksums(this.responseText || '', processChecksum);
- checksumsReceived();
- };
-
- repoMetadata = new RepoMetadata();
- repoMetadata.waiting.push(callback);
- readRepoFile('assets/checksums.txt', onRepoChecksumsLoaded);
- getTextFileFromURL(vAPI.getURL('assets/checksums.txt'), onDefaultChecksumsLoaded);
- readLocalFile('assets/checksums.txt', onLocalChecksumsLoaded);
+ saveAssetSourceRegistry();
+ });
};
-// https://www.youtube.com/watch?v=-t3WYfgM4x8
-
-/******************************************************************************/
-
-exports.setHomeURL = function(path, homeURL) {
- if ( typeof homeURL !== 'string' || homeURL === '' ) {
+var getAssetSourceRegistry = function(callback) {
+ // Already loaded.
+ if ( assetSourceRegistryStatus === 'ready' ) {
+ callback(assetSourceRegistry);
return;
}
- homeURLs[path] = homeURL;
-};
-/******************************************************************************/
+ // Being loaded.
+ if ( Array.isArray(assetSourceRegistryStatus) ) {
+ assetSourceRegistryStatus.push(callback);
+ return;
+ }
-// Get a local asset, do not look-up repo or remote location if local asset
-// is not found.
+ // Not loaded: load it.
+ assetSourceRegistryStatus = [ callback ];
-var readLocalFile = function(path, callback) {
- var reportBack = function(content, err) {
- var details = {
- 'path': path,
- 'content': content
- };
- if ( err ) {
- details.error = err;
+ var registryReady = function() {
+ var callers = assetSourceRegistryStatus;
+ assetSourceRegistryStatus = 'ready';
+ var fn;
+ while ( (fn = callers.shift()) ) {
+ fn(assetSourceRegistry);
}
- callback(details);
- };
-
- var onInstallFileLoaded = function() {
- //console.log('µBlock> readLocalFile("%s") / onInstallFileLoaded()', path);
- reportBack(this.responseText);
- };
-
- var onInstallFileError = function() {
- console.error('µBlock> readLocalFile("%s") / onInstallFileError()', path);
- reportBack('', 'Error');
};
- var onCachedContentLoaded = function(details) {
- //console.log('µBlock> readLocalFile("%s") / onCachedContentLoaded()', path);
- reportBack(details.content);
+ // First-install case.
+ var createRegistry = function() {
+ getTextFileFromURL(
+ µBlock.assetsBootstrapLocation || 'assets/assets.json',
+ function() {
+ updateAssetSourceRegistry(this.responseText);
+ registryReady();
+ }
+ );
};
- var onCachedContentError = function(details) {
- //console.error('µBlock> readLocalFile("%s") / onCachedContentError()', path);
- if ( reIsExternalPath.test(path) ) {
- reportBack('', 'Error: asset not found');
- return;
- }
- // It's ok for user data to not be found
- if ( reIsUserPath.test(path) ) {
- reportBack('');
+ vAPI.cacheStorage.get('assetSourceRegistry', function(bin) {
+ if ( !bin || !bin.assetSourceRegistry ) {
+ createRegistry();
return;
}
- getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError);
- };
+ assetSourceRegistry = bin.assetSourceRegistry;
+ registryReady();
+ });
+};
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
+api.registerAssetSource = function(assetKey, details) {
+ getAssetSourceRegistry(function() {
+ registerAssetSource(assetKey, details);
+ saveAssetSourceRegistry(true);
+ });
};
-// https://www.youtube.com/watch?v=r9KVpuFPtHc
+api.unregisterAssetSource = function(assetKey) {
+ getAssetSourceRegistry(function() {
+ unregisterAssetSource(assetKey);
+ saveAssetSourceRegistry(true);
+ });
+};
-/******************************************************************************/
+/*******************************************************************************
-// Get the repository copy of a built-in asset.
+ The purpose of the asset cache registry is to keep track of all assets
+ which have been persisted into the local cache.
-var readRepoFile = function(path, callback) {
- // https://github.com/chrisaljoudi/uBlock/issues/426
- if ( exports.remoteFetchBarrier !== 0 ) {
- readLocalFile(path, callback);
+**/
+
+var assetCacheRegistryStatus,
+ assetCacheRegistryStartTime = Date.now(),
+ assetCacheRegistry = {};
+
+var getAssetCacheRegistry = function(callback) {
+ // Already loaded.
+ if ( assetCacheRegistryStatus === 'ready' ) {
+ callback(assetCacheRegistry);
return;
}
- var reportBack = function(content, err) {
- var details = {
- 'path': path,
- 'content': content,
- 'error': err
- };
- callback(details);
- };
+ // Being loaded.
+ if ( Array.isArray(assetCacheRegistryStatus) ) {
+ assetCacheRegistryStatus.push(callback);
+ return;
+ }
- var repositoryURL = toRepoURL(path);
+ // Not loaded: load it.
+ assetCacheRegistryStatus = [ callback ];
- var onRepoFileLoaded = function() {
- //console.log('µBlock> readRepoFile("%s") / onRepoFileLoaded()', path);
- // https://github.com/gorhill/httpswitchboard/issues/263
- if ( this.status === 200 ) {
- reportBack(this.responseText);
- } else {
- reportBack('', 'Error: ' + this.statusText);
+ var registryReady = function() {
+ var callers = assetCacheRegistryStatus;
+ assetCacheRegistryStatus = 'ready';
+ var fn;
+ while ( (fn = callers.shift()) ) {
+ fn(assetCacheRegistry);
}
};
- var onRepoFileError = function() {
- console.error(errorCantConnectTo.replace('{{url}}', repositoryURL));
- reportBack('', 'Error');
+ var migrationDone = function() {
+ vAPI.cacheStorage.get('assetCacheRegistry', function(bin) {
+ if ( bin && bin.assetCacheRegistry ) {
+ assetCacheRegistry = bin.assetCacheRegistry;
+ }
+ registryReady();
+ });
};
- // '_=...' is to skip browser cache
- getTextFileFromURL(
- repositoryURL + '?_=' + Date.now(),
- onRepoFileLoaded,
- onRepoFileError
- );
+ migrate(migrationDone);
};
-/******************************************************************************/
-
-// An asset from an external source with a copy shipped with the extension:
-// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL
-// External -->
-// Repository --> has checksum (to detect need for update only)
-// Cache --> has expiration timestamp (in cache)
-// Local --> install time version
+var saveAssetCacheRegistry = (function() {
+ var timer;
+ var save = function() {
+ timer = undefined;
+ vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry });
+ };
+ return function(lazily) {
+ if ( timer !== undefined ) { clearTimeout(timer); }
+ if ( lazily ) {
+ timer = vAPI.setTimeout(save, 500);
+ } else {
+ save();
+ }
+ };
+})();
-var readRepoCopyAsset = function(path, callback) {
- var assetEntry;
- var homeURL = homeURLs[path];
+var assetCacheRead = function(assetKey, callback) {
+ var internalKey = 'cache/' + assetKey;
var reportBack = function(content, err) {
- var details = {
- 'path': path,
- 'content': content
- };
- if ( err ) {
- details.error = err;
- }
+ var details = { assetKey: assetKey, content: content };
+ if ( err ) { details.error = err; }
callback(details);
};
- var updateChecksum = function() {
- if ( assetEntry !== undefined && assetEntry.repoChecksum !== assetEntry.localChecksum ) {
- assetEntry.localChecksum = assetEntry.repoChecksum;
- updateLocalChecksums();
+ var onAssetRead = function(bin) {
+ if ( !bin || !bin[internalKey] ) {
+ return reportBack('', 'E_NOTFOUND');
}
+ var entry = assetCacheRegistry[assetKey];
+ if ( entry === undefined ) {
+ return reportBack('', 'E_NOTFOUND');
+ }
+ entry.readTime = Date.now();
+ saveAssetCacheRegistry(true);
+ reportBack(bin[internalKey]);
};
- var onInstallFileLoaded = function() {
- //console.log('µBlock> readRepoCopyAsset("%s") / onInstallFileLoaded()', path);
- reportBack(this.responseText);
- };
-
- var onInstallFileError = function() {
- console.error('µBlock> readRepoCopyAsset("%s") / onInstallFileError():', path, this.statusText);
- reportBack('', 'Error');
+ var onReady = function() {
+ vAPI.cacheStorage.get(internalKey, onAssetRead);
};
- var onCachedContentLoaded = function(details) {
- //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileLoaded()', path);
- reportBack(details.content);
- };
+ getAssetCacheRegistry(onReady);
+};
- var onCachedContentError = function(details) {
- //console.log('µBlock> readRepoCopyAsset("%s") / onCacheFileError()', path);
- getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError);
- };
+var assetCacheWrite = function(assetKey, details, callback) {
+ var internalKey = 'cache/' + assetKey;
+ var content = '';
+ if ( typeof details === 'string' ) {
+ content = details;
+ } else if ( details instanceof Object ) {
+ content = details.content || '';
+ }
- var repositoryURL = toRepoURL(path);
- var repositoryURLSkipCache = repositoryURL + '?_=' + Date.now();
+ if ( content === '' ) {
+ return assetCacheRemove(assetKey, callback);
+ }
- var onRepoFileLoaded = function() {
- if ( stringIsNotEmpty(this.responseText) === false ) {
- console.error('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s"): error', path, repositoryURL);
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- return;
+ var reportBack = function(content) {
+ var details = { assetKey: assetKey, content: content };
+ if ( typeof callback === 'function' ) {
+ callback(details);
}
- //console.log('µBlock> readRepoCopyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL);
- updateChecksum();
- cachedAssetsManager.save(path, this.responseText, callback);
+ fireNotification('after-asset-updated', details);
};
- var onRepoFileError = function() {
- console.error(errorCantConnectTo.replace('{{url}}', repositoryURL));
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- };
-
- var onHomeFileLoaded = function() {
- if ( stringIsNotEmpty(this.responseText) === false ) {
- console.error('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s"): no response', path, homeURL);
- // Fetch from repo only if obsolescence was due to repo checksum
- if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) {
- getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError);
- } else {
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- }
- return;
+ var onReady = function() {
+ var entry = assetCacheRegistry[assetKey];
+ if ( entry === undefined ) {
+ entry = assetCacheRegistry[assetKey] = {};
}
- //console.log('µBlock> readRepoCopyAsset("%s") / onHomeFileLoaded("%s")', path, homeURL);
- updateChecksum();
- cachedAssetsManager.save(path, this.responseText, callback);
- };
-
- var onHomeFileError = function() {
- console.error(errorCantConnectTo.replace('{{url}}', homeURL));
- // Fetch from repo only if obsolescence was due to repo checksum
- if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) {
- getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError);
- } else {
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
+ entry.writeTime = entry.readTime = Date.now();
+ if ( details instanceof Object && typeof details.url === 'string' ) {
+ entry.remoteURL = details.url;
}
+ var bin = { assetCacheRegistry: assetCacheRegistry };
+ bin[internalKey] = content;
+ vAPI.cacheStorage.set(bin);
+ reportBack(content);
};
+ getAssetCacheRegistry(onReady);
+};
- var onCacheMetaReady = function(entries) {
- // Fetch from remote if:
- // - Auto-update enabled AND (not in cache OR in cache but obsolete)
- var timestamp = entries[path];
- var inCache = typeof timestamp === 'number';
- if (
- exports.remoteFetchBarrier === 0 &&
- exports.autoUpdate && stringIsNotEmpty(homeURL)
- ) {
- if ( inCache === false || cacheIsObsolete(timestamp) ) {
- //console.log('µBlock> readRepoCopyAsset("%s") / onCacheMetaReady(): not cached or obsolete', path);
- getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError);
- return;
+var assetCacheRemove = function(pattern, callback) {
+ var onReady = function() {
+ var cacheDict = assetCacheRegistry,
+ removedEntries = [],
+ removedContent = [];
+ for ( var assetKey in cacheDict ) {
+ if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
+ continue;
+ }
+ if ( typeof pattern === 'string' && assetKey !== pattern ) {
+ continue;
}
+ removedEntries.push(assetKey);
+ removedContent.push('cache/' + assetKey);
+ delete cacheDict[assetKey];
}
-
- // In cache
- if ( inCache ) {
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- return;
+ if ( removedContent.length !== 0 ) {
+ vAPI.cacheStorage.remove(removedContent);
+ var bin = { assetCacheRegistry: assetCacheRegistry };
+ vAPI.cacheStorage.set(bin);
+ }
+ if ( typeof callback === 'function' ) {
+ callback();
+ }
+ for ( var i = 0; i < removedEntries.length; i++ ) {
+ fireNotification('after-asset-updated', { assetKey: removedEntries[i] });
}
-
- // Not in cache
- getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError);
};
- var onRepoMetaReady = function(meta) {
- assetEntry = meta.entries[path];
-
- // Asset doesn't exist
- if ( assetEntry === undefined ) {
- reportBack('', 'Error: asset not found');
- return;
- }
+ getAssetCacheRegistry(onReady);
+};
- // Repo copy changed: fetch from home URL
- if (
- exports.remoteFetchBarrier === 0 &&
- exports.autoUpdate &&
- assetEntry.localChecksum !== assetEntry.repoChecksum
- ) {
- //console.log('µBlock> readRepoCopyAsset("%s") / onRepoMetaReady(): repo has newer version', path);
- if ( stringIsNotEmpty(homeURL) ) {
- getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError);
- } else {
- getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError);
+var assetCacheMarkAsDirty = function(pattern, callback) {
+ var onReady = function() {
+ var cacheDict = assetCacheRegistry,
+ cacheEntry,
+ mustSave = false;
+ for ( var assetKey in cacheDict ) {
+ if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
+ continue;
}
- return;
+ if ( typeof pattern === 'string' && assetKey !== pattern ) {
+ continue;
+ }
+ cacheEntry = cacheDict[assetKey];
+ if ( !cacheEntry.writeTime ) { continue; }
+ cacheDict[assetKey].writeTime = 0;
+ mustSave = true;
+ }
+ if ( mustSave ) {
+ var bin = { assetCacheRegistry: assetCacheRegistry };
+ vAPI.cacheStorage.set(bin);
+ }
+ if ( typeof callback === 'function' ) {
+ callback();
}
-
- // Load from cache
- cachedAssetsManager.entries(onCacheMetaReady);
};
- getRepoMetadata(onRepoMetaReady);
+ getAssetCacheRegistry(onReady);
};
-// https://www.youtube.com/watch?v=uvUW4ozs7pY
-
/******************************************************************************/
-// An important asset shipped with the extension -- typically small, or
-// doesn't change often:
-// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL
-// Repository --> has checksum (to detect need for update and corruption)
-// Cache --> whatever from above
-// Local --> install time version
-
-var readRepoOnlyAsset = function(path, callback) {
-
- var assetEntry;
+var stringIsNotEmpty = function(s) {
+ return typeof s === 'string' && s !== '';
+};
- var reportBack = function(content, err) {
- var details = {
- 'path': path,
- 'content': content
- };
- if ( err ) {
- details.error = err;
- }
- callback(details);
- };
+/*******************************************************************************
- var onInstallFileLoaded = function() {
- //console.log('µBlock> readRepoOnlyAsset("%s") / onInstallFileLoaded()', path);
- reportBack(this.responseText);
- };
+ User assets are NOT persisted in the cache storage. User assets are
+ recognized by the asset key which always starts with 'user-'.
- var onInstallFileError = function() {
- console.error('µBlock> readRepoOnlyAsset("%s") / onInstallFileError()', path);
- reportBack('', 'Error');
- };
+ TODO(seamless migration):
+ Can remove instances of old user asset keys when I am confident all users
+ are using uBO v1.11 and beyond.
- var onCachedContentLoaded = function(details) {
- //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentLoaded()', path);
- reportBack(details.content);
- };
+**/
- var onCachedContentError = function() {
- //console.log('µBlock> readRepoOnlyAsset("%s") / onCachedContentError()', path);
- getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError);
+var readUserAsset = function(assetKey, callback) {
+ var reportBack = function(content) {
+ callback({ assetKey: assetKey, content: content });
};
- var repositoryURL = toRepoURL(path + '?_=' + Date.now());
-
- var onRepoFileLoaded = function() {
- if ( typeof this.responseText !== 'string' ) {
- console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): no response', path, repositoryURL);
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- return;
- }
- if ( YaMD5.hashStr(this.responseText) !== assetEntry.repoChecksum ) {
- console.error('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): bad md5 checksum', path, repositoryURL);
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- return;
+ var onLoaded = function(bin) {
+ if ( !bin ) { return reportBack(''); }
+ var content = '';
+ if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
+ content = bin['cached_asset_content://assets/user/filters.txt'];
+ vAPI.cacheStorage.remove('cached_asset_content://assets/user/filters.txt');
+ }
+ if ( typeof bin['assets/user/filters.txt'] === 'string' ) {
+ content = bin['assets/user/filters.txt'];
+ // TODO(seamless migration):
+ // Uncomment once all moved to v1.11+.
+ //vAPI.storage.remove('assets/user/filters.txt');
+ }
+ if ( typeof bin[assetKey] === 'string' ) {
+ // TODO(seamless migration):
+ // Replace conditional with assignment once all moved to v1.11+
+ if ( content !== bin[assetKey] ) {
+ saveUserAsset(assetKey, content);
+ }
+ } else if ( content !== '' ) {
+ saveUserAsset(assetKey, content);
}
- //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL);
- assetEntry.localChecksum = assetEntry.repoChecksum;
- updateLocalChecksums();
- cachedAssetsManager.save(path, this.responseText, callback);
- };
-
- var onRepoFileError = function() {
- console.error(errorCantConnectTo.replace('{{url}}', repositoryURL));
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
+ return reportBack(content);
};
+ var toRead = assetKey;
+ if ( assetKey === µBlock.userFiltersPath ) {
+ toRead = [
+ assetKey,
+ 'assets/user/filters.txt',
+ 'cached_asset_content://assets/user/filters.txt'
+ ];
+ }
+ vAPI.storage.get(toRead, onLoaded);
+};
- var onRepoMetaReady = function(meta) {
- assetEntry = meta.entries[path];
-
- // Asset doesn't exist
- if ( assetEntry === undefined ) {
- reportBack('', 'Error: asset not found');
- return;
- }
-
- // Asset added or changed: load from repo URL and then cache result
- if (
- exports.remoteFetchBarrier === 0 &&
- exports.autoUpdate &&
- assetEntry.localChecksum !== assetEntry.repoChecksum
- ) {
- //console.log('µBlock> readRepoOnlyAsset("%s") / onRepoMetaReady(): repo has newer version', path);
- getTextFileFromURL(repositoryURL, onRepoFileLoaded, onRepoFileError);
- return;
+var saveUserAsset = function(assetKey, content, callback) {
+ var bin = {};
+ bin[assetKey] = content;
+ // TODO(seamless migration):
+ // This is for forward compatibility. Only for a limited time. Remove when
+ // everybody moved to 1.11.0 and beyond.
+ // >>>>>>>>
+ if ( assetKey === µBlock.userFiltersPath ) {
+ bin['assets/user/filters.txt'] = content;
+ }
+ // <<<<<<<<
+ var onSaved = function() {
+ if ( callback instanceof Function ) {
+ callback({ assetKey: assetKey, content: content });
}
-
- // Load from cache
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
};
-
- getRepoMetadata(onRepoMetaReady);
+ vAPI.storage.set(bin, onSaved);
};
/******************************************************************************/
-// Asset doesn't exist. Just for symmetry purpose.
-
-var readNilAsset = function(path, callback) {
- callback({
- 'path': path,
- 'content': '',
- 'error': 'Error: asset not found'
- });
-};
-
-/******************************************************************************/
+api.get = function(assetKey, callback) {
+ if ( assetKey === µBlock.userFiltersPath ) {
+ readUserAsset(assetKey, callback);
+ return;
+ }
-// An external asset:
-// Path --> starts with 'http'
-// External --> https://..., http://...
-// Cache --> has expiration timestamp (in cache)
+ var assetDetails = {},
+ contentURLs,
+ contentURL;
-var readExternalAsset = function(path, callback) {
var reportBack = function(content, err) {
- var details = {
- 'path': path,
- 'content': content
- };
+ var details = { assetKey: assetKey, content: content };
if ( err ) {
- details.error = err;
+ details.error = assetDetails.lastError = err;
+ } else {
+ assetDetails.lastError = undefined;
}
callback(details);
};
- var onCachedContentLoaded = function(details) {
- //console.log('µBlock> readExternalAsset("%s") / onCachedContentLoaded()', path);
- reportBack(details.content);
- };
-
- var onCachedContentError = function() {
- console.error('µBlock> readExternalAsset("%s") / onCachedContentError()', path);
- reportBack('', 'Error');
+ var onContentNotLoaded = function() {
+ var isExternal;
+ while ( (contentURL = contentURLs.shift()) ) {
+ isExternal = reIsExternalPath.test(contentURL);
+ if ( isExternal === false || assetDetails.hasLocalURL !== true ) {
+ break;
+ }
+ }
+ if ( !contentURL ) {
+ return reportBack('', 'E_NOTFOUND');
+ }
+ getTextFileFromURL(contentURL, onContentLoaded, onContentNotLoaded);
};
- var onExternalFileLoaded = function() {
- // https://github.com/chrisaljoudi/uBlock/issues/708
- // A successful download should never return an empty file: turn this
- // into an error condition.
+ var onContentLoaded = function() {
if ( stringIsNotEmpty(this.responseText) === false ) {
- onExternalFileError();
+ onContentNotLoaded();
return;
}
- //console.log('µBlock> readExternalAsset("%s") / onExternalFileLoaded1()', path);
- cachedAssetsManager.save(path, this.responseText);
+ if ( reIsExternalPath.test(contentURL) ) {
+ assetCacheWrite(assetKey, {
+ content: this.responseText,
+ url: contentURL
+ });
+ }
reportBack(this.responseText);
};
- var onExternalFileError = function() {
- console.error(errorCantConnectTo.replace('{{url}}', path));
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
- };
-
- var onCacheMetaReady = function(entries) {
- // Fetch from remote if:
- // - Not in cache OR
- //
- // - Auto-update enabled AND in cache but obsolete
- var timestamp = entries[path];
- var notInCache = typeof timestamp !== 'number';
- var updateCache = exports.remoteFetchBarrier === 0 &&
- exports.autoUpdate &&
- cacheIsObsolete(timestamp);
- if ( notInCache || updateCache ) {
- getTextFileFromURL(path, onExternalFileLoaded, onExternalFileError);
- return;
- }
-
- // In cache
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
+ var onCachedContentLoaded = function(details) {
+ if ( details.content !== '' ) {
+ return reportBack(details.content);
+ }
+ getAssetSourceRegistry(function(registry) {
+ assetDetails = registry[assetKey] || {};
+ if ( typeof assetDetails.contentURL === 'string' ) {
+ contentURLs = [ assetDetails.contentURL ];
+ } else if ( Array.isArray(assetDetails.contentURL) ) {
+ contentURLs = assetDetails.contentURL.slice(0);
+ } else {
+ contentURLs = [];
+ }
+ onContentNotLoaded();
+ });
};
- cachedAssetsManager.entries(onCacheMetaReady);
+ assetCacheRead(assetKey, onCachedContentLoaded);
};
/******************************************************************************/
-// User data:
-// Path --> starts with 'assets/user/'
-// Cache --> whatever user saved
-
-var readUserAsset = function(path, callback) {
- // TODO: remove when confident all users no longer have their custom
- // filters saved into vAPI.cacheStorage.
- var onCachedContentLoaded = function(details) {
- saveUserAsset(path, details.content);
- //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentLoaded()', path);
- callback({ 'path': path, 'content': details.content });
- };
-
- var onCachedContentError = function() {
- saveUserAsset(path, '');
- //console.log('µBlock.assets/readUserAsset("%s")/onCachedContentError()', path);
- callback({ 'path': path, 'content': '' });
- };
+var getRemote = function(assetKey, callback) {
+ var assetDetails = {},
+ contentURLs,
+ contentURL;
- var onLoaded = function(bin) {
- var content = bin && bin[path];
- if ( typeof content === 'string' ) {
- callback({ 'path': path, 'content': content });
- return;
+ var reportBack = function(content, err) {
+ var details = { assetKey: assetKey, content: content };
+ if ( err ) {
+ details.error = assetDetails.lastError = err;
+ } else {
+ assetDetails.lastError = undefined;
}
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
+ callback(details);
};
- vAPI.storage.get(path, onLoaded);
-};
-
-var saveUserAsset = function(path, content, callback) {
- var bin = {};
- bin[path] = content;
- var onSaved = function() {
- // Saving over an existing asset must be seen as removing an
- // existing asset and adding a new one.
- if ( onAssetRemovedListener instanceof Function ) {
- onAssetRemovedListener([ path ]);
- }
- if ( callback instanceof Function ) {
- callback({ path: path, content: content });
+ var onRemoteContentLoaded = function() {
+ if ( stringIsNotEmpty(this.responseText) === false ) {
+ registerAssetSource(assetKey, { error: { time: Date.now(), error: 'No content' } });
+ tryLoading();
+ return;
}
- };
- vAPI.storage.set(bin, onSaved);
-};
-
-/******************************************************************************/
-
-// Asset available only from the cache.
-// Cache data:
-// Path --> starts with 'cache://'
-// Cache --> whatever
-
-var readCacheAsset = function(path, callback) {
- var onCachedContentLoaded = function(details) {
- //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentLoaded()', path);
- callback({ 'path': path, 'content': details.content });
+ assetCacheWrite(assetKey, {
+ content: this.responseText,
+ url: contentURL
+ });
+ registerAssetSource(assetKey, { error: undefined });
+ reportBack(this.responseText);
};
- var onCachedContentError = function() {
- //console.log('µBlock.assets/readCacheAsset("%s")/onCachedContentError()', path);
- callback({ 'path': path, 'content': '' });
+ var onRemoteContentError = function() {
+ registerAssetSource(assetKey, { error: { time: Date.now(), error: this.statusText } });
+ tryLoading();
};
- cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError);
-};
-
-/******************************************************************************/
-
-// Assets
-//
-// A copy of an asset from an external source shipped with the extension:
-// Path --> starts with 'assets/(thirdparties|ublock)/', with a home URL
-// External -->
-// Repository --> has checksum (to detect obsolescence)
-// Cache --> has expiration timestamp (to detect obsolescence)
-// Local --> install time version
-//
-// An important asset shipped with the extension (usually small, or doesn't
-// change often):
-// Path --> starts with 'assets/(thirdparties|ublock)/', without a home URL
-// Repository --> has checksum (to detect obsolescence or data corruption)
-// Cache --> whatever from above
-// Local --> install time version
-//
-// An external filter list:
-// Path --> starts with 'http'
-// External -->
-// Cache --> has expiration timestamp (to detect obsolescence)
-//
-// User data:
-// Path --> starts with 'assets/user/'
-// Cache --> whatever user saved
-//
-// When a checksum is present, it is used to determine whether the asset
-// needs to be updated.
-// When an expiration timestamp is present, it is used to determine whether
-// the asset needs to be updated.
-//
-// If no update required, an asset if first fetched from the cache. If the
-// asset is not cached it is fetched from the closest location: local for
-// an asset shipped with the extension, external for an asset not shipped
-// with the extension.
-
-exports.get = function(path, callback) {
-
- if ( reIsUserPath.test(path) ) {
- readUserAsset(path, callback);
- return;
- }
-
- if ( reIsCachePath.test(path) ) {
- readCacheAsset(path, callback);
- return;
- }
-
- if ( reIsExternalPath.test(path) ) {
- readExternalAsset(path, callback);
- return;
- }
-
- var onRepoMetaReady = function(meta) {
- var assetEntry = meta.entries[path];
-
- // Asset doesn't exist
- if ( assetEntry === undefined ) {
- readNilAsset(path, callback);
- return;
+ var tryLoading = function() {
+ while ( (contentURL = contentURLs.shift()) ) {
+ if ( reIsExternalPath.test(contentURL) ) { break; }
}
-
- // Asset is repo copy of external content
- if ( stringIsNotEmpty(homeURLs[path]) ) {
- readRepoCopyAsset(path, callback);
- return;
+ if ( !contentURL ) {
+ return reportBack('', 'E_NOTFOUND');
}
-
- // Asset is repo only
- readRepoOnlyAsset(path, callback);
+ getTextFileFromURL(contentURL, onRemoteContentLoaded, onRemoteContentError);
};
- getRepoMetadata(onRepoMetaReady);
+ getAssetSourceRegistry(function(registry) {
+ assetDetails = registry[assetKey] || {};
+ if ( typeof assetDetails.contentURL === 'string' ) {
+ contentURLs = [ assetDetails.contentURL ];
+ } else if ( Array.isArray(assetDetails.contentURL) ) {
+ contentURLs = assetDetails.contentURL.slice(0);
+ } else {
+ contentURLs = [];
+ }
+ tryLoading();
+ });
};
-// https://www.youtube.com/watch?v=98y0Q7nLGWk
-
-/******************************************************************************/
-
-exports.getLocal = readLocalFile;
-
/******************************************************************************/
-exports.put = function(path, content, callback) {
- if ( reIsUserPath.test(path) ) {
- saveUserAsset(path, content, callback);
- return;
+api.put = function(assetKey, content, callback) {
+ if ( reIsUserAsset.test(assetKey) ) {
+ return saveUserAsset(assetKey, content, callback);
}
-
- cachedAssetsManager.save(path, content, callback);
-};
-
-/******************************************************************************/
-
-exports.rmrf = function() {
- cachedAssetsManager.rmrf();
-};
-
-/******************************************************************************/
-
-exports.rename = function(from, to, callback) {
- var done = function() {
- if ( typeof callback === 'function' ) {
- callback();
- }
- };
-
- var fromLoaded = function(details) {
- cachedAssetsManager.remove(from);
- cachedAssetsManager.save(to, details.content, callback);
- done();
- };
-
- var toLoaded = function(details) {
- // `to` already exists: do nothing
- if ( details.content !== '' ) {
- return done();
- }
- cachedAssetsManager.load(from, fromLoaded);
- };
-
- // If `to` content already exists, do nothing.
- cachedAssetsManager.load(to, toLoaded);
+ assetCacheWrite(assetKey, content, callback);
};
/******************************************************************************/
-exports.metadata = function(callback) {
- var out = {};
-
- // https://github.com/chrisaljoudi/uBlock/issues/186
- // We need to check cache obsolescence when both cache and repo meta data
- // has been gathered.
- var checkCacheObsolescence = function() {
- var entry, homeURL;
- for ( var path in out ) {
- if ( out.hasOwnProperty(path) === false ) {
- continue;
- }
- entry = out[path];
- // https://github.com/gorhill/uBlock/issues/528
- // Not having a homeURL property does not mean the filter list
- // is not external.
- homeURL = reIsExternalPath.test(path) ? path : homeURLs[path];
- entry.cacheObsolete = stringIsNotEmpty(homeURL) &&
- cacheIsObsolete(entry.lastModified);
- }
- callback(out);
- };
+api.metadata = function(callback) {
+ var assetRegistryReady = false,
+ cacheRegistryReady = false;
- var onRepoMetaReady = function(meta) {
- var entries = meta.entries;
- var entryRepo, entryOut;
- for ( var path in entries ) {
- if ( entries.hasOwnProperty(path) === false ) {
- continue;
- }
- entryRepo = entries[path];
- entryOut = out[path];
- if ( entryOut === undefined ) {
- entryOut = out[path] = {};
+ var onReady = function() {
+ var assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)),
+ cacheDict = assetCacheRegistry,
+ assetEntry, cacheEntry,
+ now = Date.now(), obsoleteAfter;
+ for ( var assetKey in assetDict ) {
+ assetEntry = assetDict[assetKey];
+ cacheEntry = cacheDict[assetKey];
+ if ( cacheEntry ) {
+ assetEntry.cached = true;
+ assetEntry.writeTime = cacheEntry.writeTime;
+ obsoleteAfter = cacheEntry.writeTime + assetEntry.updateAfter * 86400000;
+ assetEntry.obsolete = obsoleteAfter < now;
+ assetEntry.remoteURL = cacheEntry.remoteURL;
+ } else {
+ assetEntry.writeTime = 0;
+ obsoleteAfter = 0;
+ assetEntry.obsolete = true;
}
- entryOut.localChecksum = entryRepo.localChecksum;
- entryOut.repoChecksum = entryRepo.repoChecksum;
- entryOut.homeURL = homeURLs[path] || '';
- entryOut.supportURL = entryRepo.supportURL || '';
- entryOut.repoObsolete = entryOut.localChecksum !== entryOut.repoChecksum;
}
- checkCacheObsolescence();
+ callback(assetDict);
};
- var onCacheMetaReady = function(entries) {
- var entryOut;
- for ( var path in entries ) {
- if ( entries.hasOwnProperty(path) === false ) {
- continue;
- }
- entryOut = out[path];
- if ( entryOut === undefined ) {
- entryOut = out[path] = {};
- }
- entryOut.lastModified = entries[path];
- // User data is not literally cache data
- if ( reIsUserPath.test(path) ) {
- continue;
- }
- entryOut.cached = true;
- if ( reIsExternalPath.test(path) ) {
- entryOut.homeURL = path;
- }
- }
- getRepoMetadata(onRepoMetaReady);
- };
+ getAssetSourceRegistry(function() {
+ assetRegistryReady = true;
+ if ( cacheRegistryReady ) { onReady(); }
+ });
- cachedAssetsManager.entries(onCacheMetaReady);
+ getAssetCacheRegistry(function() {
+ cacheRegistryReady = assetCacheRegistry;
+ if ( assetRegistryReady ) { onReady(); }
+ });
};
/******************************************************************************/
-exports.purge = function(pattern, before) {
- cachedAssetsManager.remove(pattern, before);
-};
-
-exports.purgeCacheableAsset = function(pattern, before) {
- cachedAssetsManager.remove(pattern, before);
- lastRepoMetaTimestamp = 0;
-};
-
-exports.purgeAll = function(callback) {
- cachedAssetsManager.removeAll(callback);
- lastRepoMetaTimestamp = 0;
+api.purge = function(pattern, callback) {
+ assetCacheMarkAsDirty(pattern, callback);
};
-/******************************************************************************/
-
-exports.onAssetRemoved = {
- addListener: function(callback) {
- onAssetRemovedListener = callback instanceof Function ? callback : null;
- }
+api.remove = function(pattern, callback) {
+ assetCacheRemove(pattern, callback);
};
-/******************************************************************************/
-
-return exports;
-
-})();
-
-/******************************************************************************/
-/******************************************************************************/
-
-µBlock.assetUpdater = (function() {
-
-/******************************************************************************/
-
-var µb = µBlock;
-
-var updateDaemonTimer = null;
-var autoUpdateDaemonTimerPeriod = 11 * 60 * 1000; // 11 minutes
-var manualUpdateDaemonTimerPeriod = 5 * 1000; // 5 seconds
-
-var updateCycleFirstPeriod = 7 * 60 * 1000; // 7 minutes
-var updateCycleNextPeriod = 11 * 60 * 60 * 1000; // 11 hours
-var updateCycleTime = 0;
-
-var toUpdate = {};
-var toUpdateCount = 0;
-var updated = {};
-var updatedCount = 0;
-var metadata = null;
-
-var onStartListener = null;
-var onCompletedListener = null;
-var onAssetUpdatedListener = null;
-
-var exports = {
- manualUpdate: false,
- manualUpdateProgress: {
- value: 0,
- text: null
- }
+api.rmrf = function() {
+ assetCacheRemove(/./);
};
/******************************************************************************/
-var onOneUpdated = function(details) {
- // Resource fetched, we can safely restart the daemon.
- scheduleUpdateDaemon();
-
- var path = details.path;
- if ( details.error ) {
- manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount));
- //console.debug('µBlock.assetUpdater/onOneUpdated: "%s" failed', path);
- return;
- }
-
- //console.debug('µBlock.assetUpdater/onOneUpdated: "%s"', path);
- updated[path] = true;
- updatedCount += 1;
-
- if ( typeof onAssetUpdatedListener === 'function' ) {
- onAssetUpdatedListener(details);
- }
+// Asset updater area.
+var updaterStatus,
+ updaterTimer,
+ updaterAssetDelayDefault = 120000,
+ updaterAssetDelay = updaterAssetDelayDefault,
+ updaterUpdated = [],
+ updaterFetched = new Set();
- manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount + 1));
+var updateFirst = function() {
+ updaterStatus = 'updating';
+ updaterFetched.clear();
+ updaterUpdated = [];
+ fireNotification('before-assets-updated');
+ updateNext();
};
-/******************************************************************************/
-
-var updateOne = function() {
- // Because this can be called from outside the daemon's main loop
- µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate;
-
- var metaEntry;
- var updatingCount = 0;
- var updatingText = null;
+var updateNext = function() {
+ var assetDict, cacheDict;
- for ( var path in toUpdate ) {
- if ( toUpdate.hasOwnProperty(path) === false ) {
- continue;
- }
- if ( toUpdate[path] !== true ) {
- continue;
- }
- toUpdate[path] = false;
- toUpdateCount -= 1;
- if ( metadata.hasOwnProperty(path) === false ) {
- continue;
+ // This will remove a cached asset when it's no longer in use.
+ var garbageCollectOne = function(assetKey) {
+ var cacheEntry = cacheDict[assetKey];
+ if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) {
+ assetCacheRemove(assetKey);
}
- metaEntry = metadata[path];
- if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) {
- continue;
- }
-
- // Will restart the update daemon once the resource is received: the
- // fetching of a resource may take some time, possibly beyond the
- // next scheduled daemon cycle, so this ensure the daemon won't do
- // anything else before the resource is fetched (or times out).
- suspendUpdateDaemon();
-
- //console.debug('µBlock.assetUpdater/updateOne: assets.get("%s")', path);
- µb.assets.get(path, onOneUpdated);
- updatingCount = 1;
- updatingText = metaEntry.homeURL || path;
- break;
- }
-
- manualUpdateNotify(
- false,
- (updatedCount + updatingCount/2) / (updatedCount + toUpdateCount + updatingCount + 1),
- updatingText
- );
-};
-
-/******************************************************************************/
-
-// Update one asset, fetch metadata if not done yet.
-
-var safeUpdateOne = function() {
- if ( metadata !== null ) {
- updateOne();
- return;
- }
-
- // Because this can be called from outside the daemon's main loop
- µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate;
-
- var onMetadataReady = function(response) {
- scheduleUpdateDaemon();
- metadata = response;
- updateOne();
};
- suspendUpdateDaemon();
- µb.assets.metadata(onMetadataReady);
-};
-
-/******************************************************************************/
-
-var safeStartListener = function(callback) {
- // Because this can be called from outside the daemon's main loop
- µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate;
-
- var onStartListenerDone = function(assets) {
- scheduleUpdateDaemon();
- assets = assets || {};
- for ( var path in assets ) {
- if ( assets.hasOwnProperty(path) === false ) {
+ var findOne = function() {
+ var now = Date.now(),
+ assetEntry, cacheEntry;
+ for ( var assetKey in assetDict ) {
+ assetEntry = assetDict[assetKey];
+ if ( assetEntry.hasRemoteURL !== true ) { continue; }
+ if ( updaterFetched.has(assetKey) ) { continue; }
+ cacheEntry = cacheDict[assetKey];
+ if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) {
continue;
}
- if ( toUpdate.hasOwnProperty(path) ) {
- continue;
+ if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) {
+ return assetKey;
}
- //console.debug('assets.js > µBlock.assetUpdater/safeStartListener: "%s"', path);
- toUpdate[path] = true;
- toUpdateCount += 1;
- }
- if ( typeof callback === 'function' ) {
- callback();
+ garbageCollectOne(assetKey);
}
};
- if ( typeof onStartListener === 'function' ) {
- suspendUpdateDaemon();
- onStartListener(onStartListenerDone);
- } else {
- onStartListenerDone(null);
- }
-};
-
-/******************************************************************************/
-
-var updateDaemon = function() {
- updateDaemonTimer = null;
- scheduleUpdateDaemon();
-
- µb.assets.autoUpdate = µb.userSettings.autoUpdate || exports.manualUpdate;
-
- if ( µb.assets.autoUpdate !== true ) {
- return;
- }
-
- // Start an update cycle?
- if ( updateCycleTime !== 0 ) {
- if ( Date.now() >= updateCycleTime ) {
- //console.debug('µBlock.assetUpdater/updateDaemon: update cycle started');
- reset();
- safeStartListener();
+ var updatedOne = function(details) {
+ if ( details.content !== '' ) {
+ updaterUpdated.push(details.assetKey);
+ if ( details.assetKey === 'assets.json' ) {
+ updateAssetSourceRegistry(details.content);
+ }
}
- return;
- }
-
- // Any asset to update?
- if ( toUpdateCount !== 0 ) {
- safeUpdateOne();
- return;
- }
- // Nothing left to update
-
- // In case of manual update, fire progress notifications
- manualUpdateNotify(true, 1, '');
-
- // If anything was updated, notify listener
- if ( updatedCount !== 0 ) {
- if ( typeof onCompletedListener === 'function' ) {
- //console.debug('µBlock.assetUpdater/updateDaemon: update cycle completed');
- onCompletedListener({
- updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy
- updatedCount: updatedCount
- });
+ if ( findOne() !== undefined ) {
+ vAPI.setTimeout(updateNext, updaterAssetDelay);
+ } else {
+ updateDone();
}
- }
-
- // Schedule next update cycle
- if ( updateCycleTime === 0 ) {
- reset();
- //console.debug('µBlock.assetUpdater/updateDaemon: update cycle re-scheduled');
- updateCycleTime = Date.now() + updateCycleNextPeriod;
- }
-};
-
-/******************************************************************************/
-
-var scheduleUpdateDaemon = function() {
- if ( updateDaemonTimer !== null ) {
- clearTimeout(updateDaemonTimer);
- }
- updateDaemonTimer = vAPI.setTimeout(
- updateDaemon,
- exports.manualUpdate ? manualUpdateDaemonTimerPeriod : autoUpdateDaemonTimerPeriod
- );
-};
-
-var suspendUpdateDaemon = function() {
- if ( updateDaemonTimer !== null ) {
- clearTimeout(updateDaemonTimer);
- updateDaemonTimer = null;
- }
-};
-
-scheduleUpdateDaemon();
-
-/******************************************************************************/
-
-var reset = function() {
- toUpdate = {};
- toUpdateCount = 0;
- updated = {};
- updatedCount = 0;
- updateCycleTime = 0;
- metadata = null;
-};
-
-/******************************************************************************/
-
-var manualUpdateNotify = function(done, value, text) {
- if ( exports.manualUpdate === false ) {
- return;
- }
+ };
- exports.manualUpdate = !done;
- exports.manualUpdateProgress.value = value || 0;
- if ( typeof text === 'string' ) {
- exports.manualUpdateProgress.text = text;
- }
+ var updateOne = function() {
+ var assetKey = findOne();
+ if ( assetKey === undefined ) {
+ return updateDone();
+ }
+ updaterFetched.add(assetKey);
+ getRemote(assetKey, updatedOne);
+ };
- vAPI.messaging.broadcast({
- what: 'forceUpdateAssetsProgress',
- done: !exports.manualUpdate,
- progress: exports.manualUpdateProgress,
- updatedCount: updatedCount
+ getAssetSourceRegistry(function(dict) {
+ assetDict = dict;
+ if ( !cacheDict ) { return; }
+ updateOne();
});
- // When manually updating, whatever launched the manual update is
- // responsible to launch a reload of the filter lists.
- if ( exports.manualUpdate !== true ) {
- reset();
- }
+ getAssetCacheRegistry(function(dict) {
+ cacheDict = dict;
+ if ( !assetDict ) { return; }
+ updateOne();
+ });
};
-/******************************************************************************/
-
-// Manual update: just a matter of forcing the update daemon to work on a
-// tighter schedule.
-
-exports.force = function() {
- if ( exports.manualUpdate ) {
- return;
- }
-
- reset();
-
- exports.manualUpdate = true;
-
- var onStartListenerDone = function() {
- if ( toUpdateCount === 0 ) {
- updateCycleTime = Date.now() + updateCycleNextPeriod;
- manualUpdateNotify(true, 1);
- } else {
- manualUpdateNotify(false, 0);
- safeUpdateOne();
- }
- };
-
- safeStartListener(onStartListenerDone);
+var updateDone = function() {
+ var assetKeys = updaterUpdated.slice(0);
+ updaterFetched.clear();
+ updaterUpdated = [];
+ updaterStatus = undefined;
+ updaterAssetDelay = updaterAssetDelayDefault;
+ fireNotification('after-assets-updated', { assetKeys: assetKeys });
};
-/******************************************************************************/
-
-exports.onStart = {
- addEventListener: function(callback) {
- onStartListener = callback || null;
- if ( typeof onStartListener === 'function' ) {
- updateCycleTime = Date.now() + updateCycleFirstPeriod;
+api.updateStart = function(details) {
+ var oldUpdateDelay = updaterAssetDelay,
+ newUpdateDelay = details.delay || updaterAssetDelayDefault;
+ updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay);
+ if ( updaterStatus !== undefined ) {
+ if ( newUpdateDelay < oldUpdateDelay ) {
+ clearTimeout(updaterTimer);
+ updaterTimer = vAPI.setTimeout(updateNext, updaterAssetDelay);
}
+ return;
}
+ updateFirst();
};
-/******************************************************************************/
-
-exports.onAssetUpdated = {
- addEventListener: function(callback) {
- onAssetUpdatedListener = callback || null;
+api.updateStop = function() {
+ if ( updaterTimer ) {
+ clearTimeout(updaterTimer);
+ updaterTimer = undefined;
}
-};
-
-/******************************************************************************/
-
-exports.onCompleted = {
- addEventListener: function(callback) {
- onCompletedListener = callback || null;
+ if ( updaterStatus !== undefined ) {
+ updateDone();
}
};
/******************************************************************************/
-// Typically called when an update has been forced.
-
-exports.restart = function() {
- reset();
- updateCycleTime = Date.now() + updateCycleNextPeriod;
-};
-
-/******************************************************************************/
-
-// Call when disabling uBlock, to ensure it doesn't stick around as a detached
-// window object in Firefox.
-
-exports.shutdown = function() {
- suspendUpdateDaemon();
- reset();
-};
+return api;
/******************************************************************************/
-return exports;
-
})();
/******************************************************************************/
diff --git a/src/js/background.js b/src/js/background.js
index 00b82a1c..791e56be 100644
--- a/src/js/background.js
+++ b/src/js/background.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -19,20 +19,16 @@
Home: https://github.com/gorhill/uBlock
*/
-/* exported µBlock */
-
'use strict';
/******************************************************************************/
-var µBlock = (function() {
+var µBlock = (function() { // jshint ignore:line
/******************************************************************************/
var oneSecond = 1000;
var oneMinute = 60 * oneSecond;
-var oneHour = 60 * oneMinute;
-// var oneDay = 24 * oneHour;
/******************************************************************************/
@@ -71,8 +67,12 @@ return {
},
hiddenSettingsDefault: {
+ assetFetchTimeout: 30,
+ autoUpdateAssetFetchPeriod: 120,
+ autoUpdatePeriod: 7,
ignoreRedirectFilters: false,
ignoreScriptInjectFilters: false,
+ manualUpdateAssetFetchPeriod: 2000,
popupFontSize: 'unset',
suspendTabsUntilReady: false
},
@@ -119,92 +119,15 @@ return {
lastBackupTime: 0
},
- // EasyList, EasyPrivacy and many others have an 4-day update period,
- // as per list headers.
- updateAssetsEvery: 97 * oneHour,
- projectServerRoot: 'https://raw.githubusercontent.com/gorhill/uBlock/master/',
- userFiltersPath: 'assets/user/filters.txt',
- pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat',
-
- // permanent lists
- permanentLists: {
- // User
- 'assets/user/filters.txt': {
- group: 'default'
- },
- // uBlock
- 'assets/ublock/filters.txt': {
- title: 'uBlock filters',
- group: 'default'
- },
- 'assets/ublock/privacy.txt': {
- title: 'uBlock filters – Privacy',
- group: 'default'
- },
- 'assets/ublock/unbreak.txt': {
- title: 'uBlock filters – Unbreak',
- group: 'default'
- },
- 'assets/ublock/badware.txt': {
- title: 'uBlock filters – Badware risks',
- group: 'default',
- supportURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks',
- instructionURL: 'https://github.com/gorhill/uBlock/wiki/Badware-risks'
- },
- 'assets/ublock/experimental.txt': {
- title: 'uBlock filters – Experimental',
- group: 'default',
- off: true,
- supportURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters',
- instructionURL: 'https://github.com/gorhill/uBlock/wiki/Experimental-filters'
- }
- },
+ // Allows to fully customize uBO's assets, typically set through admin
+ // settings. The content of 'assets.json' will also tell which filter
+ // lists to enable by default when uBO is first installed.
+ assetsBootstrapLocation: 'assets/assets.json',
- // current lists
- remoteBlacklists: {},
- oldListToNewListMap: {
- "assets/thirdparties/adblock.gardar.net/is.abp.txt": "http://adblock.gardar.net/is.abp.txt",
- "assets/thirdparties/adblock.schack.dk/block.txt": "https://adblock.dk/block.csv",
- "https://adblock.schack.dk/block.txt": "https://adblock.dk/block.csv",
- "assets/thirdparties/dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt": "https://dl.dropboxusercontent.com/u/1289327/abpxfiles/filtri.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/advblock.txt": "https://easylist-downloads.adblockplus.org/advblock.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/bitblock.txt": "https://easylist-downloads.adblockplus.org/bitblock.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/easylist_noelemhide.txt": "https://easylist-downloads.adblockplus.org/easylist_noelemhide.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/easylistchina.txt": "https://easylist-downloads.adblockplus.org/easylistchina.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/easylistdutch.txt": "https://easylist-downloads.adblockplus.org/easylistdutch.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/easylistgermany.txt": "https://easylist-downloads.adblockplus.org/easylistgermany.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/easylistitaly.txt": "https://easylist-downloads.adblockplus.org/easylistitaly.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-annoyance.txt": "https://easylist-downloads.adblockplus.org/fanboy-annoyance.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/fanboy-social.txt": "https://easylist-downloads.adblockplus.org/fanboy-social.txt",
- "assets/thirdparties/easylist-downloads.adblockplus.org/liste_fr.txt": "https://easylist-downloads.adblockplus.org/liste_fr.txt",
- "assets/thirdparties/gitorious.org/adblock-latvian/adblock-latvian/raw/master_lists/latvian-list.txt": "https://notabug.org/latvian-list/adblock-latvian/raw/master/lists/latvian-list.txt",
- "assets/thirdparties/home.fredfiber.no/langsholt/adblock.txt": "http://home.fredfiber.no/langsholt/adblock.txt",
- "assets/thirdparties/hosts-file.net/ad-servers": "http://hosts-file.net/.%5Cad_servers.txt",
- "assets/thirdparties/http://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt",
- "assets/thirdparties/liste-ar-adblock.googlecode.com/hg/Liste_AR.txt": "https://liste-ar-adblock.googlecode.com/hg/Liste_AR.txt",
- "assets/thirdparties/margevicius.lt/easylistlithuania.txt": "http://margevicius.lt/easylistlithuania.txt",
- "assets/thirdparties/mirror1.malwaredomains.com/files/immortal_domains.txt": "http://malwaredomains.lehigh.edu/files/immortal_domains.txt",
- "assets/thirdparties/raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt": "https://raw.githubusercontent.com/AdBlockPlusIsrael/EasyListHebrew/master/EasyListHebrew.txt",
- "assets/thirdparties/raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt": "https://raw.githubusercontent.com/cjx82630/cjxlist/master/cjxlist.txt",
- "assets/thirdparties/raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt": "https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt",
- "assets/thirdparties/raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt": "https://raw.githubusercontent.com/szpeter80/hufilter/master/hufilter.txt",
- "assets/thirdparties/raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt": "https://raw.githubusercontent.com/tomasko126/easylistczechandslovak/master/filters.txt",
- "assets/thirdparties/someonewhocares.org/hosts/hosts": "http://someonewhocares.org/hosts/hosts",
- "assets/thirdparties/spam404bl.com/spam404scamlist.txt": "https://spam404bl.com/spam404scamlist.txt",
- "assets/thirdparties/stanev.org/abp/adblock_bg.txt": "http://stanev.org/abp/adblock_bg.txt",
- "assets/thirdparties/winhelp2002.mvps.org/hosts.txt": "http://winhelp2002.mvps.org/hosts.txt",
- "assets/thirdparties/www.fanboy.co.nz/enhancedstats.txt": "https://www.fanboy.co.nz/enhancedstats.txt",
- "assets/thirdparties/www.fanboy.co.nz/fanboy-antifacebook.txt": "https://www.fanboy.co.nz/fanboy-antifacebook.txt",
- "assets/thirdparties/www.fanboy.co.nz/fanboy-korean.txt": "https://www.fanboy.co.nz/fanboy-korean.txt",
- "assets/thirdparties/www.fanboy.co.nz/fanboy-swedish.txt": "https://www.fanboy.co.nz/fanboy-swedish.txt",
- "assets/thirdparties/www.fanboy.co.nz/fanboy-ultimate.txt": "https://www.fanboy.co.nz/r/fanboy-ultimate.txt",
- "assets/thirdparties/www.fanboy.co.nz/fanboy-vietnam.txt": "https://www.fanboy.co.nz/fanboy-vietnam.txt",
- "assets/thirdparties/www.void.gr/kargig/void-gr-filters.txt": "https://www.void.gr/kargig/void-gr-filters.txt",
- "assets/thirdparties/www.zoso.ro/pages/rolist.txt": "",
- "https://iadb.azurewebsites.net/Finland_adb.txt": "http://adb.juvander.net/Finland_adb.txt",
- "https://www.certyficate.it/adblock/adblock.txt": "https://raw.githubusercontent.com/MajkiIT/polish-ads-filter/master/polish-adblock-filters/adblock.txt",
- "https://raw.githubusercontent.com/heradhis/indonesianadblockrules/master/subscriptions/abpindo.txt": "https://raw.githubusercontent.com/ABPindo/indonesianadblockrules/master/subscriptions/abpindo.txt"
- },
+ userFiltersPath: 'user-filters',
+ pslAssetKey: 'public_suffix_list.dat',
+
+ availableFilterLists: {},
selfieAfter: 23 * oneMinute,
diff --git a/src/js/logger.js b/src/js/logger.js
index b9a7bee3..700521ab 100644
--- a/src/js/logger.js
+++ b/src/js/logger.js
@@ -19,15 +19,13 @@
Home: https://github.com/gorhill/uBlock
*/
-/* global µBlock */
+'use strict';
/******************************************************************************/
/******************************************************************************/
µBlock.logger = (function() {
-'use strict';
-
/******************************************************************************/
/******************************************************************************/
diff --git a/src/js/messaging.js b/src/js/messaging.js
index 360fc7e0..4cc466f7 100644
--- a/src/js/messaging.js
+++ b/src/js/messaging.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -77,7 +77,7 @@ var onMessage = function(request, sender, callback) {
return;
case 'reloadAllFilters':
- µb.reloadAllFilters(callback);
+ µb.loadFilterLists();
return;
case 'scriptlet':
@@ -121,7 +121,8 @@ var onMessage = function(request, sender, callback) {
break;
case 'forceUpdateAssets':
- µb.assetUpdater.force();
+ µb.scheduleAssetUpdater(0);
+ µb.assets.updateStart({ delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod || 2000 });
break;
case 'getAppData':
@@ -160,7 +161,7 @@ var onMessage = function(request, sender, callback) {
break;
case 'selectFilterLists':
- µb.selectFilterLists(request.switches);
+ µb.saveSelectedFilterLists(request.keys, request.append);
break;
case 'setWhitelist':
@@ -753,7 +754,7 @@ var backupUserData = function(callback) {
timeStamp: Date.now(),
version: vAPI.app.version,
userSettings: µb.userSettings,
- filterLists: {},
+ selectedFilterLists: [],
hiddenSettingsString: µb.stringFromHiddenSettings(),
netWhitelist: µb.stringFromWhitelist(µb.netWhitelist),
dynamicFilteringString: µb.permanentFirewall.toString(),
@@ -762,8 +763,17 @@ var backupUserData = function(callback) {
userFilters: ''
};
- var onSelectedListsReady = function(filterLists) {
- userData.filterLists = filterLists;
+ var onSelectedListsReady = function(selectedFilterLists) {
+ userData.selectedFilterLists = selectedFilterLists;
+
+ // TODO(seamless migration):
+ // The following is strictly for convenience, to be minimally
+ // forward-compatible. This will definitely be removed in the
+ // short term, as I do not expect the need to install an older
+ // version of uBO to ever be needed beyond the short term.
+ // >>>>>>>>
+ userData.filterLists = µb.oldDataFromNewListKeys(selectedFilterLists);
+ // <<<<<<<<
var filename = vAPI.i18n('aboutBackupFilename')
.replace('{{datetime}}', µb.dateNowToSensibleString())
@@ -773,17 +783,15 @@ var backupUserData = function(callback) {
'url': 'data:text/plain;charset=utf-8,' + encodeURIComponent(JSON.stringify(userData, null, ' ')),
'filename': filename
});
-
µb.restoreBackupSettings.lastBackupFile = filename;
µb.restoreBackupSettings.lastBackupTime = Date.now();
vAPI.storage.set(µb.restoreBackupSettings);
-
getLocalData(callback);
};
var onUserFiltersReady = function(details) {
userData.userFilters = details.content;
- µb.extractSelectedFilterLists(onSelectedListsReady);
+ µb.loadSelectedFilterLists(onSelectedListsReady);
};
µb.assets.get(µb.userFiltersPath, onUserFiltersReady);
@@ -791,32 +799,32 @@ var backupUserData = function(callback) {
var restoreUserData = function(request) {
var userData = request.userData;
- var countdown = 8;
- var onCountdown = function() {
- countdown -= 1;
- if ( countdown === 0 ) {
- vAPI.app.restart();
- }
- };
var onAllRemoved = function() {
- // Be sure to adjust `countdown` if adding/removing anything below
- µb.keyvalSetOne('version', userData.version);
µBlock.saveLocalSettings();
- vAPI.storage.set(userData.userSettings, onCountdown);
- µb.keyvalSetOne('remoteBlacklists', userData.filterLists, onCountdown);
+ vAPI.storage.set(userData.userSettings);
µb.hiddenSettingsFromString(userData.hiddenSettingsString || '');
- µb.keyvalSetOne('netWhitelist', userData.netWhitelist || '', onCountdown);
- µb.keyvalSetOne('dynamicFilteringString', userData.dynamicFilteringString || '', onCountdown);
- µb.keyvalSetOne('urlFilteringString', userData.urlFilteringString || '', onCountdown);
- µb.keyvalSetOne('hostnameSwitchesString', userData.hostnameSwitchesString || '', onCountdown);
- µb.assets.put(µb.userFiltersPath, userData.userFilters, onCountdown);
vAPI.storage.set({
+ netWhitelist: userData.netWhitelist || '',
+ dynamicFilteringString: userData.dynamicFilteringString || '',
+ urlFilteringString: userData.urlFilteringString || '',
+ hostnameSwitchesString: userData.hostnameSwitchesString || '',
lastRestoreFile: request.file || '',
lastRestoreTime: Date.now(),
lastBackupFile: '',
lastBackupTime: 0
- }, onCountdown);
+ });
+ µb.assets.put(µb.userFiltersPath, userData.userFilters);
+
+ // 'filterLists' is available up to uBO v1.10.4, not beyond.
+ // 'selectedFilterLists' is available from uBO v1.11 and beyond.
+ if ( Array.isArray(userData.selectedFilterLists) ) {
+ µb.saveSelectedFilterLists(userData.selectedFilterLists);
+ } else if ( userData.filterLists instanceof Object ) {
+ µb.saveSelectedFilterLists(µb.newListKeysFromOldData(userData.filterLists));
+ }
+
+ vAPI.app.restart();
};
// https://github.com/chrisaljoudi/uBlock/issues/1102
@@ -848,9 +856,7 @@ var prepListEntries = function(entries) {
var µburi = µb.URI;
var entry, hn;
for ( var k in entries ) {
- if ( entries.hasOwnProperty(k) === false ) {
- continue;
- }
+ if ( entries.hasOwnProperty(k) === false ) { continue; }
entry = entries[k];
if ( typeof entry.supportURL === 'string' && entry.supportURL !== '' ) {
entry.supportName = µburi.hostnameFromURI(entry.supportURL);
@@ -869,16 +875,14 @@ var getLists = function(callback) {
cache: null,
parseCosmeticFilters: µb.userSettings.parseAllABPHideFilters,
cosmeticFilterCount: µb.cosmeticFilteringEngine.getFilterCount(),
- current: µb.remoteBlacklists,
+ current: µb.availableFilterLists,
ignoreGenericCosmeticFilters: µb.userSettings.ignoreGenericCosmeticFilters,
- manualUpdate: false,
netFilterCount: µb.staticNetFilteringEngine.getFilterCount(),
- userFiltersPath: µb.userFiltersPath
+ userFiltersPath: µb.userFiltersPath,
+ aliases: µb.assets.listKeyAliases
};
var onMetadataReady = function(entries) {
r.cache = entries;
- r.manualUpdate = µb.assetUpdater.manualUpdate;
- r.manualUpdateProgress = µb.assetUpdater.manualUpdateProgress;
prepListEntries(r.cache);
callback(r);
};
@@ -952,9 +956,6 @@ var onMessage = function(request, sender, callback) {
case 'getLocalData':
return getLocalData(callback);
- case 'purgeAllCaches':
- return µb.assets.purgeAll(callback);
-
case 'readUserFilters':
return µb.loadUserFilters(callback);
@@ -973,8 +974,18 @@ var onMessage = function(request, sender, callback) {
response = getRules();
break;
+ case 'purgeAllCaches':
+ if ( request.hard ) {
+ µb.assets.remove(/./);
+ } else {
+ µb.assets.remove(/compiled\//);
+ µb.assets.purge(/./);
+ }
+ break;
+
case 'purgeCache':
- µb.assets.purgeCacheableAsset(request.path);
+ µb.assets.purge(request.assetKey);
+ µb.assets.remove('compiled/' + request.assetKey);
break;
case 'readHiddenSettings':
diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js
index 473bffa7..dfae64ff 100644
--- a/src/js/redirect-engine.js
+++ b/src/js/redirect-engine.js
@@ -402,27 +402,15 @@ RedirectEngine.prototype.resourceContentFromName = function(name, mime) {
// TODO: combine same key-redirect pairs into a single regex.
RedirectEngine.prototype.resourcesFromString = function(text) {
- var textEnd = text.length;
- var lineBeg = 0, lineEnd;
- var line, fields, encoded;
- var reNonEmptyLine = /\S/;
+ var line, fields, encoded,
+ reNonEmptyLine = /\S/,
+ lineIter = new µBlock.LineIterator(text);
this.resources = new Map();
- while ( lineBeg < textEnd ) {
- lineEnd = text.indexOf('\n', lineBeg);
- if ( lineEnd < 0 ) {
- lineEnd = text.indexOf('\r', lineBeg);
- if ( lineEnd < 0 ) {
- lineEnd = textEnd;
- }
- }
- line = text.slice(lineBeg, lineEnd);
- lineBeg = lineEnd + 1;
-
- if ( line.startsWith('#') ) {
- continue;
- }
+ while ( lineIter.eot() === false ) {
+ line = lineIter.next();
+ if ( line.startsWith('#') ) { continue; }
if ( fields === undefined ) {
fields = line.trim().split(/\s+/);
diff --git a/src/js/reverselookup-worker.js b/src/js/reverselookup-worker.js
index f52002e6..17e8f991 100644
--- a/src/js/reverselookup-worker.js
+++ b/src/js/reverselookup-worker.js
@@ -1,7 +1,7 @@
/*******************************************************************************
- uBlock - a browser extension to block requests.
- Copyright (C) 2015 Raymond Hill
+ uBlock Origin - a browser extension to block requests.
+ Copyright (C) 2015-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -43,8 +43,8 @@ var fromNetFilter = function(details) {
var lists = [];
var compiledFilter = details.compiledFilter;
var entry, content, pos, c;
- for ( var path in listEntries ) {
- entry = listEntries[path];
+ for ( var assetKey in listEntries ) {
+ entry = listEntries[assetKey];
if ( entry === undefined ) {
continue;
}
@@ -173,11 +173,11 @@ var fromCosmeticFilter = function(details) {
);
}
- var re, path, entry;
+ var re, assetKey, entry;
for ( var candidate in candidates ) {
re = candidates[candidate];
- for ( path in listEntries ) {
- entry = listEntries[path];
+ for ( assetKey in listEntries ) {
+ entry = listEntries[assetKey];
if ( entry === undefined ) {
continue;
}
@@ -206,7 +206,7 @@ var reHighMedium = /^\[href\^="https?:\/\/([^"]{8})[^"]*"\]$/;
/******************************************************************************/
-onmessage = function(e) {
+onmessage = function(e) { // jshint ignore:line
var msg = e.data;
switch ( msg.what ) {
@@ -215,7 +215,7 @@ onmessage = function(e) {
break;
case 'setList':
- listEntries[msg.details.path] = msg.details;
+ listEntries[msg.details.assetKey] = msg.details;
break;
case 'fromNetFilter':
diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js
index 79af14e7..c18bfba6 100644
--- a/src/js/reverselookup.js
+++ b/src/js/reverselookup.js
@@ -1,7 +1,7 @@
/*******************************************************************************
- uBlock - a browser extension to block requests.
- Copyright (C) 2015 Raymond Hill
+ uBlock Origin - a browser extension to block requests.
+ Copyright (C) 2015-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -19,14 +19,12 @@
Home: https://github.com/gorhill/uBlock
*/
-/* global µBlock */
+'use strict';
/******************************************************************************/
µBlock.staticFilteringReverseLookup = (function() {
-'use strict';
-
/******************************************************************************/
var worker = null;
@@ -77,16 +75,16 @@ var initWorker = function(callback) {
var countdown = 0;
var onListLoaded = function(details) {
- var entry = entries[details.path];
+ var entry = entries[details.assetKey];
// https://github.com/gorhill/uBlock/issues/536
- // Use path string when there is no filter list title.
+ // Use assetKey when there is no filter list title.
worker.postMessage({
what: 'setList',
details: {
- path: details.path,
- title: entry.title || details.path,
+ assetKey: details.assetKey,
+ title: entry.title || details.assetKey,
supportURL: entry.supportURL,
content: details.content
}
@@ -99,18 +97,18 @@ var initWorker = function(callback) {
};
var µb = µBlock;
- var path, entry;
+ var listKey, entry;
- for ( path in µb.remoteBlacklists ) {
- if ( µb.remoteBlacklists.hasOwnProperty(path) === false ) {
- continue;
- }
- entry = µb.remoteBlacklists[path];
- if ( entry.off === true ) {
+ for ( listKey in µb.availableFilterLists ) {
+ if ( µb.availableFilterLists.hasOwnProperty(listKey) === false ) {
continue;
}
- entries[path] = {
- title: path !== µb.userFiltersPath ? entry.title : vAPI.i18n('1pPageName'),
+ entry = µb.availableFilterLists[listKey];
+ if ( entry.off === true ) { continue; }
+ entries[listKey] = {
+ title: listKey !== µb.userFiltersPath ?
+ entry.title :
+ vAPI.i18n('1pPageName'),
supportURL: entry.supportURL || ''
};
countdown += 1;
@@ -121,8 +119,8 @@ var initWorker = function(callback) {
return;
}
- for ( path in entries ) {
- µb.getCompiledFilterList(path, onListLoaded);
+ for ( listKey in entries ) {
+ µb.getCompiledFilterList(listKey, onListLoaded);
}
};
diff --git a/src/js/scriptlets/subscriber.js b/src/js/scriptlets/subscriber.js
index 9c525de0..887e95db 100644
--- a/src/js/scriptlets/subscriber.js
+++ b/src/js/scriptlets/subscriber.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2015-2016 Raymond Hill
+ Copyright (C) 2015-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -21,6 +21,8 @@
/* global vAPI, HTMLDocument */
+'use strict';
+
/******************************************************************************/
// Injected into specific web pages, those which have been pre-selected
@@ -30,8 +32,6 @@
(function() {
-'use strict';
-
/******************************************************************************/
// https://github.com/chrisaljoudi/uBlock/issues/464
@@ -100,7 +100,8 @@ var onAbpLinkClicked = function(ev) {
'scriptlets',
{
what: 'selectFilterLists',
- switches: [ { location: location, off: false } ]
+ keys: [ location ],
+ append: true
},
onListsSelectionDone
);
diff --git a/src/js/settings.js b/src/js/settings.js
index 6e75aa15..16582955 100644
--- a/src/js/settings.js
+++ b/src/js/settings.js
@@ -56,7 +56,10 @@ var handleImportFilePicker = function() {
if ( typeof userData.netWhitelist !== 'string' ) {
throw 'Invalid';
}
- if ( typeof userData.filterLists !== 'object' ) {
+ if (
+ typeof userData.filterLists !== 'object' &&
+ Array.isArray(userData.selectedFilterLists) === false
+ ) {
throw 'Invalid';
}
}
diff --git a/src/js/start.js b/src/js/start.js
index 12a583db..2ccd4fc8 100644
--- a/src/js/start.js
+++ b/src/js/start.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -39,7 +39,7 @@ var µb = µBlock;
vAPI.app.onShutdown = function() {
µb.staticFilteringReverseLookup.shutdown();
- µb.assetUpdater.shutdown();
+ µb.assets.updateStop();
µb.staticNetFilteringEngine.reset();
µb.cosmeticFilteringEngine.reset();
µb.sessionFirewall.reset();
@@ -58,14 +58,8 @@ vAPI.app.onShutdown = function() {
var onAllReady = function() {
// https://github.com/chrisaljoudi/uBlock/issues/184
// Check for updates not too far in the future.
- µb.assetUpdater.onStart.addEventListener(µb.updateStartHandler.bind(µb));
- µb.assetUpdater.onCompleted.addEventListener(µb.updateCompleteHandler.bind(µb));
- µb.assetUpdater.onAssetUpdated.addEventListener(µb.assetUpdatedHandler.bind(µb));
- µb.assets.onAssetRemoved.addListener(µb.assetCacheRemovedHandler.bind(µb));
-
- // Important: remove barrier to remote fetching, this was useful only
- // for launch time.
- µb.assets.remoteFetchBarrier -= 1;
+ µb.assets.addObserver(µb.assetObserver.bind(µb));
+ µb.scheduleAssetUpdater(µb.userSettings.autoUpdate ? 7 * 60 * 1000 : 0);
// vAPI.cloud is optional.
if ( µb.cloudStorageSupported ) {
@@ -129,7 +123,7 @@ var onSelfieReady = function(selfie) {
return false;
}
- µb.remoteBlacklists = selfie.filterLists;
+ µb.availableFilterLists = selfie.availableFilterLists;
µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine);
µb.redirectEngine.fromSelfie(selfie.redirectEngine);
µb.cosmeticFilteringEngine.fromSelfie(selfie.cosmeticFilteringEngine);
@@ -157,12 +151,6 @@ var onUserSettingsReady = function(fetched) {
fromFetch(userSettings, fetched);
- // https://github.com/chrisaljoudi/uBlock/issues/426
- // Important: block remote fetching for when loading assets at launch
- // time.
- µb.assets.autoUpdate = userSettings.autoUpdate;
- µb.assets.autoUpdateDelay = µb.updateAssetsEvery;
-
if ( µb.privacySettingsSupported ) {
vAPI.browserSettings.set({
'hyperlinkAuditing': !userSettings.hyperlinkAuditingDisabled,
@@ -192,7 +180,7 @@ var onUserSettingsReady = function(fetched) {
var onSystemSettingsReady = function(fetched) {
var mustSaveSystemSettings = false;
if ( fetched.compiledMagic !== µb.systemSettings.compiledMagic ) {
- µb.assets.purge(/^cache:\/\/compiled-/);
+ µb.assets.remove(/^compiled\//);
mustSaveSystemSettings = true;
}
if ( fetched.selfieMagic !== µb.systemSettings.selfieMagic ) {
@@ -254,9 +242,6 @@ var fromFetch = function(to, fetched) {
/******************************************************************************/
var onAdminSettingsRestored = function() {
- // Forbid remote fetching of assets
- µb.assets.remoteFetchBarrier += 1;
-
var fetchableProps = {
'compiledMagic': '',
'dynamicFilteringString': 'behind-the-scene * 3p noop\nbehind-the-scene * 3p-frame noop',
diff --git a/src/js/storage.js b/src/js/storage.js
index db079f5a..c5879194 100644
--- a/src/js/storage.js
+++ b/src/js/storage.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -19,7 +19,7 @@
Home: https://github.com/gorhill/uBlock
*/
-/* global YaMD5, objectAssign, punycode, publicSuffixList */
+/* global objectAssign, punycode, publicSuffixList */
'use strict';
@@ -104,6 +104,12 @@
case 'string':
out[name] = value;
break;
+ case 'number':
+ out[name] = parseInt(value, 10);
+ if ( isNaN(out[name]) ) {
+ out[name] = this.hiddenSettingsDefault[name];
+ }
+ break;
default:
break;
}
@@ -151,55 +157,107 @@
this.netWhitelistModifyTime = Date.now();
};
-/******************************************************************************/
+/*******************************************************************************
-// This will remove all unused filter list entries from
-// µBlock.remoteBlacklists`. This helps reduce the size of backup files.
+ TODO(seamless migration):
+ The code related to 'remoteBlacklist' can be removed when I am confident
+ all users have moved to a version of uBO which no longer depends on
+ the property 'remoteBlacklists, i.e. v1.11 and beyond.
-µBlock.extractSelectedFilterLists = function(callback) {
- var µb = this;
+**/
- var onBuiltinListsLoaded = function(details) {
- var builtin;
- try {
- builtin = JSON.parse(details.content);
- } catch (e) {
- builtin = {};
+µBlock.loadSelectedFilterLists = function(callback) {
+ var µb = this;
+ vAPI.storage.get([ 'selectedFilterLists', 'remoteBlacklists' ], function(bin) {
+ if ( !bin || !bin.selectedFilterLists && !bin.remoteBlacklists ) {
+ return callback();
}
-
- var result = JSON.parse(JSON.stringify(µb.remoteBlacklists));
- var entry, builtinPath, defaultState;
-
- for ( var path in result ) {
- if ( result.hasOwnProperty(path) === false ) {
- continue;
- }
- entry = result[path];
- // https://github.com/gorhill/uBlock/issues/277
- // uBlock's filter lists are always enabled by default, so we
- // have to include in backup only those which are turned off.
- if ( path.startsWith('assets/ublock/') ) {
- if ( entry.off !== true ) {
- delete result[path];
- }
- continue;
- }
- builtinPath = path.replace(/^assets\/thirdparties\//, '');
- defaultState = builtin.hasOwnProperty(builtinPath) === false ||
- builtin[builtinPath].off === true;
- if ( entry.off === true && entry.off === defaultState ) {
- delete result[path];
+ var listKeys = [];
+ if ( bin.selectedFilterLists ) {
+ listKeys = bin.selectedFilterLists;
+ }
+ if ( bin.remoteBlacklists ) {
+ var oldListKeys = µb.newListKeysFromOldData(bin.remoteBlacklists);
+ if ( oldListKeys.sort().join() !== listKeys.sort().join() ) {
+ listKeys = oldListKeys;
+ µb.saveSelectedFilterLists(listKeys);
}
+ // TODO(seamless migration):
+ // Uncomment when all have moved to v1.11 and beyond.
+ //vAPI.storage.remove('remoteBlacklists');
}
+ callback(listKeys);
+ });
+};
- callback(result);
+µBlock.saveSelectedFilterLists = function(listKeys, append) {
+ var µb = this;
+ var save = function(keys) {
+ var bin = {
+ selectedFilterLists: keys,
+ remoteBlacklists: µb.oldDataFromNewListKeys(keys)
+ };
+ vAPI.storage.set(bin);
};
+ if ( append ) {
+ this.loadSelectedFilterLists(function(keys) {
+ listKeys = listKeys.concat(keys || []);
+ save(listKeys);
+ });
+ } else {
+ save(listKeys);
+ }
+};
+
+// TODO(seamless migration):
+// Remove when all have moved to v1.11 and beyond.
+// >>>>>>>>
+µBlock.newListKeysFromOldData = function(oldLists) {
+ var aliases = this.assets.listKeyAliases,
+ listKeys = [], newKey;
+ for ( var oldKey in oldLists ) {
+ if ( oldLists[oldKey].off !== true ) {
+ newKey = aliases[oldKey];
+ listKeys.push(newKey ? newKey : oldKey);
+ }
+ }
+ return listKeys;
+};
- // https://github.com/gorhill/uBlock/issues/63
- // Get built-in block lists: this will help us determine whether a
- // specific list must be included in the result.
- this.loadAndPatchStockFilterLists(onBuiltinListsLoaded);
+µBlock.oldDataFromNewListKeys = function(selectedFilterLists) {
+ var µb = this,
+ remoteBlacklists = {};
+ var reverseAliases = Object.keys(this.assets.listKeyAliases).reduce(
+ function(a, b) {
+ a[µb.assets.listKeyAliases[b]] = b; return a;
+ },
+ {}
+ );
+ remoteBlacklists = selectedFilterLists.reduce(
+ function(a, b) {
+ a[reverseAliases[b] || b] = { off: false };
+ return a;
+ },
+ {}
+ );
+ remoteBlacklists = Object.keys(µb.assets.listKeyAliases).reduce(
+ function(a, b) {
+ var aliases = µb.assets.listKeyAliases;
+ if (
+ b.startsWith('assets/') &&
+ aliases[b] !== 'public_suffix_list.dat' &&
+ aliases[b] !== 'ublock-resources' &&
+ !a[b]
+ ) {
+ a[b] = { off: true };
+ }
+ return a;
+ },
+ remoteBlacklists
+ );
+ return remoteBlacklists;
};
+// <<<<<<<<
/******************************************************************************/
@@ -207,14 +265,11 @@
// https://github.com/gorhill/uBlock/issues/1022
// Be sure to end with an empty line.
content = content.trim();
- if ( content !== '' ) {
- content += '\n';
- }
+ if ( content !== '' ) { content += '\n'; }
this.assets.put(this.userFiltersPath, content, callback);
+ this.removeCompiledFilterList(this.userFiltersPath);
};
-/******************************************************************************/
-
µBlock.loadUserFilters = function(callback) {
return this.assets.get(this.userFiltersPath, callback);
};
@@ -222,25 +277,23 @@
/******************************************************************************/
µBlock.appendUserFilters = function(filters) {
- if ( filters.length === 0 ) {
- return;
- }
+ if ( filters.length === 0 ) { return; }
var µb = this;
var onSaved = function() {
- var compiledFilters = µb.compileFilters(filters);
- var snfe = µb.staticNetFilteringEngine;
- var cfe = µb.cosmeticFilteringEngine;
- var acceptedCount = snfe.acceptedCount + cfe.acceptedCount;
- var discardedCount = snfe.discardedCount + cfe.discardedCount;
+ var compiledFilters = µb.compileFilters(filters),
+ snfe = µb.staticNetFilteringEngine,
+ cfe = µb.cosmeticFilteringEngine,
+ acceptedCount = snfe.acceptedCount + cfe.acceptedCount,
+ discardedCount = snfe.discardedCount + cfe.discardedCount;
µb.applyCompiledFilters(compiledFilters, true);
- var entry = µb.remoteBlacklists[µb.userFiltersPath];
- var deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount;
- var deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount);
+ var entry = µb.availableFilterLists[µb.userFiltersPath],
+ deltaEntryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount,
+ deltaEntryUsedCount = deltaEntryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount);
entry.entryCount += deltaEntryCount;
entry.entryUsedCount += deltaEntryUsedCount;
- vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists });
+ vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists });
µb.staticNetFilteringEngine.freeze();
µb.redirectEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
@@ -248,9 +301,7 @@
};
var onLoaded = function(details) {
- if ( details.error ) {
- return;
- }
+ if ( details.error ) { return; }
// https://github.com/chrisaljoudi/uBlock/issues/976
// If we reached this point, the filter quite probably needs to be
// added for sure: do not try to be too smart, trying to avoid
@@ -263,166 +314,194 @@
/******************************************************************************/
-µBlock.getAvailableLists = function(callback) {
- var availableLists = {};
- var relocationMap = {};
-
- var fixLocation = function(location) {
- // https://github.com/chrisaljoudi/uBlock/issues/418
- // We now support built-in external filter lists
- if ( /^https?:/.test(location) === false ) {
- location = 'assets/thirdparties/' + location;
+µBlock.listKeysFromCustomFilterLists = function(raw) {
+ var out = {};
+ var reIgnore = /^[!#]|[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/,
+ lineIter = new this.LineIterator(raw),
+ location;
+ while ( lineIter.eot() === false ) {
+ location = lineIter.next().trim();
+ if ( location === '' || reIgnore.test(location) ) { continue; }
+ out[location] = true;
+ }
+ return Object.keys(out);
+};
+
+/******************************************************************************/
+
+µBlock.autoSelectRegionalFilterLists = function(lists) {
+ var lang = self.navigator.language.slice(0, 2),
+ selectedListKeys = [],
+ list;
+ for ( var key in lists ) {
+ if ( lists.hasOwnProperty(key) === false ) { continue; }
+ list = lists[key];
+ if ( list.off !== true ) {
+ selectedListKeys.push(key);
+ continue;
}
- return location;
- };
+ if ( list.lang === lang ) {
+ selectedListKeys.push(key);
+ list.off = false;
+ }
+ }
+ return selectedListKeys;
+};
- // selected lists
- var onSelectedListsLoaded = function(store) {
- var µb = µBlock;
- var lists = store.remoteBlacklists;
- var locations = Object.keys(lists);
- var location, availableEntry, storedEntry;
- var off;
-
- while ( (location = locations.pop()) ) {
- storedEntry = lists[location];
- off = storedEntry.off === true;
- // New location?
- if ( relocationMap.hasOwnProperty(location) ) {
- µb.purgeFilterList(location);
- location = relocationMap[location];
- if ( off && lists.hasOwnProperty(location) ) {
- off = lists[location].off === true;
- }
+/******************************************************************************/
+
+µBlock.changeExternalFilterLists = function(before, after) {
+ var µb = µBlock;
+ var onLoaded = function(keys) {
+ var fullDict = new Set(keys || []),
+ mustSave = false,
+ oldKeys = µb.listKeysFromCustomFilterLists(before),
+ oldDict = new Set(oldKeys),
+ newKeys = µb.listKeysFromCustomFilterLists(after),
+ newDict = new Set(newKeys),
+ i, key;
+ i = oldKeys.length;
+ while ( i-- ) {
+ key = oldKeys[i];
+ if ( fullDict.has(key) && !newDict.has(key) ) {
+ fullDict.delete(key);
+ mustSave = true;
}
- availableEntry = availableLists[location];
- if ( availableEntry === undefined ) {
- µb.purgeFilterList(location);
- continue;
+ }
+ i = newKeys.length;
+ while ( i-- ) {
+ key = newKeys[i];
+ if ( !fullDict.has(key) && !oldDict.has(key) ) {
+ fullDict.add(key);
+ mustSave = true;
}
- availableEntry.off = off;
- if ( typeof availableEntry.homeURL === 'string' ) {
- µb.assets.setHomeURL(location, availableEntry.homeURL);
+ }
+ if ( mustSave ) {
+ µb.saveSelectedFilterLists(µb.setToArray(fullDict));
+ }
+ };
+ this.loadSelectedFilterLists(onLoaded);
+};
+
+/******************************************************************************/
+
+µBlock.getAvailableLists = function(callback) {
+ var µb = this,
+ oldAvailableLists = {},
+ newAvailableLists = {};
+
+ // User filter list.
+ newAvailableLists[this.userFiltersPath] = {
+ group: 'default',
+ title: vAPI.i18n('1pPageName')
+ };
+
+ // Custom filter lists.
+ var importedListKeys = this.listKeysFromCustomFilterLists(µb.userSettings.externalLists),
+ i = importedListKeys.length, listKey, entry;
+ while ( i-- ) {
+ listKey = importedListKeys[i];
+ entry = {
+ content: 'filters',
+ contentURL: importedListKeys[i],
+ external: true,
+ group: 'custom',
+ submitter: 'user',
+ title: ''
+ };
+ newAvailableLists[listKey] = entry;
+ this.assets.registerAssetSource(listKey, entry);
+ }
+
+ // Final steps:
+ // - reuse existing list metadata if any;
+ // - unregister unreferenced imported filter lists if any.
+ var finalize = function() {
+ var assetKey, newEntry, oldEntry;
+
+ // Reuse existing metadata.
+ for ( assetKey in oldAvailableLists ) {
+ oldEntry = oldAvailableLists[assetKey];
+ newEntry = newAvailableLists[assetKey];
+ if ( newEntry === undefined ) {
+ µb.removeFilterList(assetKey);
+ continue;
}
- if ( storedEntry.entryCount !== undefined ) {
- availableEntry.entryCount = storedEntry.entryCount;
+ if ( oldEntry.entryCount !== undefined ) {
+ newEntry.entryCount = oldEntry.entryCount;
}
- if ( storedEntry.entryUsedCount !== undefined ) {
- availableEntry.entryUsedCount = storedEntry.entryUsedCount;
+ if ( oldEntry.entryUsedCount !== undefined ) {
+ newEntry.entryUsedCount = oldEntry.entryUsedCount;
}
// This may happen if the list name was pulled from the list
// content.
// https://github.com/chrisaljoudi/uBlock/issues/982
// There is no guarantee the title was successfully extracted from
// the list content.
- if ( availableEntry.title === '' &&
- typeof storedEntry.title === 'string' &&
- storedEntry.title !== ''
+ if (
+ newEntry.title === '' &&
+ typeof oldEntry.title === 'string' &&
+ oldEntry.title !== ''
) {
- availableEntry.title = storedEntry.title;
+ newEntry.title = oldEntry.title;
}
}
- // https://github.com/gorhill/uBlock/issues/747
- if ( µb.firstInstall ) {
- µb.autoSelectFilterLists(availableLists);
+ // Remove unreferenced imported filter lists.
+ var dict = new Set(importedListKeys);
+ for ( assetKey in newAvailableLists ) {
+ newEntry = newAvailableLists[assetKey];
+ if ( newEntry.submitter !== 'user' ) { continue; }
+ if ( dict.has(assetKey) ) { continue; }
+ delete newAvailableLists[assetKey];
+ µb.assets.unregisterAssetSource(assetKey);
+ µb.removeFilterList(assetKey);
}
-
- callback(availableLists);
};
- // built-in lists
- var onBuiltinListsLoaded = function(details) {
- var location, locations;
- try {
- locations = JSON.parse(details.content);
- } catch (e) {
- locations = {};
- }
- var entry;
- for ( location in locations ) {
- if ( locations.hasOwnProperty(location) === false ) {
- continue;
- }
- entry = locations[location];
- location = fixLocation(location);
- // Migrate obsolete location to new location, if any
- if ( typeof entry.oldLocation === 'string' ) {
- entry.oldLocation = fixLocation(entry.oldLocation);
- relocationMap[entry.oldLocation] = location;
+ // Selected lists.
+ var onSelectedListsLoaded = function(keys) {
+ var listKey;
+ // No user lists data means use default settings.
+ if ( Array.isArray(keys) ) {
+ var listKeySet = new Set(keys);
+ for ( listKey in newAvailableLists ) {
+ if ( newAvailableLists.hasOwnProperty(listKey) ) {
+ newAvailableLists[listKey].off = !listKeySet.has(listKey);
+ }
}
- availableLists[location] = entry;
+ } else if ( µb.firstInstall ) {
+ µb.saveSelectedFilterLists(µb.autoSelectRegionalFilterLists(newAvailableLists));
}
- // Now get user's selection of lists
- vAPI.storage.get(
- { 'remoteBlacklists': availableLists },
- onSelectedListsLoaded
- );
+ finalize();
+ callback(newAvailableLists);
};
- // permanent lists
- var location;
- var lists = this.permanentLists;
- for ( location in lists ) {
- if ( lists.hasOwnProperty(location) === false ) {
- continue;
- }
- availableLists[location] = lists[location];
- }
-
- // custom lists
- var c;
- var locations = this.userSettings.externalLists.split('\n');
- for ( var i = 0; i < locations.length; i++ ) {
- location = locations[i].trim();
- c = location.charAt(0);
- if ( location === '' || c === '!' || c === '#' ) {
- continue;
- }
- // Coarse validation
- if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) {
- continue;
+ // Built-in filter lists.
+ var onBuiltinListsLoaded = function(entries) {
+ for ( var assetKey in entries ) {
+ if ( entries.hasOwnProperty(assetKey) === false ) { continue; }
+ entry = entries[assetKey];
+ if ( entry.content !== 'filters' ) { continue; }
+ newAvailableLists[assetKey] = objectAssign({}, entry);
}
- availableLists[location] = {
- title: '',
- group: 'custom',
- external: true
- };
- }
- // get built-in block lists.
- this.loadAndPatchStockFilterLists(onBuiltinListsLoaded);
-};
-
-/******************************************************************************/
-
-µBlock.autoSelectFilterLists = function(lists) {
- var lang = self.navigator.language.slice(0, 2),
- list;
- for ( var path in lists ) {
- if ( lists.hasOwnProperty(path) === false ) {
- continue;
- }
- list = lists[path];
- if ( list.off !== true ) {
- continue;
- }
- if ( list.lang === lang ) {
- list.off = false;
- }
- }
-};
+ // Load set of currently selected filter lists.
+ µb.loadSelectedFilterLists(onSelectedListsLoaded);
+ };
-/******************************************************************************/
+ // Available lists previously computed.
+ var onOldAvailableListsLoaded = function(bin) {
+ oldAvailableLists = bin && bin.availableFilterLists || {};
+ µb.assets.metadata(onBuiltinListsLoaded);
+ };
-µBlock.createShortUniqueId = function(path) {
- var md5 = YaMD5.hashStr(path);
- return md5.slice(0, 4) + md5.slice(-4);
+ // Load previously saved available lists -- these contains data
+ // computed at run-time, we will reuse this data if possible.
+ vAPI.storage.get('availableFilterLists', onOldAvailableListsLoaded);
};
-µBlock.createShortUniqueId.idLength = 8;
-
/******************************************************************************/
// This is used to be re-entrancy resistant.
@@ -444,18 +523,11 @@
callback = this.noopFunc;
}
- // Never fetch from remote servers when we load filter lists: this has to
- // be as fast as possible.
- µb.assets.remoteFetchBarrier += 1;
-
var onDone = function() {
- // Remove barrier to remote fetching
- µb.assets.remoteFetchBarrier -= 1;
-
µb.staticNetFilteringEngine.freeze();
µb.cosmeticFilteringEngine.freeze();
µb.redirectEngine.freeze();
- vAPI.storage.set({ 'remoteBlacklists': µb.remoteBlacklists });
+ vAPI.storage.set({ 'availableFilterLists': µb.availableFilterLists });
//quickProfiler.stop(0);
@@ -473,15 +545,15 @@
var acceptedCount = snfe.acceptedCount + cfe.acceptedCount;
var discardedCount = snfe.discardedCount + cfe.discardedCount;
µb.applyCompiledFilters(compiled, path === µb.userFiltersPath);
- if ( µb.remoteBlacklists.hasOwnProperty(path) ) {
- var entry = µb.remoteBlacklists[path];
+ if ( µb.availableFilterLists.hasOwnProperty(path) ) {
+ var entry = µb.availableFilterLists[path];
entry.entryCount = snfe.acceptedCount + cfe.acceptedCount - acceptedCount;
entry.entryUsedCount = entry.entryCount - (snfe.discardedCount + cfe.discardedCount - discardedCount);
}
};
var onCompiledListLoaded = function(details) {
- applyCompiledFilters(details.path, details.content);
+ applyCompiledFilters(details.assetKey, details.content);
filterlistsCount -= 1;
if ( filterlistsCount === 0 ) {
onDone();
@@ -489,7 +561,7 @@
};
var onFilterListsReady = function(lists) {
- µb.remoteBlacklists = lists;
+ µb.availableFilterLists = lists;
µb.redirectEngine.reset();
µb.cosmeticFilteringEngine.reset();
@@ -502,14 +574,10 @@
// This happens for assets which do not exist, ot assets with no
// content.
var toLoad = [];
- for ( var path in lists ) {
- if ( lists.hasOwnProperty(path) === false ) {
- continue;
- }
- if ( lists[path].off ) {
- continue;
- }
- toLoad.push(path);
+ for ( var assetKey in lists ) {
+ if ( lists.hasOwnProperty(assetKey) === false ) { continue; }
+ if ( lists[assetKey].off ) { continue; }
+ toLoad.push(assetKey);
}
filterlistsCount = toLoad.length;
if ( filterlistsCount === 0 ) {
@@ -528,32 +596,17 @@
/******************************************************************************/
-µBlock.getCompiledFilterListPath = function(path) {
- return 'cache://compiled-filter-list:' + this.createShortUniqueId(path);
-};
-
-/******************************************************************************/
-
-µBlock.getCompiledFilterList = function(path, callback) {
- var compiledPath = this.getCompiledFilterListPath(path);
- var µb = this;
+µBlock.getCompiledFilterList = function(assetKey, callback) {
+ var µb = this,
+ compiledPath = 'compiled/' + assetKey;
var onRawListLoaded = function(details) {
+ details.assetKey = assetKey;
if ( details.content === '' ) {
callback(details);
return;
}
- var listMeta = µb.remoteBlacklists[path];
- // https://github.com/gorhill/uBlock/issues/313
- // Always try to fetch the name if this is an external filter list.
- if ( listMeta && (listMeta.title === '' || listMeta.group === 'custom') ) {
- var matches = details.content.slice(0, 1024).match(/(?:^|\n)!\s*Title:([^\n]+)/i);
- if ( matches !== null ) {
- listMeta.title = matches[1].trim();
- }
- }
-
- //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path);
+ µb.extractFilterListMetadata(assetKey, details.content);
details.content = µb.compileFilters(details.content);
µb.assets.put(compiledPath, details.content);
callback(details);
@@ -561,12 +614,10 @@
var onCompiledListLoaded = function(details) {
if ( details.content === '' ) {
- //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: no compiled version for "%s"', path);
- µb.assets.get(path, onRawListLoaded);
+ µb.assets.get(assetKey, onRawListLoaded);
return;
}
- //console.debug('µBlock.getCompiledFilterList/onCompiledListLoaded: using compiled version for "%s"', path);
- details.path = path;
+ details.assetKey = assetKey;
callback(details);
};
@@ -575,61 +626,70 @@
/******************************************************************************/
-µBlock.purgeCompiledFilterList = function(path) {
- this.assets.purge(this.getCompiledFilterListPath(path));
+µBlock.extractFilterListMetadata = function(assetKey, raw) {
+ var listEntry = this.availableFilterLists[assetKey];
+ if ( listEntry === undefined ) { return; }
+ // Metadata expected to be found at the top of content.
+ var head = raw.slice(0, 1024),
+ matches, v;
+ // https://github.com/gorhill/uBlock/issues/313
+ // Always try to fetch the name if this is an external filter list.
+ if ( listEntry.title === '' || listEntry.group === 'custom' ) {
+ matches = head.match(/(?:^|\n)!\s*Title:([^\n]+)/i);
+ if ( matches !== null ) {
+ listEntry.title = matches[1].trim();
+ }
+ }
+ // Extract update frequency information
+ matches = head.match(/(?:^|\n)![\t ]*Expires:[\t ]*([\d]+)[\t ]*days?/i);
+ if ( matches !== null ) {
+ v = Math.max(parseInt(matches[1], 10), 2);
+ if ( v !== listEntry.updateAfter ) {
+ this.assets.registerAssetSource(assetKey, { updateAfter: v });
+ }
+ }
};
/******************************************************************************/
-µBlock.purgeFilterList = function(path) {
- this.purgeCompiledFilterList(path);
- this.assets.purge(path);
+µBlock.removeCompiledFilterList = function(assetKey) {
+ this.assets.remove('compiled/' + assetKey);
+};
+
+µBlock.removeFilterList = function(assetKey) {
+ this.removeCompiledFilterList(assetKey);
+ this.assets.remove(assetKey);
};
/******************************************************************************/
µBlock.compileFilters = function(rawText) {
- var rawEnd = rawText.length;
var compiledFilters = [];
// Useful references:
// https://adblockplus.org/en/filter-cheatsheet
// https://adblockplus.org/en/filters
- var staticNetFilteringEngine = this.staticNetFilteringEngine;
- var cosmeticFilteringEngine = this.cosmeticFilteringEngine;
- var reIsWhitespaceChar = /\s/;
- var reMaybeLocalIp = /^[\d:f]/;
- var reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/;
- var reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/;
-
- var lineBeg = 0, lineEnd, currentLineBeg;
- var line, lineRaw, c, pos;
-
- while ( lineBeg < rawEnd ) {
- lineEnd = rawText.indexOf('\n', lineBeg);
- if ( lineEnd === -1 ) {
- lineEnd = rawText.indexOf('\r', lineBeg);
- if ( lineEnd === -1 ) {
- lineEnd = rawEnd;
- }
- }
+ var staticNetFilteringEngine = this.staticNetFilteringEngine,
+ cosmeticFilteringEngine = this.cosmeticFilteringEngine,
+ reIsWhitespaceChar = /\s/,
+ reMaybeLocalIp = /^[\d:f]/,
+ reIsLocalhostRedirect = /\s+(?:broadcasthost|local|localhost|localhost\.localdomain)(?=\s|$)/,
+ reLocalIp = /^(?:0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)/,
+ line, lineRaw, c, pos,
+ lineIter = new this.LineIterator(rawText);
+
+ while ( lineIter.eot() === false ) {
+ line = lineRaw = lineIter.next().trim();
// rhill 2014-04-18: The trim is important here, as without it there
// could be a lingering `\r` which would cause problems in the
// following parsing code.
- line = lineRaw = rawText.slice(lineBeg, lineEnd).trim();
- currentLineBeg = lineBeg;
- lineBeg = lineEnd + 1;
- if ( line.length === 0 ) {
- continue;
- }
+ if ( line.length === 0 ) { continue; }
// Strip comments
c = line.charAt(0);
- if ( c === '!' || c === '[' ) {
- continue;
- }
+ if ( c === '!' || c === '[' ) { continue; }
// Parse or skip cosmetic filters
// All cosmetic filters are caught here
@@ -640,9 +700,7 @@
// Whatever else is next can be assumed to not be a cosmetic filter
// Most comments start in first column
- if ( c === '#' ) {
- continue;
- }
+ if ( c === '#' ) { continue; }
// Catch comments somewhere on the line
// Remove:
@@ -663,15 +721,11 @@
// Ignore hosts file redirect configuration
// 127.0.0.1 localhost
// 255.255.255.255 broadcasthost
- if ( reIsLocalhostRedirect.test(line) ) {
- continue;
- }
+ if ( reIsLocalhostRedirect.test(line) ) { continue; }
line = line.replace(reLocalIp, '').trim();
}
- if ( line.length === 0 ) {
- continue;
- }
+ if ( line.length === 0 ) { continue; }
staticNetFilteringEngine.compile(line, compiledFilters);
}
@@ -699,55 +753,6 @@
/******************************************************************************/
-// `switches` contains the filter lists for which the switch must be revisited.
-
-µBlock.selectFilterLists = function(switches) {
- switches = switches || {};
-
- // Only the lists referenced by the switches are touched.
- var filterLists = this.remoteBlacklists;
- var entry, state, location;
- var i = switches.length;
- while ( i-- ) {
- entry = switches[i];
- state = entry.off === true;
- location = entry.location;
- if ( filterLists.hasOwnProperty(location) === false ) {
- if ( state !== true ) {
- filterLists[location] = { off: state };
- }
- continue;
- }
- if ( filterLists[location].off === state ) {
- continue;
- }
- filterLists[location].off = state;
- }
-
- vAPI.storage.set({ 'remoteBlacklists': filterLists });
-};
-
-/******************************************************************************/
-
-// Plain reload of all filters.
-
-µBlock.reloadAllFilters = function() {
- var µb = this;
-
- // We are just reloading the filter lists: we do not want assets to update.
- // TODO: probably not needed anymore, since filter lists are now always
- // loaded without update => see `µb.assets.remoteFetchBarrier`.
- this.assets.autoUpdate = false;
-
- var onFiltersReady = function() {
- µb.assets.autoUpdate = µb.userSettings.autoUpdate;
- };
-
- this.loadFilterLists(onFiltersReady);
-};
-
-/******************************************************************************/
-
µBlock.loadRedirectResources = function(callback) {
var µb = this;
@@ -762,40 +767,46 @@
callback();
};
- this.assets.get('assets/ublock/resources.txt', onResourcesLoaded);
+ this.assets.get('ublock-resources', onResourcesLoaded);
};
/******************************************************************************/
µBlock.loadPublicSuffixList = function(callback) {
- var µb = this;
- var path = µb.pslPath;
- var compiledPath = 'cache://compiled-publicsuffixlist';
+ var µb = this,
+ assetKey = µb.pslAssetKey,
+ compiledAssetKey = 'compiled/' + assetKey;
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
var onRawListLoaded = function(details) {
if ( details.content !== '' ) {
- //console.debug('µBlock.loadPublicSuffixList/onRawListLoaded: compiling "%s"', path);
- publicSuffixList.parse(details.content, punycode.toASCII);
- µb.assets.put(compiledPath, JSON.stringify(publicSuffixList.toSelfie()));
+ µb.compilePublicSuffixList(details.content);
}
callback();
};
var onCompiledListLoaded = function(details) {
if ( details.content === '' ) {
- //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: no compiled version for "%s"', path);
- µb.assets.get(path, onRawListLoaded);
+ µb.assets.get(assetKey, onRawListLoaded);
return;
}
- //console.debug('µBlock.loadPublicSuffixList/onCompiledListLoaded: using compiled version for "%s"', path);
publicSuffixList.fromSelfie(JSON.parse(details.content));
callback();
};
- this.assets.get(compiledPath, onCompiledListLoaded);
+ this.assets.get(compiledAssetKey, onCompiledListLoaded);
+};
+
+/******************************************************************************/
+
+µBlock.compilePublicSuffixList = function(content) {
+ publicSuffixList.parse(content, punycode.toASCII);
+ this.assets.put(
+ 'compiled/' + this.pslAssetKey,
+ JSON.stringify(publicSuffixList.toSelfie())
+ );
};
/******************************************************************************/
@@ -814,7 +825,7 @@
var selfie = {
magic: µb.systemSettings.selfieMagic,
publicSuffixList: publicSuffixList.toSelfie(),
- filterLists: µb.remoteBlacklists,
+ availableFilterLists: µb.availableFilterLists,
staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(),
redirectEngine: µb.redirectEngine.toSelfie(),
cosmeticFilteringEngine: µb.cosmeticFilteringEngine.toSelfie()
@@ -885,6 +896,13 @@
var bin = {};
var binNotEmpty = false;
+ // Allows an admin to set their own 'assets.json' file, with their own
+ // set of stock assets.
+ if ( typeof data.assetsBootstrapLocation === 'string' ) {
+ bin.assetsBootstrapLocation = data.assetsBootstrapLocation;
+ binNotEmpty = true;
+ }
+
if ( typeof data.userSettings === 'object' ) {
for ( var name in µb.userSettings ) {
if ( µb.userSettings.hasOwnProperty(name) === false ) {
@@ -898,8 +916,13 @@
}
}
- if ( typeof data.filterLists === 'object' ) {
- bin.remoteBlacklists = data.filterLists;
+ // 'selectedFilterLists' is an array of filter list tokens. Each token
+ // is a reference to an asset in 'assets.json'.
+ if ( Array.isArray(data.selectedFilterLists) ) {
+ bin.selectedFilterLists = data.selectedFilterLists;
+ binNotEmpty = true;
+ } else if ( typeof data.filterLists === 'object' ) {
+ bin.selectedFilterLists = µb.newListKeysFromOldData(data.filterLists);
binNotEmpty = true;
}
@@ -939,203 +962,95 @@
/******************************************************************************/
-µBlock.updateStartHandler = function(callback) {
- var µb = this;
- var onListsReady = function(lists) {
- var assets = {};
- for ( var location in lists ) {
- if ( lists.hasOwnProperty(location) === false ) {
- continue;
- }
- if ( lists[location].off ) {
- continue;
- }
- assets[location] = true;
- }
- assets[µb.pslPath] = true;
- assets['assets/ublock/resources.txt'] = true;
- callback(assets);
- };
-
- this.getAvailableLists(onListsReady);
-};
-
-/******************************************************************************/
-
-µBlock.assetUpdatedHandler = function(details) {
- var path = details.path || '';
- if ( this.remoteBlacklists.hasOwnProperty(path) === false ) {
- return;
- }
- var entry = this.remoteBlacklists[path];
- if ( entry.off ) {
- return;
- }
- // Compile the list while we have the raw version in memory
- //console.debug('µBlock.getCompiledFilterList/onRawListLoaded: compiling "%s"', path);
- this.assets.put(
- this.getCompiledFilterListPath(path),
- this.compileFilters(details.content)
- );
-};
-
-/******************************************************************************/
-
-µBlock.updateCompleteHandler = function(details) {
- var µb = this;
- var updatedCount = details.updatedCount;
-
- // Assets are supposed to have been all updated, prevent fetching from
- // remote servers.
- µb.assets.remoteFetchBarrier += 1;
-
- var onFiltersReady = function() {
- µb.assets.remoteFetchBarrier -= 1;
- };
-
- var onPSLReady = function() {
- if ( updatedCount !== 0 ) {
- //console.debug('storage.js > µBlock.updateCompleteHandler: reloading filter lists');
- µb.loadFilterLists(onFiltersReady);
- } else {
- onFiltersReady();
+µBlock.scheduleAssetUpdater = (function() {
+ var timer, next = 0;
+ return function(updateDelay) {
+ if ( timer ) {
+ clearTimeout(timer);
+ timer = undefined;
}
- };
-
- if ( details.hasOwnProperty(this.pslPath) ) {
- //console.debug('storage.js > µBlock.updateCompleteHandler: reloading PSL');
- this.loadPublicSuffixList(onPSLReady);
- updatedCount -= 1;
- } else {
- onPSLReady();
- }
-};
-
-/******************************************************************************/
-
-µBlock.assetCacheRemovedHandler = (function() {
- var barrier = false;
-
- var handler = function(paths) {
- if ( barrier ) {
+ if ( updateDelay === 0 ) {
+ next = 0;
return;
}
- barrier = true;
- var i = paths.length;
- var path;
- while ( i-- ) {
- path = paths[i];
- if ( this.remoteBlacklists.hasOwnProperty(path) ) {
- //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path);
- this.purgeCompiledFilterList(path);
- continue;
- }
- if ( path === this.pslPath ) {
- //console.debug('µBlock.assetCacheRemovedHandler: decompiling "%s"', path);
- this.assets.purge('cache://compiled-publicsuffixlist');
- continue;
- }
+ var now = Date.now();
+ // Use the new schedule if and only if it is earlier than the previous
+ // one.
+ if ( next !== 0 ) {
+ updateDelay = Math.min(updateDelay, Math.max(next - now, 0));
}
- this.selfieManager.destroy();
- barrier = false;
+ next = now + updateDelay;
+ timer = vAPI.setTimeout(function() {
+ timer = undefined;
+ next = 0;
+ var µb = µBlock;
+ µb.assets.updateStart({
+ delay: µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 || 120000
+ });
+ }, updateDelay);
};
-
- return handler;
})();
/******************************************************************************/
-// https://github.com/gorhill/uBlock/issues/602
-// - Load and patch `filter-list.json`
-// - Load and patch user's `remoteBlacklists`
-// - Load and patch cached filter lists
-// - Load and patch compiled filter lists
-//
-// Once enough time has passed to safely assume all uBlock Origin
-// installations have been converted to the new stock filter lists, this code
-// can be removed.
-
-µBlock.patchFilterLists = function(filterLists) {
- var modified = false;
- var oldListKey, newListKey, listEntry;
- for ( var listKey in filterLists ) {
- if ( filterLists.hasOwnProperty(listKey) === false ) {
- continue;
- }
- oldListKey = listKey;
- if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) {
- oldListKey = 'assets/thirdparties/' + listKey;
- if ( this.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) {
- continue;
- }
+µBlock.assetObserver = function(topic, details) {
+ // Do not update filter list if not in use.
+ if ( topic === 'before-asset-updated' ) {
+ if (
+ this.availableFilterLists.hasOwnProperty(details.assetKey) &&
+ this.availableFilterLists[details.assetKey].off === true
+ ) {
+ return false;
}
- newListKey = this.oldListToNewListMap[oldListKey];
- // https://github.com/gorhill/uBlock/issues/668
- // https://github.com/gorhill/uBlock/issues/669
- // Beware: an entry for the new list key may already exists. If it is
- // the case, leave it as is.
- if ( newListKey !== '' && filterLists.hasOwnProperty(newListKey) === false ) {
- listEntry = filterLists[listKey];
- listEntry.homeURL = undefined;
- filterLists[newListKey] = listEntry;
- }
- delete filterLists[listKey];
- modified = true;
+ return;
}
- return modified;
-};
-
-µBlock.loadAndPatchStockFilterLists = function(callback) {
- var onStockListsLoaded = function(details) {
- var µb = µBlock;
- var stockLists;
- try {
- stockLists = JSON.parse(details.content);
- } catch (e) {
- stockLists = {};
- }
- // Migrate assets affected by the change to their new name.
- var reExternalURL = /^https?:\/\//;
- var newListKey;
- for ( var oldListKey in stockLists ) {
- if ( stockLists.hasOwnProperty(oldListKey) === false ) {
- continue;
- }
- // https://github.com/gorhill/uBlock/issues/708
- // Support migrating external stock filter lists as well.
- if ( reExternalURL.test(oldListKey) === false ) {
- oldListKey = 'assets/thirdparties/' + oldListKey;
+ // Compile the list while we have the raw version in memory
+ if ( topic === 'after-asset-updated' ) {
+ var cached = typeof details.content === 'string' && details.content !== '';
+ if ( this.availableFilterLists.hasOwnProperty(details.assetKey) ) {
+ if ( cached ) {
+ if ( this.availableFilterLists[details.assetKey].off !== true ) {
+ this.extractFilterListMetadata(
+ details.assetKey,
+ details.content
+ );
+ this.assets.put(
+ 'compiled/' + details.assetKey,
+ this.compileFilters(details.content)
+ );
+ }
+ } else {
+ this.removeCompiledFilterList(details.assetKey);
}
- if ( µb.oldListToNewListMap.hasOwnProperty(oldListKey) === false ) {
- continue;
+ } else if ( details.assetKey === this.pslAssetKey ) {
+ if ( cached ) {
+ this.compilePublicSuffixList(details.content);
}
- newListKey = µb.oldListToNewListMap[oldListKey];
- if ( newListKey === '' ) {
- continue;
+ } else if ( details.assetKey === 'ublock-resources' ) {
+ if ( cached ) {
+ this.redirectEngine.resourcesFromString(details.content);
}
- // Rename cached asset to preserve content -- so it does not
- // need to be fetched from remote server.
- µb.assets.rename(oldListKey, newListKey);
- µb.assets.purge(µb.getCompiledFilterListPath(oldListKey));
}
- µb.patchFilterLists(stockLists);
-
- // Stock lists information cascades into
- // - In-memory user's selected filter lists, so we need to patch this.
- µb.patchFilterLists(µb.remoteBlacklists);
-
- // Stock lists information cascades into
- // - In-storage user's selected filter lists, so we need to patch this.
- vAPI.storage.get('remoteBlacklists', function(bin) {
- var userLists = bin.remoteBlacklists || {};
- if ( µb.patchFilterLists(userLists) ) {
- µb.keyvalSetOne('remoteBlacklists', userLists);
- }
- details.content = JSON.stringify(stockLists);
- callback(details);
+ vAPI.messaging.broadcast({
+ what: 'assetUpdated',
+ key: details.assetKey,
+ cached: cached
+
});
- };
+ return;
+ }
- this.assets.get('assets/ublock/filter-lists.json', onStockListsLoaded);
+ // Reload all filter lists if needed.
+ if ( topic === 'after-assets-updated' ) {
+ if ( details.assetKeys.length !== 0 ) {
+ this.loadFilterLists();
+ }
+ if ( this.userSettings.autoUpdate ) {
+ this.scheduleAssetUpdater(this.hiddenSettings.assetAutoUpdatePeriod * 3600000 || 25200000);
+ } else {
+ this.scheduleAssetUpdater(0);
+ }
+ return;
+ }
};
diff --git a/src/js/ublock.js b/src/js/ublock.js
index 528edb53..f765fbd7 100644
--- a/src/js/ublock.js
+++ b/src/js/ublock.js
@@ -1,7 +1,7 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
- Copyright (C) 2014-2016 Raymond Hill
+ Copyright (C) 2014-2017 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -317,6 +317,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/,
// Pre-change
switch ( name ) {
+ case 'externalLists':
+ this.changeExternalFilterLists(us.externalLists, value);
+ break;
case 'largeMediaSize':
if ( typeof value !== 'number' ) {
value = parseInt(value, 10) || 0;
@@ -340,6 +343,9 @@ var reInvalidHostname = /[^a-z0-9.\-\[\]:]/,
us.dynamicFilteringEnabled = true;
}
break;
+ case 'autoUpdate':
+ this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0);
+ break;
case 'collapseBlocked':
if ( value === false ) {
this.cosmeticFilteringEngine.removeFromSelectorCache('*', 'net');
diff --git a/tools/make-assets.sh b/tools/make-assets.sh
index fade452d..cebea3f4 100755
--- a/tools/make-assets.sh
+++ b/tools/make-assets.sh
@@ -24,8 +24,6 @@ cp -R ../uAssets/thirdparties/www.malwaredomainlist.com $DES/thirdparti
mkdir $DES/ublock
cp -R ../uAssets/filters/* $DES/ublock/
-cp -R ./assets/ublock/filter-lists.json $DES/ublock/
-
-cp ../uAssets/checksums/ublock0.txt $DES/checksums.txt
+cp -R ./assets/assets.json $DES/
echo "done."
diff --git a/tools/make-chromium.sh b/tools/make-chromium.sh
index 8d1f5891..eda30f29 100755
--- a/tools/make-chromium.sh
+++ b/tools/make-chromium.sh
@@ -5,7 +5,11 @@
echo "*** uBlock0.chromium: Creating web store package"
echo "*** uBlock0.chromium: Copying files"
-DES=dist/build/uBlock0.chromium
+if [ "$1" = experimental ]; then
+ DES=dist/build/experimental/uBlock0.chromium
+else
+ DES=dist/build/uBlock0.chromium
+fi
rm -rf $DES
mkdir -p $DES
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-28 09:11:36 +0000