From 710965eb1244af941c209da65504b09d9d5c0b80 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Sun, 30 Apr 2017 00:01:55 +0300 Subject: [PATCH] adblock: backport updates to 2.6.2 Backport updates in 2.5.0-2.6.2 from master. Signed-off-by: Hannu Nyman --- net/adblock/Makefile | 14 +-- net/adblock/files/README.md | 132 ++++++++---------------- net/adblock/files/adblock.conf | 23 ++++- net/adblock/files/adblock.init | 67 ++++++------- net/adblock/files/adblock.sh | 178 +++++++++++++++++++++++---------- 5 files changed, 220 insertions(+), 194 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 9c7f746b8f..15117902e9 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,28 +6,28 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=2.4.0 -PKG_RELEASE:=2 +PKG_VERSION:=2.6.2 +PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken include $(INCLUDE_DIR)/package.mk -define Package/$(PKG_NAME) +define Package/adblock SECTION:=net CATEGORY:=Network TITLE:=Powerful adblock script to block ad/abuse domains PKGARCH:=all endef -define Package/$(PKG_NAME)/description +define Package/adblock/description Powerful adblock script to block ad/abuse domains via dnsmasq or unbound dns backend. The script supports many domain blacklist sites plus manual black- and whitelist overrides. Please see https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md for further information. endef -define Package/$(PKG_NAME)/conffiles +define Package/adblock/conffiles /etc/config/adblock /etc/adblock/adblock.whitelist /etc/adblock/adblock.blacklist @@ -42,7 +42,7 @@ endef define Build/Compile endef -define Package/$(PKG_NAME)/install +define Package/adblock/install $(INSTALL_DIR) $(1)/usr/bin $(INSTALL_BIN) ./files/adblock.sh $(1)/usr/bin/ @@ -57,4 +57,4 @@ define Package/$(PKG_NAME)/install $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock/ endef -$(eval $(call BuildPackage,$(PKG_NAME))) +$(eval $(call BuildPackage,adblock)) diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 172551f5cd..6950ec792b 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -29,10 +29,14 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 15 entries * [ransomware tracker](https://ransomwaretracker.abuse.ch) * => daily updates, approx. 150 entries - * [rolist/easylist](https://easylist-downloads.adblockplus.org/rolist+easylist.txt) - * => weekly updates, approx. 600 entries - * [ruadlist/easylist](https://code.google.com/p/ruadlist) - * => weekly updates, approx. 2.000 entries + * [reg_cn](https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt) + * => regional blocklist for China, daily updates, approx. 1.600 entries + * [reg_pl](http://adblocklist.org) + * => regional blocklist for Poland, daily updates, approx. 50 entries + * [reg_ro](https://easylist-downloads.adblockplus.org/rolist+easylist.txt) + * => regional blocklist for Romania, weekly updates, approx. 600 entries + * [reg_ru](https://code.google.com/p/ruadlist) + * => regional blocklist for Russia, weekly updates, approx. 2.000 entries * [securemecca](http://www.securemecca.com) * => infrequent updates, approx. 25.000 entries * [shallalist](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default) @@ -55,6 +59,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * simple but yet powerful adblock engine: adblock does not use error prone external iptables rulesets, http pixel server instances and things like that * automatically selects dnsmasq or unbound as dns backend * automatically selects uclient-fetch or wget as download utility (other tools like curl or aria2c are supported as well) +* support http only mode (without installed ssl library) for all non-SSL blocklist sources * automatically supports a wide range of router modes, even AP modes are supported * full IPv4 and IPv6 support * supports tld compression (top level domain compression), this feature removes thousands of needless host entries from the block lists and lowers the memory footprint for the dns backends @@ -64,53 +69,44 @@ A lot of people already use adblocker plugins within their desktop browsers, but * additional whitelist for manual overrides, located by default in /etc/adblock/adblock.whitelist * quality checks during block list update to ensure a reliable dns backend service * minimal status & error logging to syslog, enable debug logging to receive more output -* procd based init system support (start/stop/restart/reload/suspend/resume) -* procd based hotplug support, the adblock start will be triggered by interface triggers +* procd based init system support (start/stop/restart/reload/suspend/resume/query/status) +* procd based hotplug support, the adblock start will be solely triggered by network interface triggers * suspend & resume adblock actions temporarily without block list reloading -* runtime statistics via ubus service call +* runtime information available via LuCI & via 'status' init command * query function to quickly identify blocked (sub-)domains, e.g. for whitelisting +* optional: force dns requests to local resolver +* optional: force overall sort / duplicate removal for low memory devices (handle with care!) * optional: automatic block list backup & restore, backups will be (de-)compressed and restored on the fly in case of any runtime error * optional: add new adblock sources on your own via uci config ## Prerequisites -* [openwrt](https://openwrt.org), tested with latest stable release (Chaos Calmer) and with current trunk (Designated Driver) -* [LEDE project](https://www.lede-project.org), tested with trunk > r98 +* [LEDE project](https://www.lede-project.org), tested with latest stable release (LEDE 17.01) and with current LEDE snapshot * a usual setup with an enabled dns backend at minimum - dump AP modes without a working dns backend are _not_ supported -* a download utility: full versions (with ssl support) of 'wget', 'uclient-fetch', 'aria2c' or 'curl' are supported - the Chaos Calmer built-in busybox wget is not - * Chaos Calmer: download & install the external 'wget' package - * Designated Driver/Trunk: use built-in 'uclient-fetch' or download & install the external 'wget' package +* a download utility: + * to support all blocklist sources a full version (with ssl support) of 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required + * for limited devices with real memory constraints, adblock provides also a plain http option and supports wget-nossl and uclient-fetch (without libustream-ssl), too * for more configuration options see examples below -## OpenWrt / LEDE trunk Installation & Usage -* install 'adblock' (_opkg install adblock_) and that's it - the adblock start will be automatically triggered by procd interface triggers -* start/stop/restart/reload/suspend/resume the adblock service manually with _/etc/init.d/adblock_ +## LEDE trunk Installation & Usage +* install 'adblock' (_opkg install adblock_) and that's it - the adblock start will be automatically triggered by procd interface trigger +* control the adblock service manually with _/etc/init.d/adblock_ start/stop/restart/reload/suspend/resume/status or use the LuCI frontend * enable/disable your favored block list sources in _/etc/config/adblock_ - 'adaway', 'disconnect' and 'yoyo' are enabled by default ## LuCI adblock companion package -* for easy management of the various block list sources and options you can also use a nice & efficient LuCI frontend +* for easy management of the various block list sources and all other adblock options you can also use a nice & efficient LuCI frontend * install 'luci-app-adblock' (_opkg install luci-app-adblock_) * the application is located in LuCI under 'Services' menu -* _Thanks to Hannu Nyman for this great adblock LuCI frontend!_ - -## Chaos Calmer installation notes -* 'adblock' and 'luci-app-adblock' are _not_ available as .ipk packages in the Chaos Calmer download repository -* download both packages from a development snapshot package directory: - * for 'adblock' look [here](https://downloads.lede-project.org/snapshots/packages/x86_64/packages/) - * for 'luci-app-adblock' look [here](https://downloads.lede-project.org/snapshots/packages/x86_64/luci/) -* manually transfer the packages to your routers temp directory (with tools like _sshfs_ or _winscp_) -* install the packages with _opkg install <...>_ as described above ## Tweaks -* **status/runtime statistics:** the adblock status and runtime statistics are available via ubus service call (see example below) +* **runtime information:** the adblock status is available via _/etc/init.d/adblock status_ (see example below) * **debug logging:** for script debugging please set the config option 'adb\_debug' to '1' and check the runtime output with _logread -e "adblock"_ * **storage expansion:** to process and store all block list sources at once it might helpful to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details -* **add white-/blacklist entries:** add domain white- or blacklist entries to always-allow or -deny certain (sub) domains, by default both lists are empty and located in _/etc/adblock_. Please add one domain per line - ip addresses, wildcards & regex are _not_ allowed (see example below) +* **add white- / blacklist entries:** add domain white- or blacklist entries to always-allow or -deny certain (sub) domains, by default both lists are empty and located in _/etc/adblock_. Please add one domain per line - ip addresses, wildcards & regex are _not_ allowed (see example below) * **backup & restore block lists:** enable this feature, to restore automatically the latest compressed backup of your block lists in case of any processing error (e.g. a single block list source is not available during update). Please use an (external) solid partition and _not_ your volatile router temp directory for this * **scheduled list updates:** for a scheduled call of the adblock service add an appropriate crontab entry (see example below) -* **restrict/disable procd interface trigger:** to restrict the procd interface trigger to a (list of) certain wan interface(s) or to disable it at all, set 'adb\_iface' to an existing interface like 'wan' or to a non-existing like 'false' +* **restrict procd interface trigger:** restrict the procd interface trigger to a (list of) certain interface(s) (default: wan). To disable it at all, remove all entries * **suspend & resume adblocking:** to quickly switch the adblock service 'on' or 'off', simply use _/etc/init.d/adblock [suspend|resume]_ * **domain query:** to query the active block lists for a specific domain, please run _/etc/init.d/adblock query ``_ (see example below) -* **divert dns requests:** to force dns requests to your local dns resolver add an appropriate firewall rule (see example below) * **add new list sources:** you could add new block list sources on your own via uci config, all you need is a source url and an awk one-liner (see example below) * **disable active dns probing in windows 10:** to prevent a yellow exclamation mark on your internet connection icon (which wrongly means connected, but no internet), please change the following registry key/value from "1" to "0" _HKLM\SYSTEM\CurrentControlSet\Services\NlaSvc\Parameters\Internet\EnableActiveProbing_ @@ -118,13 +114,14 @@ A lot of people already use adblocker plugins within their desktop browsers, but * usually the pre-configured adblock setup works quite well and no manual config overrides are needed, all listed options apply to the 'global' config section: * adb\_enabled => main switch to enable/disable adblock service (default: '1', enabled) * adb\_debug => enable/disable adblock debug output (default: '0', disabled) - * adb\_iface => restrict the procd interface trigger to a (list of) certain wan interface(s) or disable it at all (default: not set, disabled) + * adb\_iface => set the procd interface trigger to a (list of) lan / wan interface(s) (default: 'wan') * adb\_fetch => full path to a different download utility, see example below (default: not set, use wget) * adb\_fetchparm => options for the download utility, see example below (default: not set, use wget options) - * adb\_tldcomp => enable/disable tld compression (default: '1', enabled) + * adb\_triggerdelay => additional trigger delay in seconds before adblock processing starts (default: '2') + * adb\_forcedns => force dns requests to local resolver (default: '0', disabled) + * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (default: '0', disabled) ## Examples - **change default dns backend to 'unbound':** 

 Adblock detects the presence of an active unbound dns backend and the block lists will be automatically pulled in by unbound.
@@ -153,55 +150,17 @@ curl:
   option adb_fetchparm '-s --connect-timeout 10 --insecure -o'
-**receive adblock statistics via ubus:** +**receive adblock runtime information:** 

-ubus call service list '{"name":"adblock_stats"}'
-This will output the active block lists and other runtime information as JSON, e.g.:
-{
-    "adblock_stats": {
-        "instances": {
-            "statistics": {
-                "running": false,
-                "command": [
-                    ""
-                ],
-                "data": {
-                    "active_lists": [
-                        {
-                            "palevo": "14",
-                            "blacklist": "144",
-                            "winspy": "168",
-                            "zeus": "422",
-                            "adaway": "408",
-                            "rolist": "649",
-                            "malwarelist": "1219",
-                            "ransomware": "1495",
-                            "ruadlist": "1791",
-                            "yoyo": "2304",
-                            "openphish": "2139",
-                            "dshield": "154",
-                            "disconnect": "3176",
-                            "spam404": "6251",
-                            "adguard": "11081",
-                            "whocares": "11575",
-                            "winhelp": "10574",
-                            "malware": "13854",
-                            "sysctl": "8539",
-                            "securemecca": "9262",
-                            "shalla": "25358",
-                            "hphosts": "36256"
-                        }
-                    ],
-                    "adblock_version": "2.3.0",
-                    "blocked_domains": "146833",
-                    "dns_backend": "dnsmasq",
-                    "last_rundate": "04.02.2017 21:10:31",
-                    "system": "LEDE Reboot SNAPSHOT r3286-c980147527"
-                }
-            }
-        }
-    }
-}
+root@blackhole:~# /etc/init.d/adblock status
+::: adblock runtime information
+ status          : active
+ adblock_version : 2.6.0
+ blocked_domains : 113711
+ fetch_info      : wget (built-in)
+ dns_backend     : dnsmasq
+ last_rundate    : 12.04.2017 13:08:26
+ system          : LEDE Reboot SNAPSHOT r3900-399d5cf532
**cronjob for a regular block list update (/etc/crontabs/root):** @@ -254,17 +213,6 @@ For every domain it returns the overall count plus a distinct list of active blo In the example above whitelist "www.doubleclick.net" to free the submitted domain. -**divert dns requests to local dns resolver in /etc/config/firewall:** -

-config redirect
-    option name 'Divert DNS'
-    option src 'lan'
-    option proto 'tcp udp'
-    option src_dport '53'
-    option dest_port '53'
-    option target 'DNAT'
-
- **add a new block list source:** 

 1. the easy way ...
@@ -295,7 +243,7 @@ If your awk one-liner works quite well, add a new source section in adblock conf
## Support -Please join the adblock discussion in this [forum thread](https://forum.openwrt.org/viewtopic.php?id=59803) or contact me by mail +Please join the adblock discussion in this [forum thread](https://forum.lede-project.org/t/adblock-2-x-support-thread/507) or contact me by mail ## Removal * stop all adblock related services with _/etc/init.d/adblock stop_ diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 999a2c47a8..0e4fd02577 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -4,10 +4,15 @@ config adblock 'global' option adb_enabled '1' option adb_debug '0' + option adb_forcesrt '0' + option adb_forcedns '0' + option adb_iface 'wan' + option adb_triggerdelay '2' option adb_whitelist '/etc/adblock/adblock.whitelist' option adb_whitelist_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}' option adb_backup '0' option adb_backupdir '/mnt' + option adb_rtfile '/tmp/adb_runtime.json' config source 'adaway' option enabled '1' @@ -35,7 +40,7 @@ config source 'disconnect' config source 'dshield' option enabled '0' - option adb_src 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt' + option adb_src 'https://www.dshield.org/feeds/suspiciousdomains_Low.txt' option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' option adb_src_desc 'generic blocklist, daily updates, approx. 4.500 entries' @@ -81,13 +86,25 @@ config source 'ransomware' option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 130 entries' -config source 'rolist' +config source 'reg_cn' + option enabled '0' + option adb_src 'https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt' + option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' + option adb_src_desc 'focus on chinese ads, daily updates, approx. 1.600 entries' + +config source 'reg_pl' + option enabled '0' + option adb_src 'http://adblocklist.org/adblock-pxf-polish.txt' + option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' + option adb_src_desc 'focus on polish ads, daily updates, approx. 50 entries' + +config source 'reg_ro' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/rolist+easylist.txt' option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' option adb_src_desc 'focus on romanian ads plus generic easylist additions, weekly updates, approx. 600 entries' -config source 'ruadlist' +config source 'reg_ru' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt' option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index cf8a57b151..ee94aa8228 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -1,41 +1,33 @@ #!/bin/sh /etc/rc.common # -START=90 +START=50 USE_PROCD=1 -EXTRA_COMMANDS="suspend resume query" +EXTRA_COMMANDS="suspend resume query status" EXTRA_HELP=" suspend Suspend adblock processing resume Resume adblock processing - query Query active blocklists for specific domain" + query Query active blocklists for specific domains + status Print runtime information" +adb_init="/etc/init.d/adblock" adb_script="/usr/bin/adblock.sh" boot() { - local wanif4 wanif6 cnt=0 max_cnt=30 - - . "/lib/functions/network.sh" - while [ ${cnt} -le ${max_cnt} ] - do - network_find_wan wanif4 - network_find_wan6 wanif6 - if [ -n "${wanif4}" ] || [ -n "${wanif6}" ] || [ ${cnt} -eq ${max_cnt} ] - then - rc_procd start_service - return 0 - else - sleep 1 - cnt=$((cnt+1)) - network_flush_cache - fi - done + adb_boot=1 + ubus -t 30 wait_for network.interface 2>/dev/null + rc_procd start_service } start_service() { - if [ $(/etc/init.d/adblock enabled; printf ${?}) -eq 0 ] + if [ $("${adb_init}" enabled; printf ${?}) -eq 0 ] then + if [ -n "${adb_boot}" ] + then + return 0 + fi procd_open_instance "adblock" procd_set_param command "${adb_script}" "${@}" procd_set_param stdout 1 @@ -44,14 +36,10 @@ start_service() fi } -reload_service() -{ - rc_procd start_service reload -} - stop_service() { rc_procd "${adb_script}" stop + rc_procd start_service } restart() @@ -61,12 +49,12 @@ restart() suspend() { - rc_procd start_service suspend + rc_procd "${adb_script}" suspend } resume() { - rc_procd start_service resume + rc_procd "${adb_script}" resume } query() @@ -74,18 +62,21 @@ query() rc_procd "${adb_script}" query "${1}" } +status() +{ + rc_procd "${adb_script}" status +} + service_triggers() { local iface="$(uci -q get adblock.global.adb_iface)" + local delay="$(uci -q get adblock.global.adb_triggerdelay)" - if [ -z "${iface}" ] - then - procd_add_raw_trigger "interface.*.up" 1000 /etc/init.d/adblock start - else - for name in ${iface} - do - procd_add_interface_trigger "interface.*.up" "${name}" /etc/init.d/adblock start - done - fi - procd_add_config_trigger "config.change" "adblock" /etc/init.d/adblock start + PROCD_RELOAD_DELAY=$((${delay:=2} * 1000)) + for name in ${iface} + do + procd_add_interface_trigger "interface.*.up" "${name}" "${adb_init}" start + done + PROCD_RELOAD_DELAY=1000 + procd_add_config_trigger "config.change" "adblock" "${adb_init}" start } diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index a470d0bf7e..03f86b03e4 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,11 +10,13 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="2.4.0-2" +adb_ver="2.6.2" +adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" adb_enabled=1 adb_debug=0 +adb_forcesrt=0 +adb_forcedns=0 adb_backup=0 -adb_tldcomp=1 adb_backupdir="/mnt" adb_whitelist="/etc/adblock/adblock.whitelist" adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}" @@ -22,6 +24,7 @@ adb_fetch="/usr/bin/wget" adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O" adb_dnslist="dnsmasq unbound" adb_dnsprefix="adb_list" +adb_rtfile="/tmp/adb_runtime.json" # f_envload: load adblock environment # @@ -31,16 +34,17 @@ f_envload() # source in system library # - if [ -r "/lib/functions.sh" ] + if [ -r "/lib/functions.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ] then . "/lib/functions.sh" + . "/usr/share/libubox/jshn.sh" else - f_log "error" "system library not found" + f_log "error" "system libraries not found" fi # set dns backend environment # - while [ ${cnt} -le 10 ] + while [ ${cnt} -le 20 ] do for dns in ${adb_dnslist} do @@ -111,17 +115,43 @@ f_envload() # config_load adblock config_foreach parse_config source + + # force dns to local resolver + # + if [ ${adb_forcedns} -eq 1 ] && [ -z "$(uci -q get firewall.adblock_dns)" ] + then + uci -q set firewall.adblock_dns="redirect" + uci -q set firewall.adblock_dns.name="Adblock DNS" + uci -q set firewall.adblock_dns.src="lan" + uci -q set firewall.adblock_dns.proto="tcp udp" + uci -q set firewall.adblock_dns.src_dport="53" + uci -q set firewall.adblock_dns.dest_port="53" + uci -q set firewall.adblock_dns.target="DNAT" + elif [ ${adb_forcedns} -eq 0 ] && [ -n "$(uci -q get firewall.adblock_dns)" ] + then + uci -q delete firewall.adblock_dns + fi + if [ -n "$(uci -q changes firewall)" ] + then + uci -q commit firewall + if [ $(/etc/init.d/firewall enabled; printf ${?}) -eq 0 ] + then + /etc/init.d/firewall reload >/dev/null 2>&1 + fi + fi } # f_envcheck: check/set environment prerequisites # f_envcheck() { + local ssl_lib + # check 'enabled' option # if [ ${adb_enabled} -ne 1 ] then - if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ] + if [ -n "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)" ] then f_rmdns f_dnsrestart @@ -132,15 +162,36 @@ f_envcheck() # check fetch utility # + ssl_lib="-" + if [ -x "${adb_fetch}" ] + then + if [ "$(readlink -fn "${adb_fetch}")" = "/usr/bin/wget-nossl" ] + then + adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 -O" + elif [ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] + then + adb_fetch="/bin/busybox" + adb_fetchparm="-q -O" + else + ssl_lib="built-in" + fi + fi if [ ! -x "${adb_fetch}" ] && [ "$(readlink -fn "/bin/wget")" = "/bin/uclient-fetch" ] then adb_fetch="/bin/uclient-fetch" - adb_fetchparm="-q --timeout=10 --no-check-certificate -O" + if [ -f "/lib/libustream-ssl.so" ] + then + adb_fetchparm="-q --timeout=10 --no-check-certificate -O" + ssl_lib="libustream-ssl" + else + adb_fetchparm="-q --timeout=10 -O" + fi fi - if [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ] || [ ! -x "${adb_fetch}" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] + if [ ! -x "${adb_fetch}" ] || [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ] then - f_log "error" "required download utility with ssl support not found, e.g. install full 'wget' package" + f_log "error" "no download utility found, please install 'uclient-fetch' with 'libustream-mbedtls' or the full 'wget' package" fi + adb_fetchinfo="${adb_fetch##*/} (${ssl_lib})" # create dns hideout directory # @@ -187,8 +238,8 @@ f_rmdns() rm -f "${adb_dnsdir}/${adb_dnsprefix}"* rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz rm -rf "${adb_dnshidedir}" + > "${adb_rtfile}" fi - ubus call service delete "{\"name\":\"adblock_stats\",\"instances\":\"statistics\"}" 2>/dev/null } # f_dnsrestart: restart the dns backend @@ -203,7 +254,7 @@ f_dnsrestart() adb_dnsup="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@.${adb_dns}.instances.*.running")" if [ "${adb_dnsup}" = "true" ] then - return 0 + break fi cnt=$((cnt+1)) sleep 1 @@ -289,7 +340,7 @@ f_query() if [ -z "${dns_active}" ] then - printf "%s\n" "::: no active block lists found, please start adblock first" + printf "%s\n" "::: no active block lists found, please start / resume adblock first" elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then printf "%s\n" "::: invalid domain input, please submit a specific (sub-)domain, e.g. 'www.abc.xyz'" @@ -300,18 +351,44 @@ f_query() search="${domain//./\.}" result="$(grep -Hm1 "[/\"\.]${search}[/\"]" "${adb_dnsprefix}"* | awk -F ':|=|/|\"' '{printf(" %-20s : %s\n",$1,$4)}')" printf "%s\n" "::: distinct results for domain '${domain}'" - if [ -z "${result}" ] - then - printf "%s\n" " no match" - else - printf "%s\n" "${result}" - fi + printf "%s\n" "${result:=" no match"}" domain="${tld}" tld="${domain#*.}" done fi } +# f_status: output runtime information +# +f_status() +{ + local key keylist value + + if [ -s "${adb_rtfile}" ] + then + local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" + local dns_passive="$(find "${adb_dnshidedir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" + + if [ -n "${dns_active}" ] + then + value="active" + elif [ -n "${dns_passive}" ] || [ -z "${dns_active}" ] + then + value="no domains blocked" + fi + printf "%s\n" "::: adblock runtime information" + printf " %-15s : %s\n" "status" "${value}" + json_load "$(cat "${adb_rtfile}" 2>/dev/null)" + json_select data + json_get_keys keylist + for key in ${keylist} + do + json_get_var value ${key} + printf " %-15s : %s\n" "${key}" "${value}" + done + fi +} + # f_log: write to syslog, exit on error # f_log() @@ -323,9 +400,9 @@ f_log() logger -t "adblock-[${adb_ver}] ${class}" "${log_msg}" if [ "${class}" = "error" ] then - logger -t "adblock-[${adb_ver}] ${class}" "Please check the online documentation 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md'" + logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})" f_rmtemp - if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ] + if [ -n "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)" ] then f_rmdns f_dnsrestart @@ -340,11 +417,11 @@ f_log() f_main() { local enabled url cnt sum_cnt=0 mem_total=0 - local src_name src_rset shalla_archive list active_lists - local sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" + local src_name src_rset shalla_archive mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)" f_log "info " "start adblock processing ..." + > "${adb_rtfile}" for src_name in ${adb_sources} do eval "enabled=\"\${enabled_${src_name}}\"" @@ -365,7 +442,7 @@ f_main() # download block list # - f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}, memory: ${mem_total}" + f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, memory: ${mem_total}, force srt/dns: ${adb_forcesrt}/${adb_forcedns}" if [ "${src_name}" = "blacklist" ] then cat "${url}" 2>/dev/null > "${adb_tmpload}" @@ -398,17 +475,12 @@ f_main() # if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ] then - awk "${src_rset}" "${adb_tmpload}" > "${adb_tmpfile}" + awk "${src_rset}" "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" if [ -s "${adb_tmpfile}" ] then - if [ ${adb_tldcomp} -eq 1 ] - then - awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" | sort -u > "${adb_tmpload}" - awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" > "${adb_tmpfile}" - awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" > "${adb_tmpload}" - else - sort -u "${adb_tmpfile}" > "${adb_tmpload}" - fi + awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null | sort -u > "${adb_tmpload}" + awk '{if(NR==1){tld=$NF};while(getline){if($NF !~ tld"\\."){print tld;tld=$NF}}print tld}' "${adb_tmpload}" 2>/dev/null > "${adb_tmpfile}" + awk -F "." '{for(f=NF;f > 1;f--) printf "%s.", $f;print $1}' "${adb_tmpfile}" 2>/dev/null > "${adb_tmpload}" mv -f "${adb_tmpload}" "${adb_tmpfile}" f_list backup else @@ -424,9 +496,9 @@ f_main() then if [ -s "${adb_tmpdir}/tmp.whitelist" ] then - grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}" + grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" > "${adb_dnsfile}" else - cat "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}" + cat "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" > "${adb_dnsfile}" fi adb_rc=${?} if [ ${adb_rc} -ne 0 ] @@ -442,7 +514,7 @@ f_main() # for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null) do - if [ ${mem_total} -ge 64000 ] + if [ ${mem_total} -ge 64000 ] || [ ${adb_forcesrt} -eq 1 ] then if [ -s "${adb_tmpdir}/blocklist.overall" ] then @@ -453,16 +525,9 @@ f_main() fi cnt="$(wc -l < "${src_name}")" sum_cnt=$((sum_cnt + cnt)) - list="${src_name/*./}" - if [ -z "${active_lists}" ] - then - active_lists="\"${list}\":\"${cnt}\"" - else - active_lists="${active_lists},\"${list}\":\"${cnt}\"" - fi done - # restart the dns backend and write statistics to procd service instance + # restart the dns backend and export runtime information # mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null @@ -470,18 +535,20 @@ f_main() f_dnsrestart if [ "${adb_dnsup}" = "true" ] then - f_log "info " "block lists with overall ${sum_cnt} domains loaded successfully (${sysver})" - ubus call service set "{\"name\":\"adblock_stats\", - \"instances\":{\"statistics\":{\"command\":[\"\"], - \"data\":{\"active_lists\":[{${active_lists}}], - \"adblock_version\":\"${adb_ver}\", - \"blocked_domains\":\"${sum_cnt}\", - \"dns_backend\":\"${adb_dns}\", - \"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\", - \"system\":\"${sysver}\"}}}}" - return 0 + json_init + json_add_object "data" + json_add_string "adblock_version" "${adb_ver}" + json_add_string "blocked_domains" "${sum_cnt}" + json_add_string "fetch_info" "${adb_fetchinfo}" + json_add_string "dns_backend" "${adb_dns}" + json_add_string "last_rundate" "$(/bin/date "+%d.%m.%Y %H:%M:%S")" + json_add_string "system" "${adb_sysver}" + json_close_object + json_dump > "${adb_rtfile}" + f_log "info " "block lists with overall ${sum_cnt} domains loaded successfully (${adb_sysver})" + else + f_log "error" "dns backend restart with active block lists failed" fi - f_log "error" "dns backend restart with active block lists failed (${sysver})" } # handle different adblock actions @@ -508,6 +575,9 @@ case "${1}" in query) f_query "${2}" ;; + status) + f_status + ;; *) f_envcheck f_main