Merge fe16052056 into 6050c5f150

[ImgBot] Optimize images

README.md

@@ -154,6 +154,7 @@ A collection of delicious docker recipes.
 ## Utility
 
 - [x] registry-cli
+- [x] tldextract
 
 ## Media
 

browserless/README.md

@@ -88,11 +88,23 @@ $ pip install selenium
 $ python screenshot.py
 $ imgcat google.png
 
-$ http http://127.0.0.1:3000/screenshot \
+$ http :3000/screenshot \
        url=https://www.youtube.com \
        options[fullPage]:=true \
        gotoOptions[waitUntil]=networkidle2 > youtube.png
 $ imgcat youtube.png
+
+$ http :3000/scrape url=https://www.youtube.com elements[0][selector]=title debug[network]:=true |
+    jq -r '.debug.network.outbound[].url' |
+      xargs -r tldextract -j |
+        jq -r 'select(.fqdn|length>0).fqdn' |
+          sort -u
+accounts.google.com
+fonts.googleapis.com
+fonts.gstatic.com
+googleads.g.doubleclick.net
+i.ytimg.com
+www.youtube.com
 ```
 
 [1]: https://docs.browserless.io/

ftldns/Dockerfile

@@ -5,9 +5,10 @@
 FROM alpine:3
 MAINTAINER EasyPi Software Foundation
 
-ARG FTL_VERSION=v5.23
+ARG FTL_VERSION=v5.25.1
 ARG FTL_FILE=pihole-FTL-musl-linux-x86_64
-ARG FTL_URL=https://github.com/pi-hole/FTL/releases/download/${FTL_VERSION}/${FTL_FILE}
+ARG FTL_GIT=https://github.com/pi-hole
+ARG FTL_URL=${FTL_GIT}/FTL/releases/download/${FTL_VERSION}/${FTL_FILE}
 
 RUN set -xe \
  && apk add --no-cache curl \
@@ -15,6 +16,15 @@ RUN set -xe \
  && curl -sSL ${FTL_URL} -o /usr/bin/pihole-FTL \
  && chmod +x /usr/bin/pihole-FTL \
  && pihole-FTL --version \
+ && mkdir -p /opt/pihole \
+ && cd /opt/pihole \
+ && curl -sSL -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/list.sh \
+              -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/utils.sh \
+              -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/chronometer.sh \
+              -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/COL_TABLE \
+ && cd /usr/local/bin \
+ && curl -sSL ${FTL_GIT}/pi-hole/raw/master/pihole -o /usr/local/bin/pihole \
+ && chmod +x /usr/local/bin/pihole /opt/pihole/*.sh \
  && apk del curl
 
 VOLUME /etc/pihole

ftldns/Dockerfile.debian

@@ -0,0 +1,38 @@
+#
+# Dockerfile for FTLDNS (pihole-FTL)
+#
+
+FROM debian:12-slim
+MAINTAINER EasyPi Software Foundation
+
+ARG FTL_VERSION=v5.25.1
+ARG FTL_FILE=pihole-FTL-aarch64-linux-gnu
+ARG FTL_GIT=https://github.com/pi-hole
+ARG FTL_URL=${FTL_GIT}/FTL/releases/download/${FTL_VERSION}/${FTL_FILE}
+
+RUN set -xe \
+ && apt update -y \
+ && apt install -y curl procps \
+ && echo "conf-dir=/etc/pihole/dnsmasq.d,*.conf" >> /etc/dnsmasq.conf \
+ && curl -sSL ${FTL_URL} -o /usr/bin/pihole-FTL \
+ && chmod +x /usr/bin/pihole-FTL \
+ && pihole-FTL --version \
+ && mkdir -p /opt/pihole \
+ && cd /opt/pihole \
+ && curl -sSL -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/list.sh \
+              -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/utils.sh \
+              -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/chronometer.sh \
+              -O ${FTL_GIT}/pi-hole/raw/master/advanced/Scripts/COL_TABLE \
+ && cd /usr/local/bin \
+ && curl -sSL ${FTL_GIT}/pi-hole/raw/master/pihole -o /usr/local/bin/pihole \
+ && chmod +x /usr/local/bin/pihole /opt/pihole/*.sh \
+ && apt remove -y curl \
+ && rm -rf /var/lib/apt/lists/*
+
+VOLUME /etc/pihole
+
+EXPOSE 4711/tcp \
+       53/tcp \
+       53/udp
+
+CMD ["pihole-FTL", "-f"]

ftldns/README.md

@@ -10,14 +10,29 @@ $ docker compose up -d
 
 $ dig @127.0.0.1 -p 53 www.youtube.com
 
-$ docker compose exec ftldns pihole-FTL sql -h gravity.db
->>> .schema domainlist
->>> insert into domainlist(type, domain) values (3, '(\.|^)youtube\.com$');
->>> .quit
+$ docker compose exec ftldns bash
+>>> pihole-FTL sql -h gravity.db
+    .schema domainlist
+    insert into domainlist(type, domain) values (3, '(\.|^)youtube\.com$');
+    .quit
+>>> pihole --regex '(\.|^)baidu\.com$'
+>>> pihole --wild 'qq.com'
+>>> pihole-FTL regex-test www.baidu.com
+>>> pihole --wild -d youtube.com
+>>> pihole --regex -l
+Displaying regex blacklist:
+  1: (\.|^)baidu\.com$ (enabled, last modified Tue, 27 Feb 2024 11:17:59 +0000)
+  2: (\.|^)qq\.com$ (enabled, last modified Tue, 27 Feb 2024 11:22:17 +0000)
+>>> pihole -c
+|¯¯¯(¯)_|¯|_  ___|¯|___        Core: API Offline
+| ¯_/¯|_| ' \/ _ \ / -_)
+|_| |_| |_||_\___/_\___|
+ ——————————————————————————————————————————————————————————
+>>> exit
 
-$ docker compose exec ftldns kill -RTMIN 1
+$ docker compose kill -s RTMIN ftldns
 
-$ dig @127.0.0.1 -p 53 www.youtube.com
+$ dig @127.0.0.1 -p 53 www.baidu.com
 
 $ telnet 127.0.0.1 4711
 >version
@@ -28,7 +43,8 @@ $ telnet 127.0.0.1 4711
 >quit
 ```
 
-> Read more about [telnet-api][2].
+> Read more about [domainlist][2] and [telnet-api][3].
 
 [1]: https://github.com/pi-hole/FTL
-[2]: https://docs.pi-hole.net/ftldns/telnet-api/
+[2]: https://docs.pi-hole.net/database/gravity/#domain-tables-domainlist
+[3]: https://docs.pi-hole.net/ftldns/telnet-api/

memgraph/example/dnslookup

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+HOST=${1:?DOMAIN or IP is empty}
+
+docker exec -i memgraph_memgraph_1 mgconsole -output-format=csv <<- _CQL_ | sed -e 's/"//g' | tail -n +2 | tr '[],[]' ' ' | gawk -f /dev/fd/3 3<<- "_AWK_" | column -t -i1 -p2 -r3 -H1,2 | sed 's/─/& /'
+	MATCH p=(n)-[*]->(m)
+	WHERE any(n in nodes(p) where n.name = '$HOST') AND not exists(()-->(n)) AND not exists((m)-->())
+	UNWIND nodes(p) AS nn
+	WITH DISTINCT nn
+	CALL path.expand(nn,[">"],[],1,1) YIELD result
+	RETURN extract(i in nodes(result)|i.name);
+_CQL_
+	BEGIN {
+		split("", cache);
+		split("", roots);
+		idx=0;
+	}
+	{
+		if(!($1 in cache)) {
+			roots[$1] = cache[$1] = ++idx;
+		}
+		if(!($2 in cache)) {
+			cache[$2] = ++idx;
+		}
+		delete roots[$2];
+		print cache[$2], cache[$1], $2;
+	}
+	END {
+		print "0 -1 ."
+		for(root in roots) {
+			print cache[root], 0, root;
+		}
+	}
+_AWK_
+
+# vim: set noai noet:

memgraph/example/dump-elk.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+export PATH=/usr/local/bin:$PATH
+
+date=${1:-$(date -d yesterday +%F)}
+url=http://127.0.0.1:9200/
+index=logstash-${date//-/.}
+output=data/${date}.csv
+
+mkdir -p ${output%/*}
+
+elastic-query-export -c $url -i $index -o $output -q '+project:dns -_exists_:message +type:(A CNAME)' -fields '@timestamp,region,client,server,type,query,answer,ttl'
+
+gzip $output

memgraph/example/load-csv.cql

@@ -0,0 +1,27 @@
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH DISTINCT row.query AS query
+MERGE (d:Domain {name: query});
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH DISTINCT row.answer AS answer WHERE row.type = 'CNAME'
+MERGE (d:Domain {name: answer});
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH DISTINCT row.answer AS answer WHERE row.type = 'A'
+MERGE (i:IPv4 {name: answer});
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH row WHERE row.type = 'A'
+MATCH (d:Domain {name: row.query}), (i:IPv4 {name: row.answer})
+MERGE (d)-[r:A]->(i)
+ON CREATE SET r.created_at = timestamp()/1000000
+CALL date.parse(replace(row.`@timestamp`, "Z", ""), "s", "%Y-%m-%dT%H:%M:%S.%f", "UTC") YIELD parsed
+SET r.updated_at = parsed;
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH row WHERE row.type = 'CNAME'
+MATCH (d1:Domain {name: row.query}), (d2:Domain {name: row.answer})
+MERGE (d1)-[r:CNAME]->(d2)
+ON CREATE SET r.created_at = timestamp()/1000000
+CALL date.parse(replace(row.`@timestamp`, "Z", ""), "s", "%Y-%m-%dT%H:%M:%S.%f", "UTC") YIELD parsed
+SET r.updated_at = parsed;

memgraph/example/load-csv.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+FILE=$(date -d yesterday +%F).csv.gz
+SRC_FILE=/data/dns/data/$FILE
+DST_FILE=/path/to/dns.csv.gz
+
+echo "$(date +%FT%T) GET $FILE"
+scp elk-us:$SRC_FILE $DST_FILE
+
+echo "$(date +%FT%T) LOAD CSV"
+docker exec -i memgraph_memgraph_1 mgconsole < load-csv.cql
+
+echo "$(date +%FT%T) DONE"

outline-ss-server/data/keygen.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# pwgen 12 1
+
+NAME=${1:?name is empty}
+
+SERVER="127.0.0.1:9443"
+CIPHER="chacha20-ietf-poly1305"
+SECRET=$(yq -o json config.yml | jq -e -r --arg name "$NAME" '.keys[]|select(.id==$name).secret')
+DIGEST=$(echo -n "$CIPHER:$SECRET" | base64 | sed 's/=*$//')
+
+echo "ss://$DIGEST@$SERVER/?outline=1"

outline-ss-server/systemd/outline-ss-server.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=outline-ss-server
+After=network.target
+
+[Service]
+User=vpn
+Group=vpn
+WorkingDirectory=/etc/outline-ss-server
+ExecStart=/usr/local/bin/outline-ss-server -config=config.yml -metrics=127.0.0.1:9092 -ip_country_db=ip-country.mmdb -replay_history=10000
+ExecReload=/bin/kill -HUP $MAINPID
+LimitNOFILE=65536
+RestartSec=5
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

pihole/README.md

@@ -6,7 +6,7 @@ pihole
 The [Pi-hole®][1] is a DNS sinkhole that protects your devices from unwanted
 content, without installing any client-side software.
 
-- https://docs.pi-hole.net/
+- https://docs.pi-hole.net/core/pihole-command/
 - https://hub.docker.com/r/pihole/pihole/
 
 [1]: https://pi-hole.net/

tinyproxy/Dockerfile

@@ -10,6 +10,7 @@ RUN set -xe \
     && sed -i -e '/^Allow /s/^/#/' \
               -e '/^ConnectPort /s/^/#/' \
               -e '/^#DisableViaHeader /s/^#//' \
+              -e 's/^Timeout.*/Timeout 10/' \
               /etc/tinyproxy/tinyproxy.conf
 
 VOLUME /etc/tinyproxy

tldextract/Dockerfile

@@ -0,0 +1,13 @@
+#
+# Dockerfile for tldextract
+#
+
+FROM alpine:3
+
+RUN apk add --no-cache py3-pip \
+ && pip install tldextract \
+ && tldextract --update \
+ && tldextract --version
+
+ENTRYPOINT ["/usr/bin/tldextract"]
+CMD ["--help"]

tldextract/README.md

@@ -0,0 +1,17 @@
+tldextract
+==========
+
+[tldextract][1] accurately separates a URL's subdomain, domain, and public
+suffix, using the Public Suffix List (PSL).
+
+## Tutorial
+
+```bash
+# Create an alias
+$ alias tldextract='docker run --rm -u $(id -u):$(id -g) vimagick/tldextract'
+
+# Run it
+$ tldextract -j www.google.com
+```
+
+[1]: https://github.com/john-kurkowski/tldextract

urlwatch/data/reporters.py

@@ -0,0 +1,69 @@
+class SlackReporter(TextReporter):
+    """Send a message to a Slack channel"""
+
+    __kind__ = 'slack'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.max_length = self.config.get('max_message_length', 40000)
+
+    def submit(self):
+        webhook_url = self.config['webhook_url']
+        text = '\n'.join(super().submit())
+
+        if not text:
+            logger.debug('Not calling {} API (no changes)'.format(self.__kind__))
+            return
+
+        result = None
+        for chunk in chunkstring(text, self.max_length, numbering=True):
+            res = self.submit_chunk(webhook_url, chunk)
+            if res.status_code != requests.codes.ok or res is None:
+                result = res
+
+        return result
+
+    def submit_chunk(self, webhook_url, text):
+        logger.debug("Sending {} request with text: {}".format(self.__kind__, text))
+        post_data = self.prepare_post_data(text)
+        result = requests.post(webhook_url, json=post_data)
+        try:
+            if result.status_code == requests.codes.ok:
+                logger.info("{} response: ok".format(self.__kind__))
+            else:
+                logger.error("{} error: {}".format(self.__kind__, result.text))
+        except ValueError:
+            logger.error(
+                "Failed to parse {} response. HTTP status code: {}, content: {}".format(self.__kind__,
+                                                                                        result.status_code,
+                                                                                        result.content))
+        return result
+
+    def prepare_post_data(self, text):
+        if self.config.get('rich_text', False):
+            return {
+                "blocks": [
+                    {
+                        "type": "rich_text",
+                        "elements": [
+                            {
+                                "type": "rich_text_preformatted",
+                                "elements": [
+                                    {"type": "text", "text": text}
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            }
+        else:
+            return {"text": text}
+
+
+class MattermostReporter(SlackReporter):
+    """Send a message to a Mattermost channel"""
+
+    __kind__ = 'mattermost'
+
+    def prepare_post_data(self, text):
+        return {"text": text}

urlwatch/systemd/urlwatch.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=lightweight incoming webhook server
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/webhook -hooks=hooks.json -hooks=hooks.yaml -hotreload -port=9000 -template -urlprefix=hooks
+ExecReload=/bin/kill -HUP $MAINPID
+WorkingDirectory=/etc/webhook/
+LimitNOFILE=65536
+RestartSec=5
+Restart=always
+
+[Install]
+WantedBy=multi-user.target