diff --git a/urlwatch/Dockerfile b/urlwatch/Dockerfile index c93b643..c4b2a1b 100644 --- a/urlwatch/Dockerfile +++ b/urlwatch/Dockerfile @@ -5,29 +5,32 @@ FROM alpine MAINTAINER kev -RUN apk add -U ca-certificates \ - build-base \ - libxml2 \ - libxml2-dev \ - libxslt \ - libxslt-dev \ - make \ - python-dev \ - py-pip \ - && pip install cssselect \ - futures \ - keyring \ - lxml \ - urlwatch \ - pyyaml \ +RUN set -xe \ + && apk add --no-cache ca-certificates \ + build-base \ + libffi-dev \ + libxml2 \ + libxml2-dev \ + libxslt \ + libxslt-dev \ + openssl-dev \ + python3 \ + python3-dev \ + && python3 -m pip install appdirs \ + cssselect \ + keyring \ + lxml \ + minidb \ + pyyaml \ + requests \ && apk del build-base \ + libffi-dev \ libxml2-dev \ libxslt-dev \ - python-dev \ - && rm -rf /var/cache/apk/* \ - && echo '*/15 * * * * cd ~/.urlwatch && make' | crontab - + openssl-dev \ + python3-dev \ + && echo '*/30 * * * * urlwatch' | crontab - -COPY urlwatch /root/.urlwatch VOLUME /root/.urlwatch WORKDIR /root/.urlwatch diff --git a/urlwatch/README.md b/urlwatch/README.md index 53686a6..171c84e 100644 --- a/urlwatch/README.md +++ b/urlwatch/README.md @@ -3,82 +3,14 @@ urlwatch [urlwatch][1] is a tool for monitoring webpages for updates. -``` -cron: triggered every 15 minutes - -> make: generate urls.txt from urls.yml - -> urlwatch: fetch webpages - -> hooks.py: extract info - -> email: send via smtp - -> (^_^) -``` - ## docker-compose.yml -``` +```yaml urlwatch: image: vimagick/urlwatch volumes: - - urlwatch/Makefile:/root/.urlwatch/Makefile - - urlwatch/urls.yml:/root/.urlwatch/urls.yml + - ./data:/root/.urlwatch restart: always ``` -## Makefile - -``` -SHELL = /bin/sh -PATH := /usr/local/bin:$(PATH) - -SMTP = smtp.easypi.info:587 -FROM = urlwatch@easypi.info -PASS = password -TO = noreply@easypi.info - -all: setup urls.txt - urlwatch -s $(SMTP) -f $(FROM) -t $(TO) -A -T - -urls.txt: urls.yml - python lib/hooks.py - -setup: - python -c 'import keyring; keyring.set_password("$(SMTP)", "$(FROM)", "$(PASS)")' -``` - -> Please change `STMP`/`FROM`/`PASS`/`TO` to correct value. - -## urls.yml - -``` -python: - url: https://www.python.org/downloads/ - exp: //div[@class="download-unknown"]/p[@class]/a[1]/text() - -ubuntu: - url: http://www.ubuntu.com/download/server - exp: //div[contains(@class, "row-hero")]//h2/text() - -coreos: - url: https://coreos.com - exp: //div[@class="co-p-homepage-release-text"]/text() - -urlwatch: - url: https://github.com/thp/urlwatch/releases - exp: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text() -``` - -## Email alert - -``` -*************************************************************************** -CHANGED: https://coreos.com#coreos -*************************************************************************** ---- @ Tue, 07 Jul 2015 17:15:01 +0000 -+++ @ Tue, 07 Jul 2015 20:00:01 +0000 -@@ -1 +1 @@ --coreos: 723.1.0 -+coreos: 735.0.0 - -*************************************************************************** -``` - -[1]: thp.io/2008/urlwatch/ +[1]: https://thp.io/2008/urlwatch/ diff --git a/urlwatch/data/urls.yaml b/urlwatch/data/urls.yaml new file mode 100644 index 0000000..99ea5f5 --- /dev/null +++ b/urlwatch/data/urls.yaml @@ -0,0 +1,4 @@ +- url: "https://github.com/thp/urlwatch/releases/latest" + filter: + - xpath: '(//div[contains(@class,"release-timeline-tags")]//h4)[1]/a' + - html2text: re diff --git a/urlwatch/data/urlwatch.yaml b/urlwatch/data/urlwatch.yaml new file mode 100644 index 0000000..38152cc --- /dev/null +++ b/urlwatch/data/urlwatch.yaml @@ -0,0 +1,4 @@ +report: + slack: + webhook_url: https://hooks.slack.com/services/xxxxxx + enabled: true diff --git a/urlwatch/docker-compose.yml b/urlwatch/docker-compose.yml index 86758cd..d3db538 100644 --- a/urlwatch/docker-compose.yml +++ b/urlwatch/docker-compose.yml @@ -1,6 +1,5 @@ urlwatch: image: vimagick/urlwatch volumes: - - ./urlwatch/Makefile:/root/.urlwatch/Makefile - - ./urlwatch/urls.yml:/root/.urlwatch/urls.yml - restart: always + - ./data:/root/.urlwatch + restart: unless-stopped diff --git a/urlwatch/urlwatch/Makefile b/urlwatch/urlwatch/Makefile deleted file mode 100644 index 6f8c8b4..0000000 --- a/urlwatch/urlwatch/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -SHELL = /bin/sh -PATH := /usr/local/bin:$(PATH) - -SMTP = smtp.easypi.info:587 -FROM = urlwatch@easypi.info -PASS = password -TO = noreply@easypi.info - -all: setup urls.txt - urlwatch -s $(SMTP) -f $(FROM) -t $(TO) -A -T - -urls.txt: urls.yml - python lib/hooks.py - -setup: - python -c 'import keyring; keyring.set_password("$(SMTP)", "$(FROM)", "$(PASS)")' diff --git a/urlwatch/urlwatch/lib/hooks.py b/urlwatch/urlwatch/lib/hooks.py deleted file mode 100755 index df39190..0000000 --- a/urlwatch/urlwatch/lib/hooks.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# version detector for urlwatch -# - -import cssselect.xpath -import lxml.html -import os.path -import re -import yaml - - -urlwatch_dir = os.path.expanduser(os.path.join('~', '.urlwatch')) -urls_txt = os.path.join(urlwatch_dir, 'urls.txt') -urls_yml = os.path.join(urlwatch_dir, 'urls.yml') -config = yaml.load(open(urls_yml)) -tr = cssselect.xpath.HTMLTranslator() - - -def filter(url, data): - - try: - - for key, cfg in config.items(): - if url.endswith('#' + key): - if not url.startswith(cfg['url']): - raise Exception('url mismatch') - if 'xpath' in cfg: - xpath = cfg['xpath'] - elif 'css' in cfg: - xpath = tr.css_to_xpath(cfg['css']) - else: - raise Exception('xpath/css required') - break - else: - return data - - dom = lxml.html.fromstring(data.encode('utf-8')) - txt = dom.xpath(xpath)[0] - ver = re.search(r'(?i)([0-9]+\.)*[0-9]+(-?(alpha|beta|rc)[0-9]+)?', txt).group(0).lower() - return '{}: {}\n'.format(key, ver) - - except: - - return '{}: {}\n'.format(key, 'unknown') - - -if __name__ == '__main__': - - print 'Extract urls from <{}> ({})'.format(urls_yml, len(config)) - urls = ['{0[url]}#{1}\n'.format(cfg, key) for key, cfg in config.items()] - - print 'Write urls to <{}> ({})'.format(urls_txt, len(urls)) - with open(urls_txt, 'w') as f: - f.writelines(urls) - - print 'Everything is OK!' - diff --git a/urlwatch/urlwatch/urls.yml b/urlwatch/urlwatch/urls.yml deleted file mode 100644 index 2401dae..0000000 --- a/urlwatch/urlwatch/urls.yml +++ /dev/null @@ -1,145 +0,0 @@ -###### OS ###### - -alpine: - url: http://alpinelinux.org/ - xpath: //span[@class="icon-download"]/following-sibling::p/a/text() - -archlinux: - url: https://www.archlinux.org/download/ - xpath: //li[strong="Current Release:"]/text() - -centos: - url: http://www.centos.org/download/ - xpath: //div[@class="downloadbutton"]/a[.="DVD ISO"]/@href - -coreos: - url: https://coreos.com - xpath: //div[@class="co-p-homepage-release-text"]/text() - -debian: - url: https://www.debian.org/releases/ - xpath: //li[contains(., "current stable release")]/a/text() - -fedora: - url: https://getfedora.org/en/server/download/ - xpath: //h2[starts-with(., "Download")]/text() - -freebsd: - url: http://www.freebsd.org/releases/ - xpath: //div[@id="sidenav"]//li[contains(text(), "Production Release:")][1]/a/text() - -kali: - url: https://www.kali.org/kali-linux-releases/ - xpath: //h2[.="Kali Linux Release History"]/following-sibling::ul/li[1]/strong/text() - -raspbian: - url: https://www.raspberrypi.org/downloads/ - xpath: //div[h3="Raspbian"]//span[.="Kernel version:"]/following-sibling::strong/text() - -redhat: - url: https://access.redhat.com/products/red-hat-enterprise-linux/evaluation - xpath: //a[contains(@class, "btn-download")]/small/text() - -smartos: - url: https://github.com/joyent/smartos-live/releases - xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text() - -ubuntu: - url: http://www.ubuntu.com/download/server - xpath: //div[contains(@class, "row-hero")]//h2/text() - -###### LANG ###### - -go: - url: https://golang.org/dl/ - xpath: //h3[.="Stable versions"]/following-sibling::div[1]/@id - -nodejs: - url: https://nodejs.org/ - xpath: //div[@id="intro"]/p[@class="version"]/text() - -python: - url: https://www.python.org/downloads/ - xpath: //div[@class="download-unknown"]/p[@class]/a[1]/text() - -###### DB ###### - -mongodb: - url: https://www.mongodb.org/downloads - xpath: //div[@id="production-release-header"]/h2/@data-production-version - -redis: - url: http://redis.io - xpath: //a[contains(@href, "/releases/")]/text() - -influxdb: - url: https://influxdb.com/download/index.html - xpath: //h2[contains(., "Stable")]/text() - -###### TOOL ###### - -vagrant: - url: https://www.vagrantup.com/downloads.html - xpath: //h2[@class="os-name" and .="Mac OS X"]/following-sibling::ul/li/a/@href - -git: - url: http://git-scm.com - xpath: //span[@class="version"]/text() - -docker: - url: https://github.com/docker/docker/releases - xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text() - -phantomjs: - url: http://phantomjs.org - xpath: //span[@class="version"]/text() - -firefox: - url: https://www.mozilla.org/en-US/firefox/new/ - xpath: //li[@class="os_osx"]/a[@class="download-link"]/@href - -jquery: - url: http://jquery.com - xpath: //span[@class="download"]/following-sibling::span/text() - -vim: - url: http://en.wikipedia.org/wiki/Vim_%28text_editor%29 - xpath: //th[a="Stable release"]/following-sibling::td/text() - -scrapy: - url: https://github.com/scrapy/scrapy/releases - xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text() - -scrapyd: - url: https://github.com/scrapy/scrapyd/releases - xpath: //div[contains(@class, "label-latest")]//h1[@class="release-title"]/a/text() - -urlwatch: - url: https://github.com/thp/urlwatch/releases - xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text() - -monit: - url: http://mmonit.com/monit/changes/ - xpath: //div[@class="container"]/div[@class="row"][1]//h3/a/text() - -virtualbox: - url: https://www.virtualbox.org/wiki/Downloads - xpath: //strong/following-sibling::a[@class="ext-link" and contains(@href, "dmg")]/@href - -###### VPN ###### - -openvpn: - url: https://openvpn.net/index.php/open-source/downloads.html - xpath: //h2[contains(., "released on")]/text() - -tinc: - url: http://tinc-vpn.org/ - xpath: //h3/a[contains(@href, "download")]/text() - -shadowsocks: - url: https://github.com/shadowsocks/shadowsocks/releases - xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text() - -shadowsocks-libev: - url: https://github.com/shadowsocks/shadowsocks-libev/releases - xpath: //div[contains(@class, "label-latest")]//h1[@class="release-title"]/a/text()