moony
/
dockerfiles
mirror of https://github.com/vimagick/dockerfiles
1
2
Fork 0
Code Issues Releases Wiki Activity

update urlwatch

This commit is contained in:
kev 2019-05-02 19:34:00 +08:00
parent fb57740c81
commit 01bc6ecd52
8 changed files with 35 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
urlwatch/Dockerfile
View File

@ -5,29 +5,32 @@
FROM alpine
MAINTAINER kev <noreply@easypi.pro>
RUN apk add -U ca-certificates \
build-base \
libxml2 \
libxml2-dev \
libxslt \
libxslt-dev \
make \
python-dev \
py-pip \
&& pip install cssselect \
futures \
keyring \
lxml \
urlwatch \
pyyaml \
RUN set -xe \
&& apk add --no-cache ca-certificates \
build-base \
libffi-dev \
libxml2 \
libxml2-dev \
libxslt \
libxslt-dev \
openssl-dev \
python3 \
python3-dev \
&& python3 -m pip install appdirs \
cssselect \
keyring \
lxml \
minidb \
pyyaml \
requests \
&& apk del build-base \
libffi-dev \
libxml2-dev \
libxslt-dev \
python-dev \
&& rm -rf /var/cache/apk/* \
&& echo '*/15 * * * * cd ~/.urlwatch && make' | crontab -
openssl-dev \
python3-dev \
&& echo '*/30 * * * * urlwatch' | crontab -
COPY urlwatch /root/.urlwatch
VOLUME /root/.urlwatch
WORKDIR /root/.urlwatch

74
urlwatch/README.md
View File

@ -3,82 +3,14 @@ urlwatch
[urlwatch][1] is a tool for monitoring webpages for updates.
```
cron: triggered every 15 minutes
-> make: generate urls.txt from urls.yml
-> urlwatch: fetch webpages
-> hooks.py: extract info
-> email: send via smtp
-> (^_^)
```
## docker-compose.yml
```
```yaml
urlwatch:
image: vimagick/urlwatch
volumes:
- urlwatch/Makefile:/root/.urlwatch/Makefile
- urlwatch/urls.yml:/root/.urlwatch/urls.yml
- ./data:/root/.urlwatch
restart: always
```
## Makefile
```
SHELL = /bin/sh
PATH := /usr/local/bin:$(PATH)
SMTP = smtp.easypi.info:587
FROM = urlwatch@easypi.info
PASS = password
TO = noreply@easypi.info
all: setup urls.txt
urlwatch -s $(SMTP) -f $(FROM) -t $(TO) -A -T
urls.txt: urls.yml
python lib/hooks.py
setup:
python -c 'import keyring; keyring.set_password("$(SMTP)", "$(FROM)", "$(PASS)")'
```
> Please change `STMP`/`FROM`/`PASS`/`TO` to correct value.
## urls.yml
```
python:
url: https://www.python.org/downloads/
exp: //div[@class="download-unknown"]/p[@class]/a[1]/text()
ubuntu:
url: http://www.ubuntu.com/download/server
exp: //div[contains(@class, "row-hero")]//h2/text()
coreos:
url: https://coreos.com
exp: //div[@class="co-p-homepage-release-text"]/text()
urlwatch:
url: https://github.com/thp/urlwatch/releases
exp: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
```
## Email alert
```
***************************************************************************
CHANGED: https://coreos.com#coreos
***************************************************************************
--- @ Tue, 07 Jul 2015 17:15:01 +0000
+++ @ Tue, 07 Jul 2015 20:00:01 +0000
@@ -1 +1 @@
-coreos: 723.1.0
+coreos: 735.0.0
***************************************************************************
```
[1]: thp.io/2008/urlwatch/
[1]: https://thp.io/2008/urlwatch/

4
urlwatch/data/urls.yaml Normal file
View File

@ -0,0 +1,4 @@
- url: "https://github.com/thp/urlwatch/releases/latest"
filter:
- xpath: '(//div[contains(@class,"release-timeline-tags")]//h4)[1]/a'
- html2text: re

4
urlwatch/data/urlwatch.yaml Normal file
View File

@ -0,0 +1,4 @@
report:
slack:
webhook_url: https://hooks.slack.com/services/xxxxxx
enabled: true

5
urlwatch/docker-compose.yml
View File

@ -1,6 +1,5 @@
urlwatch:
image: vimagick/urlwatch
volumes:
- ./urlwatch/Makefile:/root/.urlwatch/Makefile
- ./urlwatch/urls.yml:/root/.urlwatch/urls.yml
restart: always
- ./data:/root/.urlwatch
restart: unless-stopped

16
urlwatch/urlwatch/Makefile
View File

@ -1,16 +0,0 @@
SHELL = /bin/sh
PATH := /usr/local/bin:$(PATH)
SMTP = smtp.easypi.info:587
FROM = urlwatch@easypi.info
PASS = password
TO = noreply@easypi.info
all: setup urls.txt
urlwatch -s $(SMTP) -f $(FROM) -t $(TO) -A -T
urls.txt: urls.yml
python lib/hooks.py
setup:
python -c 'import keyring; keyring.set_password("$(SMTP)", "$(FROM)", "$(PASS)")'

59
urlwatch/urlwatch/lib/hooks.py
View File

@ -1,59 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# version detector for urlwatch
#
import cssselect.xpath
import lxml.html
import os.path
import re
import yaml
urlwatch_dir = os.path.expanduser(os.path.join('~', '.urlwatch'))
urls_txt = os.path.join(urlwatch_dir, 'urls.txt')
urls_yml = os.path.join(urlwatch_dir, 'urls.yml')
config = yaml.load(open(urls_yml))
tr = cssselect.xpath.HTMLTranslator()
def filter(url, data):
try:
for key, cfg in config.items():
if url.endswith('#' + key):
if not url.startswith(cfg['url']):
raise Exception('url mismatch')
if 'xpath' in cfg:
xpath = cfg['xpath']
elif 'css' in cfg:
xpath = tr.css_to_xpath(cfg['css'])
else:
raise Exception('xpath/css required')
break
else:
return data
dom = lxml.html.fromstring(data.encode('utf-8'))
txt = dom.xpath(xpath)[0]
ver = re.search(r'(?i)([0-9]+\.)*[0-9]+(-?(alpha|beta|rc)[0-9]+)?', txt).group(0).lower()
return '{}: {}\n'.format(key, ver)
except:
return '{}: {}\n'.format(key, 'unknown')
if __name__ == '__main__':
print 'Extract urls from <{}> ({})'.format(urls_yml, len(config))
urls = ['{0[url]}#{1}\n'.format(cfg, key) for key, cfg in config.items()]
print 'Write urls to <{}> ({})'.format(urls_txt, len(urls))
with open(urls_txt, 'w') as f:
f.writelines(urls)
print 'Everything is OK!'

145
urlwatch/urlwatch/urls.yml
View File

@ -1,145 +0,0 @@
###### OS ######
alpine:
url: http://alpinelinux.org/
xpath: //span[@class="icon-download"]/following-sibling::p/a/text()
archlinux:
url: https://www.archlinux.org/download/
xpath: //li[strong="Current Release:"]/text()
centos:
url: http://www.centos.org/download/
xpath: //div[@class="downloadbutton"]/a[.="DVD ISO"]/@href
coreos:
url: https://coreos.com
xpath: //div[@class="co-p-homepage-release-text"]/text()
debian:
url: https://www.debian.org/releases/
xpath: //li[contains(., "current stable release")]/a/text()
fedora:
url: https://getfedora.org/en/server/download/
xpath: //h2[starts-with(., "Download")]/text()
freebsd:
url: http://www.freebsd.org/releases/
xpath: //div[@id="sidenav"]//li[contains(text(), "Production Release:")][1]/a/text()
kali:
url: https://www.kali.org/kali-linux-releases/
xpath: //h2[.="Kali Linux Release History"]/following-sibling::ul/li[1]/strong/text()
raspbian:
url: https://www.raspberrypi.org/downloads/
xpath: //div[h3="Raspbian"]//span[.="Kernel version:"]/following-sibling::strong/text()
redhat:
url: https://access.redhat.com/products/red-hat-enterprise-linux/evaluation
xpath: //a[contains(@class, "btn-download")]/small/text()
smartos:
url: https://github.com/joyent/smartos-live/releases
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
ubuntu:
url: http://www.ubuntu.com/download/server
xpath: //div[contains(@class, "row-hero")]//h2/text()
###### LANG ######
go:
url: https://golang.org/dl/
xpath: //h3[.="Stable versions"]/following-sibling::div[1]/@id
nodejs:
url: https://nodejs.org/
xpath: //div[@id="intro"]/p[@class="version"]/text()
python:
url: https://www.python.org/downloads/
xpath: //div[@class="download-unknown"]/p[@class]/a[1]/text()
###### DB ######
mongodb:
url: https://www.mongodb.org/downloads
xpath: //div[@id="production-release-header"]/h2/@data-production-version
redis:
url: http://redis.io
xpath: //a[contains(@href, "/releases/")]/text()
influxdb:
url: https://influxdb.com/download/index.html
xpath: //h2[contains(., "Stable")]/text()
###### TOOL ######
vagrant:
url: https://www.vagrantup.com/downloads.html
xpath: //h2[@class="os-name" and .="Mac OS X"]/following-sibling::ul/li/a/@href
git:
url: http://git-scm.com
xpath: //span[@class="version"]/text()
docker:
url: https://github.com/docker/docker/releases
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
phantomjs:
url: http://phantomjs.org
xpath: //span[@class="version"]/text()
firefox:
url: https://www.mozilla.org/en-US/firefox/new/
xpath: //li[@class="os_osx"]/a[@class="download-link"]/@href
jquery:
url: http://jquery.com
xpath: //span[@class="download"]/following-sibling::span/text()
vim:
url: http://en.wikipedia.org/wiki/Vim_%28text_editor%29
xpath: //th[a="Stable release"]/following-sibling::td/text()
scrapy:
url: https://github.com/scrapy/scrapy/releases
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
scrapyd:
url: https://github.com/scrapy/scrapyd/releases
xpath: //div[contains(@class, "label-latest")]//h1[@class="release-title"]/a/text()
urlwatch:
url: https://github.com/thp/urlwatch/releases
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
monit:
url: http://mmonit.com/monit/changes/
xpath: //div[@class="container"]/div[@class="row"][1]//h3/a/text()
virtualbox:
url: https://www.virtualbox.org/wiki/Downloads
xpath: //strong/following-sibling::a[@class="ext-link" and contains(@href, "dmg")]/@href
###### VPN ######
openvpn:
url: https://openvpn.net/index.php/open-source/downloads.html
xpath: //h2[contains(., "released on")]/text()
tinc:
url: http://tinc-vpn.org/
xpath: //h3/a[contains(@href, "download")]/text()
shadowsocks:
url: https://github.com/shadowsocks/shadowsocks/releases
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
shadowsocks-libev:
url: https://github.com/shadowsocks/shadowsocks-libev/releases
xpath: //div[contains(@class, "label-latest")]//h1[@class="release-title"]/a/text()