From 0719f68028062ae374713ce35579ea478de73ad0 Mon Sep 17 00:00:00 2001
From: kev <vimagick@gmail.com>
Date: Mon, 13 Jul 2015 00:43:50 +0800
Subject: [PATCH] add youtube-dl

---
 README.md                                     |  8 ++-
 {youtube-dl => youtube/youtube-dl}/Dockerfile |  0
 {youtube-dl => youtube/youtube-dl}/README.md  |  0
 youtube/youtube-worker/Dockerfile             | 16 ++++++
 youtube/youtube-worker/README.md              | 55 +++++++++++++++++++
 youtube/youtube-worker/docker-compose.yml     | 16 ++++++
 youtube/youtube-worker/worker.py              | 32 +++++++++++
 7 files changed, 126 insertions(+), 1 deletion(-)
 rename {youtube-dl => youtube/youtube-dl}/Dockerfile (100%)
 rename {youtube-dl => youtube/youtube-dl}/README.md (100%)
 create mode 100644 youtube/youtube-worker/Dockerfile
 create mode 100644 youtube/youtube-worker/README.md
 create mode 100644 youtube/youtube-worker/docker-compose.yml
 create mode 100755 youtube/youtube-worker/worker.py

diff --git a/README.md b/README.md
index af3f9a6..b287d74 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,8 @@ https://hub.docker.com/u/vimagick/
 - [x] collectd
 - [x] dante
 - [x] dnscrypt
+    - [x] dnscrypt-proxy
+    - [x] dnscrypt-wrapper
 - [x] dokuwiki
 - [x] ferm
 - [x] fteproxy
@@ -43,6 +45,8 @@ https://hub.docker.com/u/vimagick/
 - [x] pure-ftpd
 - [x] scrapyd
 - [x] shadowsocks
+    - [x] shadowsocks
+    - [x] shadowsocks-libev
 - [x] shadowvpn
 - [x] splash
 - [x] taskd
@@ -51,7 +55,9 @@ https://hub.docker.com/u/vimagick/
 - [x] tor
 - [x] urlwatch
 - [x] webkit
-- [x] youtube-dl :+1:
+- [x] youtube
+    - [x] youtube-dl
+    - [x] youtube-worker
 
 ## 3rd-party
 
diff --git a/youtube-dl/Dockerfile b/youtube/youtube-dl/Dockerfile
similarity index 100%
rename from youtube-dl/Dockerfile
rename to youtube/youtube-dl/Dockerfile
diff --git a/youtube-dl/README.md b/youtube/youtube-dl/README.md
similarity index 100%
rename from youtube-dl/README.md
rename to youtube/youtube-dl/README.md
diff --git a/youtube/youtube-worker/Dockerfile b/youtube/youtube-worker/Dockerfile
new file mode 100644
index 0000000..d581b44
--- /dev/null
+++ b/youtube/youtube-worker/Dockerfile
@@ -0,0 +1,16 @@
+#
+# Dockerfile for youtube-worker
+#
+
+FROM alpine
+MAINTAINER kev <noreplay@datageek.info>
+
+RUN apk add -U ca-certificates py-pip \
+    && rm -rf /var/cache/apk/* \
+    && pip install redis youtube-dl
+
+COPY worker.py /code/
+VOLUME /data
+WORKDIR /data
+
+CMD ["python", "/code/worker.py"]
diff --git a/youtube/youtube-worker/README.md b/youtube/youtube-worker/README.md
new file mode 100644
index 0000000..8735144
--- /dev/null
+++ b/youtube/youtube-worker/README.md
@@ -0,0 +1,55 @@
+youtube-worker
+==============
+
+youtube-worker = youtube-dl + redis
+
+## docker-compose.yml
+
+```
+worker:
+  image: vimagick/youtube-worker
+  links:
+    - redis
+  volumes:
+    - data:/data
+  environment:
+    - PASSWORD=secret-passwd
+  restart: always
+
+redis:
+  image: redis
+  command: redis-server --requirepass 'secret-passwd'
+  ports:
+    - "6379:6379"
+  restart: always
+```
+
+## server
+
+```
+$ cd ~/fig/youtube/
+
+$ docker-compose up -d
+
+$ docker-compose logs
+Attaching to youtube_worker_1
+worker_1 | 2015-07-12T16:07:07 [INFO] connect redis
+worker_1 | 2015-07-12T16:07:26 [INFO] process: %s
+worker_1 | [youtube] os6U77Hhm_s: Downloading webpage
+worker_1 | [youtube] os6U77Hhm_s: Downloading video info webpage
+worker_1 | [youtube] os6U77Hhm_s: Extracting video information
+worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest
+worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest
+worker_1 | [download] Destination: Shia LaBeouf TED Talk-os6U77Hhm_s.mp4
+[download] 100% of 11.03MiB in 00:0297MiB/s ETA 00:00known ETA
+```
+
+## client
+
+```
+$ redis-cli -h server -a 'secret-passwd'
+server> lpush urls 'https://www.youtube.com/watch?v=os6U77Hhm_s'
+server> quit
+
+$ rsync -ahP user@server:fig/youtube/data ~/Movies/youtube
+```
diff --git a/youtube/youtube-worker/docker-compose.yml b/youtube/youtube-worker/docker-compose.yml
new file mode 100644
index 0000000..fdad23e
--- /dev/null
+++ b/youtube/youtube-worker/docker-compose.yml
@@ -0,0 +1,16 @@
+worker:
+  image: vimagick/youtube-worker
+  links:
+    - redis
+  volumes:
+    - data:/data
+  environment:
+    - PASSWORD=secret-passwd
+  restart: always
+
+redis:
+  image: redis
+  command: redis-server --requirepass 'secret-passwd'
+  ports:
+    - "6379:6379"
+  restart: always
diff --git a/youtube/youtube-worker/worker.py b/youtube/youtube-worker/worker.py
new file mode 100755
index 0000000..35c98fd
--- /dev/null
+++ b/youtube/youtube-worker/worker.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# youtube_dl worker
+#
+
+import logging
+import os
+import redis
+import youtube_dl
+
+
+def download(url):
+
+    with youtube_dl.YoutubeDL() as ydl:
+        ydl.download([url])
+
+
+if __name__ == '__main__':
+
+    logging.basicConfig(format='%(asctime)s [%(levelname)s] %(msg)s', datefmt='%FT%T', level='INFO')
+    logging.info('connect redis')
+    rdb = redis.StrictRedis(host='redis', password=os.getenv('PASSWORD'))
+    rdb.ping()
+
+    while True:
+        try:
+            _, url = rdb.brpop('urls')
+            logging.info('process: %s', url)
+            download(url)
+        except Exception as ex:
+            logging.error('error: %s', ex)
+