diff --git a/youtube/youtube-worker/README.md b/youtube/youtube-worker/README.md index b0d931f..8b7bf43 100644 --- a/youtube/youtube-worker/README.md +++ b/youtube/youtube-worker/README.md @@ -33,23 +33,23 @@ $ docker-compose up -d $ docker-compose logs Attaching to youtube_worker_1 -worker_1 | 2015-07-12T16:07:07 [INFO] connect redis -worker_1 | 2015-07-12T16:07:26 [INFO] process: os6U77Hhm_s +worker_1 | 2015-07-12T17:50:02 [INFO] connect redis +worker_1 | 2015-07-12T17:50:06 [INFO] process: os6U77Hhm_s worker_1 | [youtube] os6U77Hhm_s: Downloading webpage worker_1 | [youtube] os6U77Hhm_s: Downloading video info webpage worker_1 | [youtube] os6U77Hhm_s: Extracting video information worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest worker_1 | [download] Destination: Shia LaBeouf TED Talk-os6U77Hhm_s.mp4 -[download] 100% of 11.03MiB in 00:0297MiB/s ETA 00:00known ETA -worker_1 | 2015-07-12T16:07:28 [INFO] success: True +[download] 100% of 11.03MiB in 00:0014MiB/s ETA 00:001nown ETA +worker_1 | 2015-07-12T17:50:07 [INFO] success: True ``` ## client ``` $ redis-cli -h server -a 'secret-passwd' -server> lpush urls os6U77Hhm_s +server> lpush pending os6U77Hhm_s server> keys * 1) "pending" server> keys * @@ -61,9 +61,9 @@ server> quit $ rsync -ahP user@server:fig/youtube/data receiving file list ... 2 files to consider -drwxr-xr-x 74 2015/07/13 00:52:25 data --rw-r--r-- 0 2015/07/13 01:04:40 data/Shia LaBeouf TED Talk-os6U77Hhm_s.mp4 +drwxr-xr-x 74 2015/07/13 01:50:07 data +-rw-r--r-- 11569834 2015/06/15 17:19:16 data/Shia LaBeouf TED Talk-os6U77Hhm_s.mp4 sent 16 bytes received 116 bytes 29.33 bytes/sec -total size is 0 speedup is 0.00 +total size is 11.57M speedup is 87650.26 ``` diff --git a/youtube/youtube-worker/worker.py b/youtube/youtube-worker/worker.py index 2735ec7..d45d4ab 100755 --- a/youtube/youtube-worker/worker.py +++ b/youtube/youtube-worker/worker.py @@ -19,7 +19,7 @@ def download(url): if status == 'downloading': rdb.zadd('running', now, url) - elif info['status'] == 'error': + elif status == 'error': rdb.zrem('running', url) rdb.zadd('error', now, url) elif status == 'finished': @@ -29,13 +29,18 @@ def download(url): if rdb.zrank('finished', url) != None: return False - opts = { - 'format': os.getenv('FORMAT', 'best'), - 'progress_hooks': [hook], - } - - with youtube_dl.YoutubeDL(opts) as ydl: - ydl.download([url]) + try: + opts = { + 'format': os.getenv('FORMAT', 'best'), + 'progress_hooks': [hook], + } + with youtube_dl.YoutubeDL(opts) as ydl: + ydl.download([url]) + except Exception as ex: + logging.error('error: %s', ex) + rdb.zrem('running', url) + rdb.zadd('error', int(time.time()), url) + return False return True