From e64c294ca257a14b9b4e346bcbcac71b02be768e Mon Sep 17 00:00:00 2001 From: kev Date: Thu, 28 May 2015 23:11:37 +0800 Subject: [PATCH] update --- scrapyd/001-scrapyd | 2 -- scrapyd/Dockerfile | 17 +++++++++++++---- scrapyd/scrapyd.conf | 34 +++++++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 15 deletions(-) delete mode 100644 scrapyd/001-scrapyd diff --git a/scrapyd/001-scrapyd b/scrapyd/001-scrapyd deleted file mode 100644 index 14f164b..0000000 --- a/scrapyd/001-scrapyd +++ /dev/null @@ -1,2 +0,0 @@ -[scrapyd] -items_dir = diff --git a/scrapyd/Dockerfile b/scrapyd/Dockerfile index 012646d..0f0e65c 100644 --- a/scrapyd/Dockerfile +++ b/scrapyd/Dockerfile @@ -5,6 +5,9 @@ FROM debian:jessie MAINTAINER kev +ADD ./scrapyd.conf /etc/scrapyd/ +ADD ./requirements.txt /etc/scrapyd/ + RUN apt-get update \ && apt-get install -y autoconf \ build-essential \ @@ -35,6 +38,7 @@ RUN apt-get update \ && rm -rf libxslt \ && curl -sSL https://bootstrap.pypa.io/get-pip.py | python \ && pip install scrapy==0.24.6 scrapyd==1.0.1 \ + && pip install -r /etc/scrapyd/requirements.txt \ && curl -sSL https://github.com/scrapy/scrapy/raw/master/extras/scrapy_bash_completion -o /etc/bash_completion.d/scrapy_bash_completion \ && echo 'source /etc/bash_completion.d/scrapy_bash_completion' >> /root/.bashrc \ && apt-get remove -y autoconf \ @@ -46,8 +50,13 @@ RUN apt-get update \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* -CMD scrapyd -u scrapy \ - -g nogroup \ - --pidfile /var/run/scrapyd.pid \ - -l /dev/stdout +RUN mkdir -p /var/lib/scrapyd/dbs \ + /var/lib/scrapyd/eggs \ + /var/lib/scrapyd/items \ + /var/lib/scrapyd/logs +VOLUME /etc/scrapyd/ /var/lib/scrapyd/ + +EXPOSE 6800 + +CMD ["scrapyd"] diff --git a/scrapyd/scrapyd.conf b/scrapyd/scrapyd.conf index 1b72193..71e1145 100644 --- a/scrapyd/scrapyd.conf +++ b/scrapyd/scrapyd.conf @@ -1,10 +1,26 @@ -[inet_http_server] -port = :9001 -username = admin -password = {SHA}d033e22ae348aeb5660fc2140aec35850c4da997 +[scrapyd] +eggs_dir = /var/lib/eggs +logs_dir = /var/lib/logs +items_dir = /var/lib/items +dbs_dir = /var/lib/dbs +jobs_to_keep = 5 +max_proc = 0 +max_proc_per_cpu = 4 +finished_to_keep = 100 +poll_interval = 5 +http_port = 6800 +debug = off +runner = scrapyd.runner +application = scrapyd.app.application +launcher = scrapyd.launcher.Launcher -[program:scrapyd] -command = /usr/bin/scrapyd -u scrapy - -g nogroup - --pidfile /var/run/scrapyd.pid - -l /var/log/scrapyd/scrapyd.log +[services] +schedule.json = scrapyd.webservice.Schedule +cancel.json = scrapyd.webservice.Cancel +addversion.json = scrapyd.webservice.AddVersion +listprojects.json = scrapyd.webservice.ListProjects +listversions.json = scrapyd.webservice.ListVersions +listspiders.json = scrapyd.webservice.ListSpiders +delproject.json = scrapyd.webservice.DeleteProject +delversion.json = scrapyd.webservice.DeleteVersion +listjobs.json = scrapyd.webservice.ListJobs