diff --git a/scrapyd/README.md b/scrapyd/README.md index 26a1b64..df453ae 100644 --- a/scrapyd/README.md +++ b/scrapyd/README.md @@ -18,7 +18,7 @@ utility which allows you to deploy your project to a Scrapyd server. [pillow][6] is the Python Imaging Library to support the ImagesPipeline. -This image is based on `debian:jessie`, 6 latest python packages are installed: +This image is based on `debian:stretch`, 6 latest python packages are installed: - `scrapy`: git+https://github.com/scrapy/scrapy.git - `scrapyd`: git+https://github.com/scrapy/scrapyd.git @@ -48,6 +48,16 @@ scrapy: - .:/code working_dir: /code restart: always + +scrapyrt: + image: vimagick/scrapyd + command: scrapyrt -i 0.0.0.0 -p 9080 + ports: + - "9080:9080" + volumes: + - .:/code + working_dir: /code + restart: always ``` ## Run it as background-daemon for scrapyd @@ -122,6 +132,14 @@ $ docker-compose run --rm scrapy >>> exit ``` +## Run it as realtime crawler for scrapyrt + +```bash +$ git clone https://github.com/scrapy/quotesbot.git . +$ docker-compose up -d scrapyrt +$ curl -s 'http://localhost:9080/crawl.json?spider_name=toscrape-css&callback=parse&url=http://quotes.toscrape.com/&max_requests=5' | jq -c '.items[]' +``` + [1]: https://github.com/scrapy/scrapy [2]: https://github.com/scrapy/scrapyd [3]: https://github.com/scrapy/scrapyd-client diff --git a/scrapyd/docker-compose.yml b/scrapyd/docker-compose.yml index df3b3f5..010fe42 100644 --- a/scrapyd/docker-compose.yml +++ b/scrapyd/docker-compose.yml @@ -14,3 +14,13 @@ scrapy: - .:/code working_dir: /code restart: always + +scrapyrt: + image: vimagick/scrapyd + command: scrapyrt -i 0.0.0.0 -p 9080 + ports: + - "9080:9080" + volumes: + - .:/code + working_dir: /code + restart: always