mirror of
https://github.com/vimagick/dockerfiles
synced 2024-06-16 11:58:47 +00:00
add scrapyrt tutorial for scrapyd
This commit is contained in:
parent
c349530e41
commit
299d84de53
@ -18,7 +18,7 @@ utility which allows you to deploy your project to a Scrapyd server.
|
|||||||
|
|
||||||
[pillow][6] is the Python Imaging Library to support the ImagesPipeline.
|
[pillow][6] is the Python Imaging Library to support the ImagesPipeline.
|
||||||
|
|
||||||
This image is based on `debian:jessie`, 6 latest python packages are installed:
|
This image is based on `debian:stretch`, 6 latest python packages are installed:
|
||||||
|
|
||||||
- `scrapy`: git+https://github.com/scrapy/scrapy.git
|
- `scrapy`: git+https://github.com/scrapy/scrapy.git
|
||||||
- `scrapyd`: git+https://github.com/scrapy/scrapyd.git
|
- `scrapyd`: git+https://github.com/scrapy/scrapyd.git
|
||||||
@ -48,6 +48,16 @@ scrapy:
|
|||||||
- .:/code
|
- .:/code
|
||||||
working_dir: /code
|
working_dir: /code
|
||||||
restart: always
|
restart: always
|
||||||
|
|
||||||
|
scrapyrt:
|
||||||
|
image: vimagick/scrapyd
|
||||||
|
command: scrapyrt -i 0.0.0.0 -p 9080
|
||||||
|
ports:
|
||||||
|
- "9080:9080"
|
||||||
|
volumes:
|
||||||
|
- .:/code
|
||||||
|
working_dir: /code
|
||||||
|
restart: always
|
||||||
```
|
```
|
||||||
|
|
||||||
## Run it as background-daemon for scrapyd
|
## Run it as background-daemon for scrapyd
|
||||||
@ -122,6 +132,14 @@ $ docker-compose run --rm scrapy
|
|||||||
>>> exit
|
>>> exit
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Run it as realtime crawler for scrapyrt
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ git clone https://github.com/scrapy/quotesbot.git .
|
||||||
|
$ docker-compose up -d scrapyrt
|
||||||
|
$ curl -s 'http://localhost:9080/crawl.json?spider_name=toscrape-css&callback=parse&url=http://quotes.toscrape.com/&max_requests=5' | jq -c '.items[]'
|
||||||
|
```
|
||||||
|
|
||||||
[1]: https://github.com/scrapy/scrapy
|
[1]: https://github.com/scrapy/scrapy
|
||||||
[2]: https://github.com/scrapy/scrapyd
|
[2]: https://github.com/scrapy/scrapyd
|
||||||
[3]: https://github.com/scrapy/scrapyd-client
|
[3]: https://github.com/scrapy/scrapyd-client
|
||||||
|
@ -14,3 +14,13 @@ scrapy:
|
|||||||
- .:/code
|
- .:/code
|
||||||
working_dir: /code
|
working_dir: /code
|
||||||
restart: always
|
restart: always
|
||||||
|
|
||||||
|
scrapyrt:
|
||||||
|
image: vimagick/scrapyd
|
||||||
|
command: scrapyrt -i 0.0.0.0 -p 9080
|
||||||
|
ports:
|
||||||
|
- "9080:9080"
|
||||||
|
volumes:
|
||||||
|
- .:/code
|
||||||
|
working_dir: /code
|
||||||
|
restart: always
|
||||||
|
Loading…
Reference in New Issue
Block a user