无限卷轴与Scrapy-Playwriter不工作

whhtz7ly 于 2022-11-09 发布在其他

关注(0)|答案(1)|浏览(182)

import scrapy
from ..items import CentriItem
from scrapy_playwright.page import PageMethod

class NikespiderSpider(scrapy.Spider):
    name = 'nikespider'

    def start_requests(self):
        url = "https://www.nike.com/ch/en/w/mens-nik1?q=shirts"
        yield scrapy.Request(url, meta=dict(
            playwright=True,
            playwright_include_page=True,
            playwright_page_methods=[
                PageMethod("wait_for_selector", ".product-card__link-overlay"),
                PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)"),
                PageMethod("wait_for_selector",".css-c2ovjx:nth-child(25) .product-card__link-overlay"),
            ]
        ))

    async def parse(self, response):
        page = response.meta["playwright_page"]
        await page.close()
        i = 0
        for link in response.css(".product-card__link-overlay::attr(href)"):
            i = i + 1
            print(i)

    async def parse_items(self, response,link):
        pass

这是我的代码刮耐克。网址，我刮有无限滚动。它加载24个项目在第一次，你向下滚动后，它加载24多。我试图添加这个功能使用scrapy剧作家，但它是不工作。有人能帮助我吗？

scrapy

来源：https://stackoverflow.com/questions/73851405/infinite-scroll-with-scrapy-playwright-not-working

1条答案

按热度按时间

oaxa6hgo1#

看起来从他们的api中提取数据可能会更容易。

import scrapy

class NikespiderSpider(scrapy.Spider):
    name = 'nikespider'

    def start_requests(self):
        for i in range(1, 20):
            v = i * 12
            url = f"https://api.nike.com/cic/browse/v2?queryid=products&anonymousId=699ECBF06AACCD31DCB69E0C71964748&country=ch&endpoint=%2Fproduct_feed%2Frollup_threads%2Fv2%3Ffilter%3Dmarketplace(CH)%26filter%3Dlanguage(en-GB)%26filter%3DemployeePrice(true)%26filter%3DattributeIds(0f64ecc7-d624-4e91-b171-b83a03dd8550)%26searchTerms%3Dshirts%26anchor%3D{v}%26consumerChannelId%3Dd9a5bc42-4b9c-4976-858a-f159cf99c647%26count%3D24&language=en-GB&localizedRangeStr=%7BlowestPrice%7D%E2%80%94%7BhighestPrice%7D"
            yield scrapy.Request(url=url)

    def parse(self, response):
        yield response.json()

第一页的输出如下所示：

{
    "data": {
      "products": {
        "errors": null,
        "products": [
          {
            "cardType": "default",
            "cloudProductId": "a44f2253-c745-5143-bd1e-f2c841da1550",
            "colorDescription": "Obsidian/Pepper Red/Gorge Green/Gold Dart",
            "colorways": [
              {
                "colorDescription": "Obsidian/Pepper Red/Gorge Green/Gold Dart",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/a77364ff-d0e8-4eaa-83eb-2a206896b28d/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/e1374e74-f614-40dd-aecd-ac4c0bd1a5bb/portugal-french-terry-football-hoodie-rF3Vcw.png"
                },
                "pdpUrl": "{countryLang}/t/portugal-french-terry-football-hoodie-rF3Vcw/DH4826-451",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 109.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 109.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "a44f2253-c745-5143-bd1e-f2c841da1550",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14061770",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "96633291-36b7-4a6d-a1ab-ef0e264da2c2",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/a77364ff-d0e8-4eaa-83eb-2a206896b28d/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/e1374e74-f614-40dd-aecd-ac4c0bd1a5bb/portugal-french-terry-football-hoodie-rF3Vcw.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": false,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14061770",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 109.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 109.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Nike French Terry Football Hoodie",
            "title": "Portugal",
            "url": "{countryLang}/t/portugal-french-terry-football-hoodie-rF3Vcw/DH4826-451"
          },
          {
            "cardType": "default",
            "cloudProductId": "a05bd5c6-290e-5d8c-b583-9c8fd49fee3b",
            "colorDescription": "Coastal Blue/Green Spark/Dynamic Yellow",
            "colorways": [
              {
                "colorDescription": "Coastal Blue/Green Spark/Dynamic Yellow",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/af0e8a3f-7f4b-417c-b7b1-6acd09607ee2/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/d6150c37-9086-4b1a-b1f0-66e59f3258cb/brazil-travel-fleece-football-hoodie-H4PKQk.png"
                },
                "pdpUrl": "{countryLang}/t/brazil-travel-fleece-football-hoodie-H4PKQk/DH4822-490",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 109.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 109.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "a05bd5c6-290e-5d8c-b583-9c8fd49fee3b",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14061554",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "9ba8eaff-d94d-445a-b569-8bae4c1d741b",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/af0e8a3f-7f4b-417c-b7b1-6acd09607ee2/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/d6150c37-9086-4b1a-b1f0-66e59f3258cb/brazil-travel-fleece-football-hoodie-H4PKQk.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": false,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14061554",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 109.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 109.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Fleece Football Hoodie",
            "title": "Brazil Travel",
            "url": "{countryLang}/t/brazil-travel-fleece-football-hoodie-H4PKQk/DH4822-490"
          },
          {
            "cardType": "default",
            "cloudProductId": "40d88287-0129-52c7-ad5e-646ba8333389",
            "colorDescription": "Game Royal/Blue Fury/Blue Fury",
            "colorways": [
              {
                "colorDescription": "Game Royal/Blue Fury/Blue Fury",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/b66e9663-2fd1-4398-9270-d4e4d46ed83e/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/873eaab7-cfec-4b70-b6af-19880a563637/england-strike-dri-fit-short-sleeve-football-top-1m2GSc.png"
                },
                "pdpUrl": "{countryLang}/t/england-strike-dri-fit-short-sleeve-football-top-1m2GSc/DH6443-480",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 59.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 59.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "40d88287-0129-52c7-ad5e-646ba8333389",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14062596",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "f73148f5-4b63-4b1f-84cc-b03bcc408ca1",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/b66e9663-2fd1-4398-9270-d4e4d46ed83e/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/873eaab7-cfec-4b70-b6af-19880a563637/england-strike-dri-fit-short-sleeve-football-top-1m2GSc.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": true,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14062596",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 59.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 59.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Nike Dri-FIT Short-Sleeve Football Top",
            "title": "England Strike",
            "url": "{countryLang}/t/england-strike-dri-fit-short-sleeve-football-top-1m2GSc/DH6443-480"
          },
          {
            "cardType": "default",
            "cloudProductId": "e04c4bb3-4382-5df4-9f03-7c1d393f9d05",
            "colorDescription": "Obsidian/Green Glow/Green Glow",
            "colorways": [
              {
                "colorDescription": "Obsidian/Green Glow/Green Glow",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/4cbd4672-d5ac-453b-b023-2d5160b33341/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/7f04a8ed-649c-4705-b110-0c605b852f7b/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs.png"
                },
                "pdpUrl": "{countryLang}/t/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs/DN0676-451",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 119.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 119.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "e04c4bb3-4382-5df4-9f03-7c1d393f9d05",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14063595",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "0027a0c6-0e97-445c-b6ed-a92a6bfdf18b",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/4cbd4672-d5ac-453b-b023-2d5160b33341/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/7f04a8ed-649c-4705-b110-0c605b852f7b/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": true,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14063595",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 119.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 119.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Nike Dri-FIT Football Shirt",
            "title": "Australia 2022/23 Stadium Away",
            "url": "{countryLang}/t/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs/DN0676-451"
          },
          {
            "cardType": "default",
            "cloudProductId": "758da5f5-ea79-5bf7-a401-8ccb945a5f16",
            "colorDescription": "Off-Noir/White",
            "colorways": [
              {
                "colorDescription": "Off-Noir/White",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/1efdbff3-1c9e-4984-9bf2-8da11d8f5161/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/5ca1f9a9-f1fa-42fe-a552-571ffbcc8ba6/nikecourt-dri-fit-rafa-challenger-short-sleeve-tennis-top-wNcD3N.png"
                },
                "pdpUrl": "{countryLang}/t/nikecourt-dri-fit-rafa-challenger-short-sleeve-tennis-top-wNcD3N/DD8547-045",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 84.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 84.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "758da5f5-ea79-5bf7-a401-8ccb945a5f16",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14168043",
                "prebuildId": null,
                "productInstanceId": null
              },
              {
                "colorDescription": "Pure Platinum/Pink Gaze/Black",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/dbf2edc2-da8b-4e7e-976f-5d16f207b449/image.jpg",

...

赞(0）回复(0）举报 2022-11-09

我来回答

无限卷轴与Scrapy-Playwriter不工作

1条答案

相关问题

热门标签

最新问答