scrapy 如何从HTML中精确的JSON子节点提取数据

oalqel3c  于 2023-10-20  发布在  其他
关注(0)|答案(1)|浏览(98)

我试图从json格式的网站(https://www.bergzeit.de/marken/salewa/)中提取产品数据(名称,价格,url)以及尺寸和颜色。我无法找到一种方法来获得'elementList',在那里我可以看到编码的数据。我使用这段代码找到了页面上正确的脚本。

scripts = response.xpath('//script/text()').getall()
the_script = [s for s in scripts if 'elementsList' in s] 
data = json.loads(the_script[0])

我得到JSONDecodeError: Expecting value: line 2 column 1 (char 1)
这就是HTML

<script>
window.__initialAppState = {
  global: {
    gtmId: 'GTM-NKB4VN4',
    cookiePresent: false,
    user: {
      loggedIn: false,
      b2b: false,
      b2bPrefs: {},
      b2bAdmin: false
    },

  modules: {
    checkout: {

      removePromoCodeUrl: '/summary/removevoucher/'
    },
    accountPage: {

    },
    loyaltyPage: {
      urls: {
        register: '/login/?registerAccountUrl=/register/loyalty/',
        overview: '/club'
      },

    },
    productsListPage: {

          headline: JSON.parse("\"Salewa\""),

        resultCount: 1754,
        elementsList: [{"data":{"brand":{"name":"Salewa"},"description":"Wetterfest und ultrawarm für anspruchsvolle Herren: 550 cuin Daunenjacke mit wind- und wasserabweisender Außenschicht","features":[],"googleClick":{"event":"eec.productClick","ecommerce":{"click":{"actionField":{"list":"brand"},"products":[{"brand":"Salewa","category":"Marken/Salewa","id":"1080071-051","name":"Herren Brenta Jacke","position":"1","price":"167.18","variant":"dark olive|S"}]}}},"googleImpression":{"event":"eec.productImpression","ecommerce":{"impressions":[{"brand":"Salewa","category":"Marken/Salewa","id":"1080071-051","list":"brand","name":"Herren Brenta Jacke","position":"1","price":"167.18","variant":"dark olive|S"}]}},"id":"1080071","images":[{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic1.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic3.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic4.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic5.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic6.jpg","title":"Herren Brenta Jacke"},{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic7.jpg","title":"Herren Brenta Jacke"}],"isNew":true,"itemId":"1080071-051","labels":["sale"],"manyAvailableSizes":true,"name":"Herren Brenta Jacke","price":{"savedAmount":"51,00 €","savedPercentage":20,"priceForSchemaOrgOffer":"198.95","previous":"249,95 €","old":"249,95 €","current":"198,95 €","conf":"189,00 €","isFromPrice":false,"isOldPrice":true,"isRetailPrice":false,"isSignificantlyReduced":true,"bestSavedAmount":"51,00 €"},"productId":"1080071","ratings":4.571428571428571,"ratingsCount":14,"sellingPoints":[],"sizedImages":[{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic1.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic1.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic3.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic3.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic4.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic4.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic5.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic5.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic6.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic6.jpg","title":"Herren Brenta Jacke"}},{"mobile":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular_mobile/1080071-051_pic7.jpg","title":"Herren Brenta Jacke"},"desktop":{"alt":"Herren Brenta Jacke","src":"https://static.bergzeit.com/product_tile_regular/1080071-051_pic7.jpg","title":"Herren Brenta Jacke"}}],"sustainable":true,"url":"/p/salewa-herren-brenta-jacke/1080071/#itemId=1080071-051","variations":[{"attributeName":"import:style","colorVariation":true,"display_string":"Herstellerfarbe","name":"var[import:style]","options":[{"isAnyAvailable":true,"isAvailable":false,"label":"autumnal","stock":0,"value":"4171-autumnal","valueThumb":"orange"},{"isAnyAvailable":true,"isAvailable":false,"label":"black out","stock":0,"value":"0910-black_out","valueThumb":"schwarz"},{"isAnyAvailable":true,"isAvailable":false,"label":"dark olive","stock":0,"value":"5281-dark_olive","valueThumb":"oliv"},{"isAnyAvailable":true,"isAvailable":false,"label":"electric","stock":0,"value":"8621-electric","valueThumb":"blau"},{"isAnyAvailable":true,"isAvailable":false,"label":"flame","stock":0,"value":"1501-flame","valueThumb":"rot"},{"isAnyAvailable":true,"isAvailable":false,"label":"gold","stock":0,"value":"2191-gold","valueThumb":"gelb"},{"isAnyAvailable":true,"isAvailable":false,"label":"syrah","stock":0,"value":"1571-syrah","valueThumb":"rot"},{"isAnyAvailable":true,"isAvailable":false,"label":"thyme","stock":0,"value":"5561-thyme","valueThumb":"gruen"}],"selectedOption":"5281-dark_olive"},{"attributeName":"import:manufacturer_size","colorVariation":false,"display_string":"Herstellergröße","name":"var[import:manufacturer_size]","options":[{"isAnyAvailable":true,"isAvailable":false,"label":"S","sameDiscount":true,"value":"S"},{"isAnyAvailable":true,"isAvailable":false,"label":"M","sameDiscount":true,"value":"M"},{"isAnyAvailable":true,"isAvailable":false,"label":"L","sameDiscount":true,"value":"L"},{"isAnyAvailable":true,"isAvailable":false,"label":"XL","sameDiscount":true,"value":"XL"},{"isAnyAvailable":true,"isAvailable":false,"label":"XXL","sameDiscount":true,"value":"XXL"}],"selectedOption":"S"}]},"type":"product"}],
        oneTileTeaserInsertions: [],
        salesTeaserInsertion: "",
        highlightedArticlesIds: [],
        selectableVariations: ["import:Var_Article"],

          currentCategory: {"id":"-s-2831-salewa","name":"Salewa","path":"marken-salewa","url":"/marken/salewa/?show=all","root":false},
          parentCategory: {"id":"-s-2831","name":"Marken","path":"marken","url":"/marken/?show=all","root":false},
        subCategoryTeasers: [],
        search: JSON.parse("null"),
        ms: null,
        isBot: false,
      
    },
    productPage: {

          productData: {},

    },
    cartPage:
    {}
    ,
    wishlistPage: {

    }
  }
};
</script>
nvbavucw

nvbavucw1#

你得到一个错误,因为window.__initialAppState不是JSON,而是JavaScript结构。
一种方法是将其视为string并提取elementsList,可以将其解析为JSON。你可以使用正则表达式,比如elementsList: (.*)。然后,您可以使用python json模块将结果字符串作为JSON处理。
希望这对你有帮助:)

相关问题