scrapy 我如何在所有页面上抓取此表[没有页面按钮的链接]

mec1mxoz  于 2022-11-09  发布在  其他
关注(0)|答案(1)|浏览(190)

我想不出在此网站上刮牌的方法(单击“所有移动”选项卡):https://www.investmentnews.com/aotm
有多个页面,但页面按钮没有链接(见图)。我如何从这个表中抓取数据?

sc4hvdpw

sc4hvdpw1#

所有的页面都没有链接,因为页面按钮所做的就是将页面上的所有信息替换为从 AJAX 调用API端点所接收到的信息。你可以在devtools的network标签中找到端点,当你点击其中一个按钮时,它会生成一个请求,你可以在feed中看到该请求,并从中获取URL。
使用API端点:
例如:

import scrapy

class AotmSpider(scrapy.Spider):
    name = 'aotm'

    def start_requests(self):
        for i in range(1,500):
            url = f'https://aotm.in.bonhill.pbc.io/api/getAllAdvisorsListings/{i}'
            yield scrapy.Request(url)

    def parse(self, response):
        yield {"data":response.json()}

单页输出:

"data": [
      { "TotalAdvisors": 117860 },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "ANDRES CORDOBA",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "TD Ameritrade, Inc.",
        "firm_joining": "Fidelity Personal and Workplace Advisors",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "CARLOS IVAN SALVATIERRA-DURAN",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Merrill Lynch, Pierce, Fenner & Smith Incorporated",
        "firm_joining": "Morgan Stanley",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Morgan Stanley",
        "firm_full_url": "http://www.morganstanley.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "CATHERINE LEEMING ANDERSEN",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "J.P. Morgan Securities LLC",
        "firm_joining": "Wells Fargo Clearing Services, LLC",
        "firm_city": "Wilton",
        "firm_state": "CT",
        "firm_full_name": "Wells Fargo",
        "firm_full_url": "http://www.wellsfargoadvisors.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "David Nardiello",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "MFA Wealth Advisors, LLC",
        "firm_joining": "Northwestern Mutual Investment Services, LLC",
        "firm_city": "Pittsburgh",
        "firm_state": "PA",
        "firm_full_name": "",
        "firm_full_url": "http://www.nmfn.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "Elizabeth Anne OConnor",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Wells Fargo Clearing Services, LLC",
        "firm_joining": "The Huntington Investment Company",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "https://www.huntington.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "FELICIA A JACOBS",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Merrill Lynch, Pierce, Fenner & Smith Incorporated",
        "firm_joining": "LPL Financial LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "LPL Financial LLC",
        "firm_full_url": "https://www.lpl.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "JAMES GERARD STANTON",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "PFS Investments Inc.",
        "firm_joining": "J.P. Morgan Securities LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "JP Morgan",
        "firm_full_url": "http://www.jpmorgansecurities.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "Jared Michael Cox",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "U.S. Bancorp Investments, Inc.",
        "firm_joining": "First Horizon Advisors, Inc.",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "http://www.firsthorizon.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "Joshua W Dowden",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Edward Jones",
        "firm_joining": "LPL Financial LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "LPL Financial LLC",
        "firm_full_url": "https://www.lpl.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "KOREY JOHN DOUCETTE",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "PFS Investments Inc.",
        "firm_joining": "Terra Wealth",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "LISA ANNE MALICK",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Cambridge Investment Research Advisors, Inc.",
        "firm_joining": "Petra Financial Advisors Inc",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Charles Schwab & Co., Inc.",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "MARCUS CORBIN HENDERSON",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Raymond James Financial Services, Inc.",
        "firm_joining": "Marquis Wealth Management Group",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Charles Schwab & Co., Inc.",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "MARIA ELIZABETH LACASA",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Amerant Investments, Inc.",
        "firm_joining": "IFB Wealth Management LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Federal Home Loan Bank of Atlanta",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "NAIM KAWADRI",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Merrill Lynch, Pierce, Fenner & Smith Incorporated",
        "firm_joining": "Fidelity Personal and Workplace Advisors",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "Paul M. Lamm",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Edward Jones",
        "firm_joining": "Ameriprise Financial Services, LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "American Enterprise Investment Services Inc.",
        "firm_full_url": "http://www.ameriprise.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "PETER CHASE MASON",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Aloha Wealth Partners",
        "firm_joining": "Foundations Investment Advisors LLC",
        "firm_city": "Huntington Beach",
        "firm_state": "CA",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "RICHARD SETH BIRNBAUM",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Merrill Lynch, Pierce, Fenner & Smith Incorporated",
        "firm_joining": "Wells Fargo Clearing Services, LLC",
        "firm_city": "Tenafly",
        "firm_state": "NJ",
        "firm_full_name": "Wells Fargo",
        "firm_full_url": "http://www.wellsfargoadvisors.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-23",
        "adviser_name": "William Merchad",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Maven Wealth Management",
        "firm_joining": "LPL Financial LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "LPL Financial LLC",
        "firm_full_url": "https://www.lpl.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-23",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "Adam James Kunkel",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Edward Jones",
        "firm_joining": "Commonwealth Financial Network",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "http://www.commonwealth.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "Charles Richard Vietzen",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "First Command Brokerage Services, Inc.",
        "firm_joining": "Stride Investments, LLC",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "TD Ameritrade",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "Christine Guo",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "HSBC Securities (USA) Inc.",
        "firm_joining": "Merrill Lynch, Pierce, Fenner & Smith Incorporated",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Merrill Lynch/Bank of America",
        "firm_full_url": "http://www.ml.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "Christopher A Peterson",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Edward Jones",
        "firm_joining": "Commonwealth Financial Network",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Fidelity/NFS/National Financial",
        "firm_full_url": "http://www.commonwealth.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "Colin Francis Igawa",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "J.P. Morgan Securities LLC",
        "firm_joining": "Merrill Lynch, Pierce, Fenner & Smith Incorporated",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "Merrill Lynch/Bank of America",
        "firm_full_url": "http://www.ml.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "DARLYN ALEXA ORELLANA",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Wells Fargo Clearing Services, LLC",
        "firm_joining": "City National Securities, Inc.",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "City National Bank",
        "firm_full_url": "http://www.cnb.com",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      },
      {
        "input_one": "1",
        "date_entered": "2022-09-22",
        "adviser_name": "ERIC DOUGLAS KAUFMAN",
        "subgroup_name": "",
        "firm_url": "",
        "firm_phone": "",
        "firm_email": "",
        "aum_amount": null,
        "rate": null,
        "firm_leaving": "Lockton Investment Advisors, LLC",
        "firm_joining": "Alliant Retirement Consulting",
        "firm_city": "",
        "firm_state": "",
        "firm_full_name": "",
        "firm_full_url": "",
        "firm_contact_person": "",
        "firm_date_two": "2022-09-22",
        "data_source": "Discovery Data",
        "data_source_url": "https://www.discoverydata.com/",
        "sub_adviser_name": "",
        "sub_contact_phone": "",
        "sub_contact_email": "",
        "sub_contact_url": "",
        "sub_contact_datasource": "Discovery Data"
      }
    ]
  },
更新

如果您希望提取所有页面,则可以从数据字典中的“TotalAdvisors”字段计算页面总数

api_url = 'https://aotm.in.bonhill.pbc.io/api/getAllAdvisorsListings/'

def start_requests(self)
    yield scrapy.Request(url=api_url + "1")

def parse(self, response):
   data = response.json()
   date_iter = iter(data)
   total_pages = next(data_iter)["TotalAdvisors"] // (len(data)-1)
   for i in range(2, total_pages):
       yield scrapy.Request(url=api_url + str(i), callback=parse_page)
   yield {"data": data}

def parse_page(self, response):
    yield {"data": response.json())

相关问题