爬取一次之后就不能爬取了，建议爬取的时候在请求头里增加cookie属性，

def text_bs4():
    url = "https://www.maoyan.com/films"
    headers = {'User-Agent': UserAgent().firefox,
               "Cookie": 'uuid_n_v=v1; uuid=8236EB905E8611EEB2C88F0E1D778D2976602F68457842108D4B5999AAA9AC1B; '
                         '_csrf=95caf440d93adc2cf97b85c916a33ea5d547aa09c22e33e4109c0771222dc415; '
                         '_lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; '
                         '_lxsdk_cuid=18adf587d66c8-037b0eb95fbfca-26031e51-1bcab9-18adf587d66c8; '
                         '_lxsdk=8236EB905E8611EEB2C88F0E1D778D2976602F68457842108D4B5999AAA9AC1B; '
                         'Hm_lvt_703e94591e87be68cc8da0da7cbd0be2=1695964233; '
                         'Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2=1695965039; '
                         '__mta=142327309.1695964233411.1695964233411.1695965039712.2; '
                         '_lxsdk_s=18adf587d67-d26-b7b-f21%7C%7C4'}
    resp = requests.get(url, headers=headers)

    soup = BeautifulSoup(resp.text, 'lxml')
    names = [div.text.strip('\n') for div in soup.select("div.channel-detail.movie-item-title")]
    score = [div.text for div in soup.select("div.channel-detail.channel-detail-orange")]
    with open('猫眼bs4.txt', 'w', encoding='utf-8') as f:
        for n, s in zip(names, score):
            print(n, s ,file=f)

使用studio3t的时候都需要在命令行敲一遍代码启动mongo对吗？

严格意义上来说，在搜索引擎搜内容算post请求吧？

import execjs

# 读取js代码
with open('./爬虫与反爬/16js.js','r') as f:
  js_code = f.read()

# 解析js，创建js对象
ctx = execjs.compile(js_code)


t = "MZphJmFlelDpw2aSCfdFb/P3tx6u8VHU/M7MqPRS6y6RaH/5IbXivLEiR9o33DJkTcSPLypQCFpPR82kvps4XAS/QiDAsPVBMK4HU3LUuLyxQLn42XoQKtsRU3nLrOppUcsUCaY8vfPxRtOB4RmS8utPv1yghJtEXPzFsqCxHdcMCUo/o0DpzF5NzSMvlvmYDctx2SVncj3BldMoJn2SZLwPyk2NghU08KyffZyPMaiTmaAeX42LAu8//RhilPgFkR4WUfSd2JSf5WLW1LG0xNJQXx0V1mwtdekmdeH1VkFuapV7vq+eUWCydb4g4fzb+gAwJL8FCmRzBol9j8tdr3ikRFVEttwRl9PG7/ihq/YjCAvWr4S4BAHs4ZRtfo3RMCYFHi+jPkAJWSDArZGriI069tqw9zN04c5G6N4DVQSHwOvm0/JnTWjrIJ/7YTGM+e6lE0DCglS3dHuwxQEGYp3tfxIqnuEMZglV+8rpeVwPoZcWzE3A+0zqJ1ypmhsLk6ZKqpp1jnwvnzCyc3XEvvNlC++1BOPDxaBjjWc94/mcXO37RwjEVQx/h6sCIRJzo9Qwpe2emcsK/ZvAN2433cXPrdZXn1RTWnYkI/NfUMAfib+W54hkccA1krMstvc3oahDlYhOlTlv2OmGoknLoBaWnXiU3Dv51apgdZ4XBpBfN4HOzY12/zq4cnd1u319dBu0fRETiadFXqAUE9nRbmqZtQ4L7/byasPdOOxOagv7nIH4bvcXpfXDqhz6CMDL9Ei+N/dyRWlt3BX/bUPQ6H9E89HbmeJqJFPXZ1HeSHR2Ijl601S2B03lYQcrysSwWh8kzM1D/0N3Dm5z1R3kxvJ5iYpjfPBZH6ozU/4F4E42z1BInZRHViDc79MM6yNB433MMaAl5jsQBSDy33gBwlmx21J35xzmHdsLjmCuVstWvzWgVYOrRCD3dChIoueE"
params1= 'EB444973714E4A40876CE66BE45D5930'
params2 = 'B5A8904209931867'

# 调用js里的函数
rs = ctx.call('get_rs',t,params1,params2)
print(rs)

老师我运行这个py文件报错是：

Traceback (most recent call last):

  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\lib\threading.py", line 1016, in _bootstrap_inner

    self.run()

  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\lib\threading.py", line 953, in run

    self._target(*self._args, **self._kwargs)

  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\lib\subprocess.py", line 1515, in _readerthread

    buffer.append(fh.read())

UnicodeDecodeError: 'gbk' codec can't decode byte 0x81 in position 189: illegal multibyte sequence

Traceback (most recent call last):

  File "d:\spider\spider_code\爬虫与反爬\16_js破解.py", line 16, in <module>

    rs = ctx.call('get_rs',t,params1,params2)

  File "C:\Users\zzzzzzzbw\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\execjs\_abstract_runtime_context.py", line 37, in call

    return self._call(name, *args)

  File "C:\Users\zzzzzzzbw\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\execjs\_external_runtime.py", line 92, in _call

    return self._eval("{identifier}.apply(this, {args})".format(identifier=identifier, args=args))

  File "C:\Users\zzzzzzzbw\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\execjs\_external_runtime.py", line 78, in _eval

    return self.exec_(code)

  File "C:\Users\zzzzzzzbw\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\execjs\_abstract_runtime_context.py", line 18, in exec_

    return self._exec_(source)

  File "C:\Users\zzzzzzzbw\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\execjs\_external_runtime.py", line 88, in _exec_

    return self._extract_result(output)

  File "C:\Users\zzzzzzzbw\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\execjs\_external_runtime.py", line 156, in _extract_result

    output = output.replace("\r\n", "\n").replace("\r", "\n")

AttributeError: 'NoneType' object has no attribute 'replace'

为什么会出现这种情况？

说了个啥0.0，完全没懂他讲那个动态页面和静态页面的逻辑，根本没讲明白啊，感觉就是想到啥说啥

请问报错：无法在json类型的选择器上使用xpath，是什么意思？

调试模式自己就停止了，而且爬取不到任何数据

用xpath helper可以找到, 但是在scrapy中的xpath就找不到什么情况, 老师视频里也是这个情况——网页我改过。xpath 直接复制的，但是还是出不来数据

老师就是网页没有爬取完全，程序就结束了，然后我加了隐式等待和调用js，但是网页还是没有爬取完全，是不是我的逻辑方面有错误，爬取时，还会有那个提示弹窗，必须手动点掉才开始爬取，怎么在代码里就给他跳过

from selenium.webdriver.chrome.service import Service

from selenium.webdriver import Chrome

from lxml import etree

from selenium.webdriver.common.by import By

from time import sleep

def spider_huya():

    #创建驱动

    service=Service('./chromedriver.exe')

    #创建浏览器

    driver=Chrome(service=service)

    #访问网页

    driver.get('https://www.huya.com/g/lol')

    count=1

    while True:

        print('获取了第{%d}页' % count)

        count +=1

        #拖动滚动条到底部

        js='document.documentElement.scrollTop=1000'

        driver.execute_script(js)

         #设置隐式等待

        driver.implicitly_wait(20)

        #创建etree对象

        e=etree.HTML(driver.page_source)

        #提取数据

        names=e.xpath('//i[@class="nick"]/text()')

        persons=e.xpath('//i[@class="js-num"]/text()')

        sleep(3)

        #打印数据

        #for n,p in zip(names,persons):

             #print(f'主播名:{n} 人气:{p}')

        #获取选项卡，点击

        if driver.page_source.find('laypage_next') ==-1:

            break

        next_btn=driver.find_element(By.XPATH,'//a[@class="laypage_next"]')

        next_btn.click()

        sleep(3)

    driver.quit()

if  __name__=='__main__':

    spider_huya()

这个怎么解决呀

import requests

from fake_useragent import UserAgent

from lxml import etree

#找到url地址

url='https://www.zongheng.com/rank?nav=recommend&rankType=6'

#设置请求头

headers={'User-Agent':UserAgent().edge}

#发送请求

resp=requests.get(url,headers=headers)

#解析数据

e=etree.HTML(resp.text)

#提取数据

names=e.xpath('//div[@class="rank-modules-works--main-item-title ellipsis-two-lines global-hover"]/span/text()')

authors=e.xpath('//div[@class="rank-modules-works--main-item-author ellipsis"]/span[1]/text()')

#打印数据

for n,a in zip(names,authors):

    print(n,a)

老师代码运行之后，没有出结果，2个表达式在浏览器上面试出了结果

为什么我上面显示的origin地址，我自己去ping这个地址的时候ping不通呢