import scrapy
import re
from chaojiying_Python.chaojiying import get_code

class Login1Spider(scrapy.Spider):
    name = 'login1'
    allowed_domains = ['ganji.com']
    start_urls = ['https://passport.ganji.com/login.php']
    def parse(self, response):
        img_url = 'https://passport.ganji.com/ajax.php?dir=captcha&module=login_captcha'
        hash_code =re.search(r'"__hash__":"(.+)"',response.text).group(1)
        yield scrapy.Request(img_url,callback=self.do_fromdata,meta={'hash_code':hash_code})
    def do_fromdata(self,response):
        with open('code.jpg','wb') as f:
            f.write(response.body)
        #code = get_code('code.jpg')
        code = input("请输入验证码：")
        hash_code = response.request.meta['hash_code']
        data = {
            'username': '17030240219',
            'password': '123456qaz',
            'setcookie': '14',
            'checkCode':code,
            'next': '/ user / register_success.php?username=17030240219&next=%2F',
            'source':'passport',
            '__hash__':hash_code
        }
        login_url = 'https://passport.ganji.com/login.php'
        yield scrapy.FormRequest(login_url,method='POST',formdata=data,callback=self.after_login)
        #print(response.text)
    def after_login(self,response):
        print(response.text)

为什么登录时显示无效数组长度，这个怎么弄

from fake_useragent import UserAgent
import requests
from lxml import etree


# 发送请求类
class Downloader():
    def download(self,url):
        print(url)
        headers = {'User-Agent':UserAgent().chrome}
        resp = requests.get(url, headers = headers)
        if resp.status_code == 200:
            resp.encoding = 'utf-8'
            return resp.text

# 数据解析类
class Parser():
    def parse(self, html):
        e = etree.HTML(html)
        contents =[div.xpath('string(.)') for div in e.xpath('//div[@class="content"]')]
        urls = ['https://www.qiushibaike.com{}'.format(url) for url in e.xpath('//ul[@class="pagination"]/li/a/@href')]
        return contents, urls

# 数据保存类
class DataOutPut():
    def save(self, datum):
        with open('duanzi1.txt', 'a', encoding='utf-8') as f:
            for data in datum:
                f.write(data+'\n')


# URL管理器
class URLManager():
    def __init__(self):
        self.new_url = set()
        self.old_url = set()
    # 加入一个Url
    def add_new_url(self,url):
        if url is not None and url !='' and url not in self.old_url:
            self.new_url.add(url)
    # 加入多个Url
    def add_new_urls(self,urls):
        for url in urls:
            self.add_new_url(url)
    # 获取一个Url
    def get_new_url(self):
        url = self.new_url.pop()
        self.old_url.add(url)
        return url
    # 获取还有多少个Url要爬取
    def get_new_url_size(self):
        return len(self.new_url)
    # 获取是否还有Url要爬取
    def is_have_new_url(self):
        return self.get_new_url_size() > 0

# 调度器
class Scheduler():
    def __init__(self):
        self.downloader = Downloader()
        self.parser = Parser()
        self.data_out_put = DataOutPut()
        self.url_manager = URLManager()
    def start(self, url):
        self.url_manager.add_new_url(url)
        url = self.url_manager.get_new_url()
        html = self.downloader.download(url)
        datum, urls = self.parser.parse(html)
        self.data_out_put.save(datum)
        self.url_manager.add_new_url(urls)


if __name__ == '__main__':
    scheduler = Scheduler()
    url = 'https://www.qiushibaike.com/text/'
    scheduler.start(url)

老师您好！我这里出现这样的BUG请问怎么结局呀！

课程里的代码运行后报错

老师为啥没有log文件啊

class XinchengRoomsSpider(scrapy.Spider):
    name = 'xincheng_rooms'
    allowed_domains = ['anjuke.com']
    start_urls = ['https://xa.anjuke.com/sale/xinchengqu/p{}/'.format(num) for num in range(1,11)]

    def parse(self, response):
        #new_url = re.search('href="(https://xa.anjuke.com/prop/view/.+)?.+"',response.text).group(0)
        new_url = response.xpath('//a[@class="houseListTitle"]/@href').extract()
        for url in new_url:
            yield scrapy.Request(url,callback=self.parse_info)

老师我这个是安居客的    new_url用xpath_helper都可以找到  但是调试的时候返回的都是空

老师帮忙看下这个报错问题，我查了下百度尝试在headers中加

"Content-type":"application/json;charset=utf-8"
然后直接报错400

老师，已导入好UserAgent, 但为什么总是报错呢

老师那个'.'join的用法怎么用的？？

老师，为啥有些是这样的符号？如何解决

<title>ç¾åº¦å®å¨éªè¯</title>

以下更改之后可以保存到相应英雄名的文件夹:

import requestsfrom fake_useragent import UserAgentfrom urllib.request import urlretrievefrom os import makedirs, pathfrom time import sleepdef ensure_directory_exists(directory):    """确保目录存在，如果不存在则创建"""    if not path.exists(directory):        makedirs(directory)def get_hero_img(hero_id=1, hero_name=""):    url = f'https://game.gtimg.cn/images/lol/act/img/js/hero/{hero_id}.js'    headers = {'User-Agent': UserAgent().chrome}    resp = requests.get(url, headers=headers)        hero_dir = f'./_反爬和反反爬/imgs/lol/{hero_name.replace(" ", "_")}'    ensure_directory_exists(hero_dir)    for sk in resp.json().get('skins', []):        if sk.get("mainImg"):            print(f'皮肤名:{sk.get("name")}    皮肤下载地址: {sk.get("mainImg")}')            img_filename = path.join(hero_dir, f'{sk.get("name").replace(" ", "_")}.jpg')            urlretrieve(sk.get("mainImg"), filename=img_filename)            sleep(1)def get_hero_list():    url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js'    headers = {'User-Agent': UserAgent().chrome}    resp = requests.get(url, headers=headers)        for h in resp.json().get('hero', []):        hero_id = h.get('heroId')        hero_name = h.get('name')        yield (hero_id, hero_name)if __name__ == '__main__':    for hero_id, hero_name in get_hero_list():        print(f'=========正在下载的： 英雄名:{hero_name}')        get_hero_img(hero_id, hero_name)        print(f'=====下载完成：英雄:{hero_name}')

老师post请求百度的url，只出现一行结果，是什么情况啊，怎么解决啊？

发现有的皮肤名字是带着\\的，保存不到文件夹中，下面是改进的版本：

import requests

from fake_useragent import UserAgent

from urllib.request import urlretrieve

from os import makedirs, path

from time import sleep

def ensure_directory_exists(directory):

    if not path.exists(directory):

        makedirs(directory)

def sanitize_filename(filename):

    # 将文件名中的特殊字符替换为下划线

    return filename.replace(" ", "_").replace("/", "_").replace("\\", "_")

def get_hero_img(hero_id=1, hero_name=""):

    url = f'https://game.gtimg.cn/images/lol/act/img/js/hero/{hero_id}.js'

    headers = {'User-Agent': UserAgent().chrome}

    resp = requests.get(url, headers=headers)

    hero_dir = path.join('./_反爬和反反爬/imgs/lolskin', sanitize_filename(hero_name))

    ensure_directory_exists(hero_dir)

    for sk in resp.json().get('skins', []):

        if sk.get("mainImg"):

            print(f'皮肤名:{sk.get("name")}    皮肤下载地址: {sk.get("mainImg")}')

            img_filename = sanitize_filename(sk.get("name")) + '.jpg'

            img_path = path.join(hero_dir, img_filename)

            urlretrieve(sk.get("mainImg"), filename=img_path)

            sleep(0.5)

# ...（其余代码保持不变）

def get_hero_list():

    url = 'https://game.gtimg.cn/images/lol/act/img/js/heroList/hero_list.js'

    headers = {'User-Agent': UserAgent().chrome}

    resp = requests.get(url, headers=headers)

    for h in resp.json().get('hero', []):

        hero_id = h.get('heroId')

        hero_name = h.get('name')

        yield (hero_id, hero_name)

if __name__ == '__main__':

    for hero_id, hero_name in get_hero_list():

        print(f'=========正在下载的： 英雄名:{hero_name}')

        get_hero_img(hero_id, hero_name)

        print(f'=====下载完成：英雄:{hero_name}')

为什么按照pyquery内容学习，爬取出来的是这样的乱码？

from urllib.request import Request,build_opener

from fake_useragent import UserAgent

from urllib.parse import urlencode

from urllib.request import HTTPCookieProcessor

login_url = 'https://www.bilibili.com/page-proxy/game-nav.html'

form_data = {

    "user":"15212123250",

    "password":"2742558764"

}

headers = {

    "User-Agent":UserAgent().random,

}

req = Request(login_url, headers=headers, data=urlencode(form_data))

handler = HTTPCookieProcessor()

opener = build_opener(handler)

resp = opener.open(req)

info_url = "https://www.bilibili.com/"

req = Request(info_url, headers=headers)

resp = opener.open(req)

print(resp.read().decode())

老师，我在使用多线程爬取文件的时候，会出现文件写入不全，然后我在写入文件的的时候加入锁，但是发现还写入不全，您帮我看一下！

from threading import Thread,Lock
import requests
from lxml import etree
from fake_useragent import UserAgent
from queue import Queue
class Spider(Thread):
    def __init__(self,url_queue,lock):
        Thread.__init__(self)
        self.url_queue = url_queue
        self.lock = lock

    def run(self):
        while not self.url_queue.empty():
            url = self.url_queue.get()
            print(url)
            headers = {'User-Agent':UserAgent().chrome}
            resp = requests.get(url,headers=headers)
            e = etree.HTML(resp.text)
            contents = [div.xpath('string(.)').strip() for div in e.xpath('//div[@class="content"]')]
            #加入锁
            self.lock.acquire()
            with open('qiushi.text', 'a', encoding='utf-8')as f:
                for content in contents:
                    f.write(content+'\n')
            self.lock.release()
if __name__ == '__main__':
    base_url = 'https://www.qiushibaike.com/text/page/{}/'
    lock = Lock()
    url_queue = Queue()
    for num in range(1,14):
        url_queue.put(base_url.format(num))
    for i in range(6):
        spider = Spider(url_queue,lock)
        spider.start()