代码:
import requests
from fake_useragent import UserAgent
from lxml import etree
from time import sleep
url="https://www.lagou.com/zhaopin/2/?filterOption=3&sid=021dfdfa56f24b11810b1f18e04902eb"
headers={"User-Agent":UserAgent().chrome}
sleep(2)
resp=requests.get(url,headers=headers)
resp.encoding="utf-8"
html=resp.text
print(html)
e=etree.HTML(html)
a=e.xpath('//div[@class="p_top"]//h3/text()')
for i in a:
print(i)运行结果:
body {
margin: 0;
width: 100%;
height: 100%;
}
@keyframes loadingAnimation {
0% {
transform: translate3d(0, 0, 0);
}
50% {
transform: translate3d(0, -10px, 0);
}
}
.loading-info {
text-align: center;
height: 100%;
position: relative;
background: #fff;
top: 50%;
margin-top: -37px;
}
.loading-info .animation-word {
width: 100%;
}
.loading-info .animation-word p {
margin-top: 10px;
color: #9fa3b0;
}
.animation-word .component-l,
.animation-word .component-a,
.animation-word .component-g,
.animation-word .component-o,
.animation-word .component-u {
display: inline-block;
width: 40px;
height: 42px;
line-height: 42px;
font-family: Helvetica Neue, Helvetica, Arial, Hiragino Sans GB, Hiragino Sans GB W3, Microsoft YaHei UI, Microsoft YaHei, WenQuanYi Micro Hei, sans-serif;
font-weight: bolder;
font-size: 40px;
color: #eceef2;
vertical-align: top;
-webkit-animation-fill-mode: both;
animation-fill-mode: both;
-webkit-animation: loadingAnimation 0.6s infinite linear alternate;
-moz-animation: loadingAnimation 0.6s infinite linear alternate;
animation: loadingAnimation 0.6s infinite linear alternate;
}
.animation-word .component-a {
-webkit-animation-delay: 0.1s;
-moz-animation-delay: 0.1s;
animation-delay: 0.1s;
}
.animation-word .component-g {
-webkit-animation-delay: 0.2s;
-moz-animation-delay: 0.2s;
animation-delay: 0.2s;
}
.animation-word .component-o {
-webkit-animation-delay: 0.3s;
-moz-animation-delay: 0.3s;
animation-delay: 0.3s;
}
.animation-word .component-u {
-webkit-animation-delay: 0.4s;
-moz-animation-delay: 0.4s;
animation-delay: 0.4s;
}
</style>
</head>
<body>
<div class="loading-info">
<div class="info-inner">
<div class="animation-word">
<span class="component-l">L</span>
<span class="component-a">a</span>
<span class="component-g">g</span>
<span class="component-o">o</span>
<span class="component-u">u</span>
<p class="gray">正在加载中...</p>
</div>
</div>
</div>
<script>
var securityPageName = "securityCheck";
!function () {
function e(c) {
var l, m, n, o, p, q, r, e = function () {
var a = location.hostname;
return "localhost" === a || /^(\d+\.){3}\d+$/.test(a) ? a : "." + a.split(".").slice(-2).join(".")
}(),
f = function (a, b) {
var f = document.createElement("script");
f.setAttribute("type", "text/javascript"), f.setAttribute("charset", "UTF-8"), f.onload = f.onreadystatechange = function () {
d && "loaded" != this.readyState && "complete" != this.readyState || b()
}, f.setAttribute("src", a), "IFRAME" != c.tagName ? c.appendChild(f) : c.contentDocument ? c.contentDocument.body ? c.contentDocument.body.appendChild(f) : c.contentDocument.documentElement.appendChild(f) : c.document && (c.document.body ? c.document.body.appendChild(f) : c.document.documentElement.appendChild(f))
},
g = function (a) {
var b = new RegExp("(^|&)" + a + "=([^&]*)(&|$)"),
c = window.location.search.substr(1).match(b);
return null != c ? unescape(c[2]) : null
},
h = {
get: function (a) {
var b, c = new RegExp("(^| )" + a + "=([^;]*)(;|$)");
return (b = document.cookie.match(c)) ? unescape(b[2]) : null
},
set: function (a, b, c, d, e) {
var g, f = a + "=" + encodeURIComponent(b);
c && (g = new Date(c).toGMTString(), f += ";expires=" + g), f = d ? f + ";domain=" + d : f, f = e ? f + ";path=" + e : f, document.cookie = f
}
},
i = function (a) {
if (a) {
window.location.replace(a)
}
},
j = function (a, c) {
c || a.indexOf("security-check.html") > -1 ? i(c) : i(a);
};
window.location.href, l = g("seed") || "", m = g("ts"), n = g("name"),
o = g("callbackUrl"),
p = g("srcReferer") || "", "null" !== n && l && n && o, l && m && n && (f("dist/" + n + ".js", function () {
var n, a = (new Date).getTime() + 1728e5,
d = "",
f = {},
g = window.gt || c.contentWindow.gt;
try {
d = (new g()).a();
} catch (k) { console.log(k) }
if (d) {
(h.set("__lg_stoken__", d, a, e, "/"), j(p, o))
}
}))
}
function j(a) {
if (!f && !g && document.addEventListener) return document.addEventListener("DOMContentLoaded", a, !1);
if (!(h.push(a) > 1))
if (f) ! function () {
try {
document.documentElement.doScroll("left"), i()
} catch (a) {
setTimeout(arguments.callee, 0)
}
}();
else if (g) var b = setInterval(function () {
/^(loaded|complete)$/.test(document.readyState) && (clearInterval(b), i())
}, 0)
}
var d, f, g, h, i, a = 0,
b = (new Date).getTime(),
c = window.navigator.userAgent;
c.indexOf("MSIE ") > -1 && (d = !0),
f = !(!window.attachEvent || window.opera),
g = /webkit\/(\d+)/i.test(navigator.userAgent) && RegExp.$1 < 525,
h = [],
i = function () {
for (var a = 0; a < h.length; a++) h[a]()
};
j(function () {
var b, a = window.navigator.userAgent.toLowerCase();
return "micromessenger" !== a.match(/micromessenger/i) || "wkwebview" == a.match(/wkwebview/i) ?
(e(document.getElementsByTagName("head").item(0)), void 0)
:
(b = document.createElement("iframe"), b.style.height = 0, b.style.width = 0, b.style.margin = 0, b.style.padding = 0, b.style.border = "0 none", b.name = "tokenframe", b.src = "about:blank", b.attachEvent ? b.attachEvent("onload", function () {
e(b)
}) : b.onload = function () {
e(b)
}, (document.body || document.documentElement).appendChild(b), void 0)
})
}();
</script>
<script type="text/javascript" crossorigin="anonymous" src="https://www.lagou.com/upload/oss.js?v=1010"></script></body>
</html>
老师请问一下，为什么我使用requests.get方法爬取拉勾网的时候，返回回来的HTML代码和使用f12查看的源代码不一致?对于这个问题当我使用selenium库爬取的时候就不会出现，请问老师在爬取拉勾网的时候是不是必须使用selenium库进行爬取呢?如果我想用requests.get方法来爬取拉勾网，有没有什么办法可以解决返回的HTML和源码不一致的问题?
京公网安备 11011402011233号