import urllib3

http = urllib3.PoolManager()

rq = http.request('GET',url='http://www.pythonscraping.com/pages/page3.html')

print('服务器响应码：', rq.status)

print('响应实体：', rq.data)

import urllib3

http = urllib3.PoolManager()

head = {'User-Agent':'Window NT 6.1;Win64; x64'}

http.request('GET',url='http://www.pythonscraping.com/pages/page3.html',headers=head)

http.request('GET',url='',headers=head，timeout=3.0)

#超过 3s 的话超时终止

http.request('GET',url='http://www.pythonscraping.com/pages/page3.html',headers=head，timeout=urllib3.Timeout(connect=1.0,read=2.0))

#链接超过 1s, 读取超过 2s 终止

import urllib3

http = urllib3.PoolManager(timeout=4.0)

head = {'User-Agent':'Window NT 6.1;Win64; x64'}

http.request('GET',url='http://www.pythonscraping.com/pages/page3.html',headers=head)

#超过 4s 超时

import urllib3

http = urllib3.PoolManager(timeout=4.0,retries=10)

head = {'User-Agent':'Window NT 6.1;Win64; x64'}

http.request('GET',url='http://www.pythonscraping.com/pages/page3.html',headers=head)

#超过 4s 超时 重试 10 次

import urllib3

#发送请求实例

http = urllib3.PoolManager()

#网址

url = 'http://www.pythonscraping.com/pages/page3.html'

#请求头

head = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'}

#超时时间

tm = urllib3.Timeout(connect=1.0,read=3.0)

#重试次数和重定向次数设置，生成请求

rq = http.request('GET',url=url,headers=head,timeout=tm,redirect=4)

print('服务器响应码：', rq.status)

print('响应实体：', rq.data.decode('utf-8'))

import chardet

chardet.detect(rq2.content)

import requests

import chardet

url = 'http://www.pythonscraping.com/pages/page3.html'

head={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36 Edg/88.0.705.56'}

rq2 = requests.get(url,headers=head,timeout=2.0)

rq2.encoding = chardet.detect(rq2.content)['encoding']

print('实体：',rq2.content)

import re

string = '1. A small sentence - 2.Anthoer tiny sentence. '

print('re.findall:',re.findall('sentence',string))

print('re.search:',re.search('sentence',string))

print('re.match:',re.match('sentence',string))

print('re.match:',re.match('1. A small sentence',string))

print('re.sub:',re.sub('small','large',string)) 

print('re.sub:',re.sub('small','',string))

string = '1. A small sentence - 2.Anthoer tiny sentence. '

re.findall('A.',string)

string = 'small smell smll smsmll sm3ll sm.ll sm?ll sm\nll sm\tll'

print('re.findall:',re.findall('sm.ll',string))

print('re.findall:',re.findall('sm[asdfg]ll',string))

print('re.findall:',re.findall('sm[a-zA-Z0-9]ll',string))

print('re.findall:',re.findall('sm\.ll',string))

print('re.findall:',re.findall('sm[.?]ll',string))

re.findall: ['small', 'smell', 'sm3ll', 'sm.ll', 'sm?ll', 'sm\tll']

re.findall: ['small']

re.findall: ['small', 'smell', 'sm3ll']

re.findall: ['sm.ll']

re.findall: ['sm.ll', 'sm?ll']

print('re.findall:',re.findall('sm..ll',string))

print('re.findall:',re.findall('sm.{2}ll',string))

print('re.findall:',re.findall('sm.{1,2}ll',string))

print('re.findall:',re.findall('sm.{1,}ll',string))

print('re.findall:',re.findall('sm.?ll',string)) # {0,1}

print('re.findall:',re.findall('sm.+ll',string)) # {0,}

print('re.findall:',re.findall('sm.*ll',string)) # {1,}

import pandas as pd

rawdata = '555-1239Moe Szyslak(636) 555-0113Burns, C.Montgomery555-6542Rev. Timothy Lovejoy555 8904Ned Flanders636-555-3226Simpson,Homer5553642Dr. Julius Hibbert'

names = re.findall('[A-Z][A-Za-z,. ]*',rawdata)

print(names)

number = re.findall('\(?[0-9]{0,3}\)?[ \-]?[0-9]{3}[ \-]?[0-9]{4}',rawdata)

print(number)

pd.DataFrame({'Name':names,'TelPhone':number})

import requests

from lxml import etree

url = "https://www.zhihu.com/hot"

hd = { 'Cookie':'你的Cookie', #'Host':'www.zhihu.com',

        'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

response = requests.get(url, headers=hd)

html_str = response.content.decode()

html = etree.HTML(html_str)

title = html.xpath("//section[@class='HotItem']/div[@class='HotItem-content']/a/@title")

href = html.xpath("//section[@class='HotItem']/div[@class='HotItem-content']/a/@href")

f = open("zhihu.txt",'r+')

for i in range(1,41):

    print(i,'.'+title[i])

    print('链接：'+href[i])

    print('-'*50)

    f.write(str(i)+'.'+title[i]+'\n')

    f.write('链接：'+href[i]+'\n')

    f.write('-'*50+'\n')

f.close()

import requests

from lxml import etree

import json

#上面代码略

with open('zhihu.json','w') as j:

    json.dump({'title':title,'hrefL':href},j,ensure_ascii=False)