PyQt4正则爬虫工具，多多指教！- Community

PyQt4正则爬虫工具，多多指教！

社区开发 2129 views · 14 replies ·

李逍遥mx

deepin

2018-03-26 01:46

Author

本帖最后由 lrbmfx 于 2018-3-25 17:50 编辑

之前很喜欢cinnamon桌面，由于不太喜欢系统自带的主题，自带的主题中心由于网络问题经常崩溃，无法更新列表。
闲着没事用PyQt4写了个正则爬虫工具爬取cinnamon官网上的主题。

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Form implementation generated from reading ui file '/home/lrbmx/workspace/MyReProject/main.ui'
#
# Created by: PyQt4 UI code generator 4.11.4
#
# WARNING! All changes made in this file will be lost!
from PyQt4 import QtCore, QtGui
import urllib2,re #导入爬虫模块和正则模快
#↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ 以下代码有Eric自动生成 ↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
try:
_fromUtf8 = QtCore.QString.fromUtf8
except AttributeError:
def _fromUtf8(s):
return s
try:
_encoding = QtGui.QApplication.UnicodeUTF8
def _translate(context, text, disambig):
return QtGui.QApplication.translate(context, text, disambig, _encoding)
except AttributeError:
def _translate(context, text, disambig):
return QtGui.QApplication.translate(context, text, disambig)
class Ui_Dialog(object):
def setupUi(self, Dialog):
Dialog.setObjectName(_fromUtf8("Dialog"))
Dialog.resize(400, 600)
Dialog.setMinimumSize(QtCore.QSize(400, 600))
Dialog.setMaximumSize(QtCore.QSize(400, 600))
Dialog.setSizeGripEnabled(True)
Dialog.setStyleSheet(_fromUtf8("background-color: rgb(255, 255, 255);"))
self.gridLayout = QtGui.QGridLayout(Dialog)
self.gridLayout.setObjectName(_fromUtf8("gridLayout"))
self.textEdit = QtGui.QTextEdit(Dialog)
self.textEdit.setObjectName(_fromUtf8("textEdit"))
self.gridLayout.addWidget(self.textEdit, 4, 0, 1, 3)
self.btn_get = QtGui.QPushButton(Dialog)
self.btn_get.setObjectName(_fromUtf8("btn_get"))
self.gridLayout.addWidget(self.btn_get, 2, 2, 1, 1)
self.label_3 = QtGui.QLabel(Dialog)
self.label_3.setObjectName(_fromUtf8("label_3"))
self.gridLayout.addWidget(self.label_3, 5, 0, 1, 1)
self.label_2 = QtGui.QLabel(Dialog)
self.label_2.setObjectName(_fromUtf8("label_2"))
self.gridLayout.addWidget(self.label_2, 3, 0, 1, 1)
self.label = QtGui.QLabel(Dialog)
self.label.setObjectName(_fromUtf8("label"))
self.gridLayout.addWidget(self.label, 2, 0, 1, 1)
self.lineEdit = QtGui.QLineEdit(Dialog)
self.lineEdit.setObjectName(_fromUtf8("lineEdit"))
self.gridLayout.addWidget(self.lineEdit, 2, 1, 1, 1)
self.label_4 = QtGui.QLabel(Dialog)
self.label_4.setObjectName(_fromUtf8("label_4"))
self.gridLayout.addWidget(self.label_4, 6, 0, 1, 1)
self.textEdit_2 = QtGui.QTextEdit(Dialog)
self.textEdit_2.setObjectName(_fromUtf8("textEdit_2"))
self.gridLayout.addWidget(self.textEdit_2, 7, 0, 1, 3)
self.lineEdit_2 = QtGui.QLineEdit(Dialog)
self.lineEdit_2.setObjectName(_fromUtf8("lineEdit_2"))
self.gridLayout.addWidget(self.lineEdit_2, 5, 1, 1, 1)
self.btn_run = QtGui.QPushButton(Dialog)
self.btn_run.setObjectName(_fromUtf8("btn_run"))
self.gridLayout.addWidget(self.btn_run, 5, 2, 1, 1)
self.retranslateUi(Dialog)
QtCore.QObject.connect(self.btn_get, QtCore.SIGNAL(_fromUtf8("clicked()")), self.get_url) # ← ← ← 这里要稍微修改一下
QtCore.QObject.connect(self.btn_run, QtCore.SIGNAL(_fromUtf8("clicked()")), self.run_re) # ← ← ← 这里要稍微修改一下
def retranslateUi(self, Dialog):
Dialog.setWindowTitle(_translate("Dialog", "正则采集分析器", None))
self.btn_get.setText(_translate("Dialog", "采集", None))
self.label_3.setText(_translate("Dialog", "正则表达式：", None))
self.label_2.setText(_translate("Dialog", "获取到的源代码：", None))
self.label.setText(_translate("Dialog", "要采集地址：", None))
self.label_4.setText(_translate("Dialog", "分析输出：", None))
self.btn_run.setText(_translate("Dialog", "分析", None))
#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ 以下代码有Eric自动生成 ↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑
def get_url(self): #爬取网页源代码
url = self.lineEdit.text()
if len(url) > 0: #判断是否输入要爬取的网址
#开始爬取网页
response = urllib2.urlopen(unicode(url.toUtf8(), 'utf-8', 'ignore'))
code = response.read()
self.textEdit.setPlainText(code.decode('utf-8')) #把爬取的网页源代码显示在textEdit上
else:
msg = QtGui.QMessageBox #继承一个QMessageBox类
msg(msg.Critical,u"错误",u"未输入要采集的网址\n请输入要采集的网址！").exec_() #弹出信息框
self
def run_re(self):
code = self.textEdit.toPlainText() #获取textEdit的内容
n_code = unicode(code.toUtf8(), 'utf-8', 'ignore').encode("utf-8") #转换获取到的textEdit的数据类型
re_code = unicode(self.lineEdit_2.text().toUtf8(), 'utf-8', 'ignore').encode('utf-8') #转换获取到的tlineEdit_2的数据类型
pattern = re.compile(re_code) #初始化正则表达式
match = pattern.findall(n_code) #正则查找
print match
t ='' #初始化一个t变量用于保存匹配到的数据
#拼接匹配数据
for i in match:
t += i + "\n"
print t
t += "共匹配到%s条数据……" %len(match) #添加统计信息
self.textEdit_2.setPlainText(t.decode('utf-8')) #在textEdit_2上输出结果
if __name__ == "__main__":
import sys
app = QtGui.QApplication(sys.argv)
Dialog = QtGui.QDialog()
ui = Ui_Dialog()
ui.setupUi(Dialog)
Dialog.show()
sys.exit(app.exec_())

Copy the Code

Reply Like 0 Favorite View the author

All Replies

李逍遥mx

deepin

2018-03-26 01:52

喜欢Python的可以交流以下哦

Reply Like 0 View the author

ritter

deepin

2018-03-26 05:06

顶一个！！！！！！！！！！

Reply Like 0 View the author

aksss

deepin

2018-03-26 05:48

python的代码果然很简洁啊。

Reply Like 0 View the author

Guumi

deepin

2018-03-26 06:09

还处于初级初级初级阶段，表示基本看不懂！

Reply Like 0 View the author

海天鹰

deepin

2018-03-26 06:42

本帖最后由 sonichy 于 2018-3-25 23:40 编辑

shell 就可以搞定，我也是刚学的:
#清屏
printf "\033c"
URL=https://cinnamon-spices.linuxmint.com/files/themes/
#创建缓存文件
HTML=`mktemp`
#下载到缓存文件
wget -q $URL -O $HTML
fileURL=($(sed -n '/zip/p' $HTML | sed "s#.*href=\"#$URL#g" | sed 's#">.*##g'))
len=${#fileURL[@]}
for i in "${!fileURL[@]}";
do
echo $i"/"$len ${fileURL[$i]}
wget -q -N ${fileURL[$i]}
done

Reply Like 0 View the author

李逍遥mx

deepin

2018-03-26 07:34

https://bbs.deepin.org/post/154657
python的代码果然很简洁啊。

确实是很简洁

Reply Like 0 View the author

李逍遥mx

deepin

2018-03-26 07:34

https://bbs.deepin.org/post/154657
还处于初级初级初级阶段，表示基本看不懂！

勤学多练，很容易看懂的

Reply Like 0 View the author

李逍遥mx

deepin

2018-03-26 07:35

https://bbs.deepin.org/post/154657
我觉得 shell 就可以搞定

shell确实可以搞定，只是个人比较喜欢Python而已。

Reply Like 0 View the author

fengying6_6

deepin

2018-03-26 21:53

https://bbs.deepin.org/post/154657
shell确实可以搞定，只是个人比较喜欢Python而已。

const request = require('request-promise-native')
const fs = require('fs')
const baseUrl = 'https://cinnamon-spices.linuxmint.com/files/themes/'
request.get(baseUrl).then(html => {
html.match(/(?<=")\w+(-\w+)?\.zip/g).forEach(fileName =>
request.get(`${baseUrl}/${fileName}`).pipe(fs.createWriteStream(`/home/feng/test/${fileName}`))
)
})

Copy the Code

瞎几把写了几行js发现也能搞定，不过好像这个网速有点慢，下到最后就超时了

Reply Like 0 View the author

Guumi

deepin

2018-03-27 05:10

#10

https://bbs.deepin.org/post/154657
瞎几把写了几行js发现也能搞定，不过好像这个网速有点慢，下到最后就超时了 ...

js 其了解，虽然也不懂，他好像收到某些限制，不能单独使用吧？！求教中……

Reply Like 0 View the author

李逍遥mx

deepin

2018-03-27 08:23

#11

https://bbs.deepin.org/post/154657
瞎几把写了几行js发现也能搞定，不过好像这个网速有点慢，下到最后就超时了 ...

其实我写这个关键是想要练习Python的GUI编程而已，
而且正则分析这一块可以独立使用，不仅仅是要做爬虫。
爬取网页源码只是我懒得点击查看源码而已，关键还是正则分析，
做了GUI可以方便随时使用，算是给自己做个简单的小工具吧。。。

Reply Like 0 View the author

李逍遥mx

deepin

2018-03-27 08:29

#12

本帖最后由 lrbmfx 于 2018-3-27 00:30 编辑

https://bbs.deepin.org/post/154657
js 其了解，虽然也不懂，他好像收到某些限制，不能单独使用吧？！求教中…… ...

他这个好像是node.js的

Reply Like 0 View the author

fengying6_6

deepin

2018-03-27 19:11

#13

https://bbs.deepin.org/post/154657
其实我写这个关键是想要练习Python的GUI编程而已，
而且正则分析这一块可以独立使用，不仅仅是要做爬虫。 ...

是时候为社区造轮子了

Reply Like 0 View the author

李逍遥mx

deepin

2018-03-27 20:23

#14

https://bbs.deepin.org/post/154657
是时候为社区造轮子了

正在打算用kivy重写一个

Reply Like 0 View the author

Featured Collection

Change

New Thread

Popular Ranking

Change

Popular Events