初识python之 APP store排行榜 蜘蛛抓取(二)

#-*- coding: utf-8 -*-
import urllib2
import urllib
import re
import time
import MySQLdb
import time,datetime
#from datetime import date

#----------- APP store 排行榜 -----------
class Spider_Model:

    def __init__(self):
        self.page = 1
        self.pages = []
        self.enable = False

    def startWork(self,url,tabName):
        nowtime = int(time.time())
        content = self.GetCon(url)
        oneItems =  self.Match(content) #匹配一级参数
        time.sleep(1)
        for index,item in enumerate(oneItems):
            content_two = self.GetCon(item[1])
            twoItems = self.Match_two(content_two)
            oneItems[index].append([twoItems[0],twoItems[1]])
            if oneItems[index][6][0] == ‘0‘:
                fabutime = ‘0‘
            else:
                fabutime=int(time.mktime(time.strptime(oneItems[index][6][0].strip(),‘%Y年%m月%d日‘)))

            sql = "INSERT INTO "+tabName+"(`rank`,`detailurl`,`logo`,`name`,`type`,`appid`,`appstoretime`,`compatible`,`ctime`) values(%s,%s,%s,%s,%s,%s,%s,%s,%s)"%(‘"‘+oneItems[index][0]+‘"‘,‘"‘+oneItems[index][1]+‘"‘,‘"‘+oneItems[index][2]+‘"‘,‘"‘+oneItems[index][3]+‘"‘,‘"‘+oneItems[index][4]+‘"‘,‘"‘+oneItems[index][5]+‘"‘,fabutime,‘"‘+oneItems[index][6][1]+‘"‘,nowtime)
            self.contentDb(sql)
            time.sleep(1)

    def GetCon(self,url):
        myUrl = url
        headers = {‘User-Agent‘: ‘Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11‘,‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘}
        //网站禁止爬虫解决方法加上上面的代码 模拟浏览器访问  
     req = urllib2.Request(myUrl, headers = headers) global myResponse try: myResponse = urllib2.urlopen(req) except urllib2.HTTPError, e: print e.fp.read()
     //异常处理必须加 否则就算模拟了浏览器 也会返回 403 原因不知道...... myPage = myResponse.read() #encode的作用是将unicode编码转换成其他编码的字符串 #decode的作用是将其他编码的字符串转换成unicode编码 #unicodePage = myPage.decode(‘utf-8‘).encode(‘gbk‘,‘ignore‘) #unicodePage = myPage.decode(‘utf-8‘,‘ignore‘) return myPage def Match(self,con): # 找出所有class="content"的div标记 #re.S是任意匹配模式,也就是.可以匹配换行符 pattenA = re.compile(r‘<section class="section apps grid">(.*?)</section>‘,re.U|re.S) pattenB = re.compile(r‘<li><strong>(.*?).</strong><a href="(.*?)".*?><img src="(.*?)".*?></a><h3><a.*?>(.*?)</a></h3><h4><a.*?>(.*?)</a></h4><a.*?>.*?</a></li>‘,re.U|re.S) match = re.findall(pattenA,con) myItems = re.findall(pattenB,match[0]) items = [] for item in myItems: items.append([item[0].replace("\n",""),item[1].replace("\n",""),item[2].replace("\n",""),(item[3].replace("\n","")).split(‘-‘)[0],item[4].replace("\n",""),(item[1].split(‘id‘)[1]).split(‘?‘)[0]]) return items def Match_two(self,con): pattenTwoA = re.compile(r‘<li.*?class="release-date"><span.*?>.*?</span>(.*?)</li>‘,re.U|re.S) pattenTwoB = re.compile(r‘<span.*?class="app-requirements">.*?</span>(.*?)</p>‘,re.U|re.S) matchTwoA = self.is_empty(re.findall(pattenTwoA,con)) matchTwoB = self.is_empty(re.findall(pattenTwoB,con)) itemsTwo = [matchTwoA,matchTwoB] return itemsTwo def is_empty(self,param): if len(param): res = param[0] else: res = ‘0‘ return res def contentDb(self,sql): try: conn = MySQLdb.connect(host="主机", user="用户", passwd="密码", db="表名",charset=‘utf8‘) cur = conn.cursor() result = cur.execute(sql) conn.commit() except MySQLdb.Error,e: print "Mysql Error %d: %s" %(e.args[0],e.args[1]) addArr = [["http://www.apple.com/jp/itunes/charts/free-apps/",‘cg_jp_free‘], ["http://www.apple.com/jp/itunes/charts/paid-apps/",‘cg_jp_paid‘]] myModel = Spider_Model() for val in addArr: myModel.startWork(val[0],val[1])

 

初识Python 代码写的有点烂,自制罪孽深重......

python版本:2.7.5  测试环境:Linux、Windows

望高手拍砖 带我一起装逼!一起飞!

郑重声明:本站内容如果来自互联网及其他传播媒体,其版权均属原媒体及文章作者所有。转载目的在于传递更多信息及用于网络分享,并不代表本站赞同其观点和对其真实性负责,也不构成任何其他建议。