网络爬虫获取网站图片
浏览数：67 /
时间：2015年06月09日
转自：http://blog.csdn.net/huangxy10/article/details/8120106
备注：把项目属性中的字符集改成多字节集合?
1 // 网络爬虫.cpp : 定义控制台应用程序的入口点。 2 // 3 4 #include "stdafx.h" 5 /* 6 7 int _tmain(int argc, _TCHAR* argv[]) 8 { 9 return 0; 10 } 11 12 */ 13 14 //#include <Windows.h> 15 #include <string> 16 #include <iostream> 17 #include <fstream> 18 #include <vector> 19 #include "winsock2.h" 20 #include <time.h> 21 #include <queue> 22 #include <hash_set> 23 24 #pragma comment(lib, "ws2_32.lib") 25 using namespace std; 26 27 #define DEFAULT_PAGE_BUF_SIZE 1048576 28 29 queue<string> hrefUrl; 30 hash_set<string> visitedUrl; 31 hash_set<string> visitedImg; 32 int depth=0; 33 int g_ImgCnt=1; 34 35 //解析URL，解析出主机名，资源名 36 bool ParseURL( const string & url, string & host, string & resource){ 37 if ( strlen(url.c_str()) > 2000 ) { 38 return false; 39 } 40 41 const char * pos = strstr( url.c_str(), "http://" ); 42 if( pos==NULL ) pos = url.c_str(); 43 else pos += strlen("http://"); 44 if( strstr( pos, "/")==0 ) 45 return false; 46 char pHost[100]; 47 char pResource[2000]; 48 sscanf( pos, "%[^/]%s", pHost, pResource ); 49 host = pHost; 50 resource = pResource; 51 return true; 52 } 53 54 //使用Get请求，得到响应 55 bool GetHttpResponse( const string & url, char * &response, int &bytesRead ){ 56 string host, resource; 57 if(!ParseURL( url, host, resource )){ 58 cout << "Can not parse the url"<<endl; 59 return false; 60 } 61 62 //建立socket 63 struct hostent * hp= gethostbyname( host.c_str() ); 64 if( hp==NULL ){ 65 cout<< "Can not find host address"<<endl; 66 return false; 67 } 68 69 SOCKET sock = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP); 70 if( sock == -1 || sock == -2 ){ 71 cout << "Can not create sock."<<endl; 72 return false; 73 } 74 75 //建立服务器地址 76 SOCKADDR_IN sa; 77 sa.sin_family = AF_INET; 78 sa.sin_port = htons( 80 ); 79 //char addr[5]; 80 //memcpy( addr, hp->h_addr, 4 ); 81 //sa.sin_addr.s_addr = inet_addr(hp->h_addr); 82 memcpy( &sa.sin_addr, hp->h_addr, 4 ); 83 84 //建立连接 85 if( 0!= connect( sock, (SOCKADDR*)&sa, sizeof(sa) ) ){ 86 cout << "Can not connect: "<< url <<endl; 87 closesocket(sock); 88 return false; 89 }; 90 91 //准备发送数据 92 string request = "GET " + resource + " HTTP/1.1\r\nHost:" + host + "\r\nConnection:Close\r\n\r\n"; 93 94 //发送数据 95 if( SOCKET_ERROR ==send( sock, request.c_str(), request.size(), 0 ) ){ 96 cout << "send error" <<endl; 97 closesocket( sock ); 98 return false; 99 } 100 101 //接收数据 102 int m_nContentLength = DEFAULT_PAGE_BUF_SIZE; 103 char *pageBuf = (char *)malloc(m_nContentLength); 104 memset(pageBuf, 0, m_nContentLength); 105 106 bytesRead = 0; 107 int ret = 1; 108 cout <<"Read: "; 109 while(ret > 0){ 110 ret = recv(sock, pageBuf + bytesRead, m_nContentLength - bytesRead, 0); 111 112 if(ret > 0) 113 { 114 bytesRead += ret; 115 } 116 117 if( m_nContentLength - bytesRead<100){ 118 cout << "\nRealloc memorry"<<endl; 119 m_nContentLength *=2; 120 pageBuf = (char*)realloc( pageBuf, m_nContentLength); //重新分配内存 121 } 122 cout << ret <<" "; 123 } 124 cout <<endl; 125 126 pageBuf[bytesRead] = ‘\0‘; 127 response = pageBuf; 128 closesocket( sock ); 129 return true; 130 //cout<< response <<endl; 131 } 132 133 //提取所有的URL以及图片URL 134 void HTMLParse ( string & htmlResponse, vector<string> & imgurls, const string & host ){ 135 //找所有连接，加入queue中 136 const char *p= htmlResponse.c_str(); 137 char *tag="href=\""; 138 const char *pos = strstr( p, tag ); 139 ofstream ofile("url.txt", ios::app); 140 while( pos ){ 141 pos +=strlen(tag); 142 const char * nextQ = strstr( pos, "\"" ); 143 if( nextQ ){ 144 char * url = new char[ nextQ-pos+1 ]; 145 //char url[100]; //固定大小的会发生缓冲区溢出的危险 146 sscanf( pos, "%[^\"]", url); 147 string surl = url; // 转换成string类型，可以自动释放内存 148 if( visitedUrl.find( surl ) == visitedUrl.end() ){ 149 visitedUrl.insert( surl ); 150 ofile << surl<<endl; 151 hrefUrl.push( surl ); 152 } 153 pos = strstr(pos, tag ); 154 delete [] url; // 释放掉申请的内存 155 } 156 } 157 ofile << endl << endl; 158 ofile.close(); 159 160 tag ="<img "; 161 const char* att1= "src=\""; 162 const char* att2="lazy-src=\""; 163 const char *pos0 = strstr( p, tag ); 164 while( pos0 ){ 165 pos0 += strlen( tag ); 166 const char* pos2 = strstr( pos0, att2 ); 167 if( !pos2 || pos2 > strstr( pos0, ">") ) { 168 pos = strstr( pos0, att1); 169 if(!pos) { 170 pos0 = strstr(att1, tag ); 171 continue; 172 } else { 173 pos = pos + strlen(att1); 174 } 175 } 176 else { 177 pos = pos2 + strlen(att2); 178 } 179 180 const char * nextQ = strstr( pos, "\""); 181 if( nextQ ){ 182 char * url = new char[nextQ-pos+1]; 183 sscanf( pos, "%[^\"]", url); 184 cout << url<<endl; 185 string imgUrl = url; 186 if( visitedImg.find( imgUrl ) == visitedImg.end() ){ 187 visitedImg.insert( imgUrl ); 188 imgurls.push_back( imgUrl ); 189 } 190 pos0 = strstr(pos0, tag ); 191 delete [] url; 192 } 193 } 194 cout << "end of Parse this html"<<endl; 195 } 196 197 //把URL转化为文件名 198 string ToFileName( const string &url ){ 199 string fileName; 200 fileName.resize( url.size()); 201 int k=0; 202 for( int i=0; i<(int)url.size(); i++){ 203 char ch = url[i]; 204 if( ch!=‘\\‘&&ch!=‘/‘&&ch!=‘:‘&&ch!=‘*‘&&ch!=‘?‘&&ch!=‘"‘&&ch!=‘<‘&&ch!=‘>‘&&ch!=‘|‘) 205 fileName[k++]=ch; 206 } 207 return fileName.substr(0,k) + ".txt"; 208 } 209 210 //下载图片到img文件夹 211 void DownLoadImg( vector<string> & imgurls, const string &url ){ 212 213 //生成保存该url下图片的文件夹 214 string foldname = ToFileName( url ); 215 foldname = "./img/"+foldname; 216 if(!CreateDirectory( (LPCSTR)foldname.c_str(),NULL )) 217 cout << "Can not create directory:"<< foldname<<endl; 218 char *image; 219 int byteRead; 220 for( int i=0; i<imgurls.size(); i++){ 221 //判断是否为图片，bmp，jgp，jpeg，gif 222 string str = imgurls[i]; 223 int pos = str.find_last_of("."); 224 if( pos == string::npos ) 225 continue; 226 else{ 227 string ext = str.substr( pos+1, str.size()-pos-1 ); 228 if( ext!="bmp"&& ext!="jpg" && ext!="jpeg"&& ext!="gif"&&ext!="png") 229 continue; 230 } 231 //下载其中的内容 232 if( GetHttpResponse(imgurls[i], image, byteRead)){ 233 if ( strlen(image) ==0 ) { 234 continue; 235 } 236 const char *p=image; 237 const char * pos = strstr(p,"\r\n\r\n")+strlen("\r\n\r\n"); 238 int index = imgurls[i].find_last_of("/"); 239 if( index!=string::npos ){ 240 string imgname = imgurls[i].substr( index , imgurls[i].size() ); 241 ofstream ofile( foldname+imgname, ios::binary ); 242 if( !ofile.is_open() ) 243 continue; 244 cout <<g_ImgCnt++<< foldname+imgname<<endl; 245 ofile.write( pos, byteRead- (pos-p) ); 246 ofile.close(); 247 } 248 free(image); 249 } 250 } 251 } 252 253 254 255 //广度遍历 256 void BFS( const string & url ){ 257 char * response; 258 int bytes; 259 // 获取网页的相应，放入response中。 260 if( !GetHttpResponse( url, response, bytes ) ){ 261 cout << "The url is wrong! ignore." << endl; 262 return; 263 } 264 string httpResponse=response; 265 free( response ); 266 string filename = ToFileName( url ); 267 ofstream ofile( "./html/"+filename ); 268 if( ofile.is_open() ){ 269 // 保存该网页的文本内容 270 ofile << httpResponse << endl; 271 ofile.close(); 272 } 273 vector<string> imgurls; 274 //解析该网页的所有图片链接，放入imgurls里面 275 HTMLParse( httpResponse, imgurls, url ); 276 277 //下载所有的图片资源 278 DownLoadImg( imgurls, url ); 279 } 280 281 void main() 282 { 283 //初始化socket，用于tcp网络连接 284 WSADATA wsaData; 285 if( WSAStartup(MAKEWORD(2,2), &wsaData) != 0 ){ 286 return; 287 } 288 289 // 创建文件夹，保存图片和网页文本文件 290 CreateDirectory((LPCSTR) "./img",0); 291 CreateDirectory((LPCSTR)"./html",0); 292 //string urlStart = "http://hao.360.cn/meinvdaohang.html"; 293 294 // 遍历的起始地址 295 // string urlStart = "http://www.wmpic.me/tupian"; 296 string urlStart = "http://item.taobao.com/item.htm?spm=a230r.1.14.19.sBBNbz&id=36366887850&ns=1#detail"; 297 298 // 使用广度遍历 299 // 提取网页中的超链接放入hrefUrl中，提取图片链接，下载图片。 300 BFS( urlStart ); 301 302 // 访问过的网址保存起来 303 visitedUrl.insert( urlStart ); 304 305 while( hrefUrl.size()!=0 ){ 306 string url = hrefUrl.front(); // 从队列的最开始取出一个网址 307 cout << url << endl; 308 BFS( url ); // 遍历提取出来的那个网页，找它里面的超链接网页放入hrefUrl，下载它里面的文本，图片 309 hrefUrl.pop(); // 遍历完之后，删除这个网址 310 } 311 WSACleanup(); 312 return; 313 }
郑重声明：本站内容如果来自互联网及其他传播媒体，其版权均属原媒体及文章作者所有。转载目的在于传递更多信息及用于网络分享，并不代表本站赞同其观点和对其真实性负责，也不构成任何其他建议。
