nodejs 采集新闻数据
使用nodejs采集新华新闻数据
代码段:
var express = require(‘express‘); var $ = require(‘jQuery‘); var app = express(); var colors = require(‘colors‘); var message_list = require(‘./zui/message_list‘); var findData = require(‘./findData‘); //设置全局跨域访问 app.all(‘*‘, function(req, res, next) { res.header("Access-Control-Allow-Origin", "*"); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); res.header("Access-Control-Allow-Methods","PUT,POST,GET,DELETE,OPTIONS"); res.header("X-Powered-By",‘ 3.2.1‘) res.header("Content-Type", "application/json;charset=utf-8"); next(); }); //输出主页列表数据 app.get(‘/getIndexData‘, function (req, res) { findData.indexData(req, res); }); /* *输出列表数据。支持范围选择 *demo:http://localhost:3000/getList/0,9 前9条数据 从0开始 *demo: http://localhost:3000/getList/0 所有的数据 从0开始 */ app.get(/^\/getList?(?:\/(\d+)(?:\,(\d+))?)?/, function (req, res) { var start=undefined,end=undefined; if(req.params.length==1){ start=req.params[0]; }else if(req.params.length==2){ start=req.params[0]; end=req.params[1]; } findData.list(req, res,start,end,function(data){ for(var i=0,j=data.length;i<j;i++){ data[i].orderby="1"; } return data; }); }); //数据列表 var newsListData=undefined; app.get(‘/getNewsList‘, function (req, res) { if(typeof newsListData=="undefined"){ findData.list(req, res,undefined,undefined,function(data,length){ for(var i=0,j=data.length;i<j;i++){ data[i].orderby=Math.round(Math.random()*6); data[i].summary="暂无摘要"; data[i].updateUserName="admin"; } newsListData=data; getNewsList(req, res); }); }else{ getNewsList(req, res); } }); //缓存数据列表 function getNewsList(req, res){ var pageNum=req.query.pageNum,pageSize=req.query.pageSize; var start=(pageNum-1)*pageSize,end=pageNum*pageSize; var newData=newsListData.slice(start, end); res.send(JSON.stringify(newData)); } app.listen(3000); console.log("nodejs was start".green);
var http = require(‘http‘), fs = require(‘fs‘), jquery = fs.readFileSync("lib/jquery.min.js", "utf-8"), jsdom = require(‘jsdom‘); function get(url, callback) { //使用代理 var opt = { host: ‘127.0.0.1‘, port: ‘7070‘, method: ‘get‘, //这里是发送的方法 path: url } //以下是接受数据的代码 var req = http.request(opt, function(res) { res.setEncoding(‘utf8‘); var html = ‘‘; res.on(‘data‘, function(d) { html += d; }).on(‘end‘, function() { jsdom.env({ html: html, src: [jquery], done: function(errors, window) { var $ = window.$; callback && callback(errors, $); window.close(); // 释放window相关资源,否则将会占用很高的内存 } }); }); }).on(‘error‘, function(e) { console.log("Got error: " + e.message); }) req.end(); } //导出list exports.indexData = function(request, response, start, end) { get(‘http://www.news.cn/edu/index.htm‘, function(errors, $) { var $list = $(".list"); var data = []; $list.each(function(index) { var $this = $(this), $li = $this.find("li"), li_data = []; $li.each(function(index) { var $a = $(this).find("a"); var href=$a.attr("href").replace("http://news.xinhuanet.com/edu/",""); var temp_href="/"+href.replace(/[^c]*/,""); var date=href.replace(temp_href,"").replace("/","-"); if(date.length>10){ date="2013-01-04"; } li_data.push({ title: $a.text(), href: $a.attr("href"), date:date, id: index }); }); data.push({ id: index, list: li_data }); }); if (typeof start != "undefined" && typeof end == "undefined") { data = data.slice(start); } else if (typeof start != "undefined" && typeof end != "undefined") { data = data.slice(start, end); } response.send(JSON.stringify(data)); }); } //导出list exports.list = function(request, response, start, end,callback) { get(‘http://www.news.cn/edu/index.htm‘, function(errors, $) { var $li = $(".list li"); var data = []; $li.each(function(index) { var $a = $(this).find("a"); var href=$a.attr("href").replace("http://news.xinhuanet.com/edu/",""); var temp_href="/"+href.replace(/[^c]*/,""); var date=href.replace(temp_href,"").replace("/","-"); if(date.length>10){ date="2013-01-04"; } data.push({ id: index, title: $a.text(), attachment: $a.attr("href"), updateTime:date }); }); var newData=data; if (typeof start != "undefined" && typeof end == "undefined") { newData = data.slice(start); } else if (typeof start != "undefined" && typeof end != "undefined") { newData = data.slice(start, end); } if(typeof callback=="function"){ newData=callback(newData,data.length); } //response.header("Access-Control-Allow-Origin", "*"); //设置跨域访问 response.send(JSON.stringify(newData)); }); }
访问地址
http://localhost:3000/getIndexData http://localhost:3000/getList/0,4 http://localhost:3000/getNewsList?pageNum=1&pageSize=10
over
郑重声明:本站内容如果来自互联网及其他传播媒体,其版权均属原媒体及文章作者所有。转载目的在于传递更多信息及用于网络分享,并不代表本站赞同其观点和对其真实性负责,也不构成任何其他建议。