Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

lessons4中的eventproxy问题。 #135

Open
l20 opened this issue May 30, 2017 · 3 comments
Open

lessons4中的eventproxy问题。 #135

l20 opened this issue May 30, 2017 · 3 comments

Comments

@l20
Copy link

l20 commented May 30, 2017

比如topicUrl是对象的集合:

var cnodeTitle = {
	id: index,
	title: title,
	href: href
}

topicUrls.push(cnodeTitle);

那我怎样在after方法里读取topic[i].href中的链接然后请求呢?

ep.after('topic_title', topicUrls.length, function(topics){
	topics = topics.map(function(){

	});
	console.log(topics.title);
});
@bingoAO
Copy link

bingoAO commented Jun 27, 2017

不是请求之后,计数完成,才会触发after的回调方法么。
另外如果要遍历一个对象集合,一样可以用foreach,然后item.href取链接就可以了。
topics.forEach(function(item){//item是一个对象
console.log(item.href);//取对象的href的值
})

@yvanwangl
Copy link

topicUrls.forEach(function (topicUrl) {
  superagent.get(topicUrl)
    .end(function (err, res) {
      console.log('fetch ' + topicUrl + ' successful');
      ep.emit('topic_html', [topicUrl, res.text]);
    });
});

如果要控制并发数,这个就没法实现了吧,例如一个并发队列,每次加载5条,如果其中一条完成,再加载下一条。

@linyisong
Copy link

linyisong commented Oct 20, 2017

`//引入外部模块
var superagent = require('superagent');//http方面的库,可以发起get post请求
var cheerio = require('cheerio'); //node.js 版的jquery
var url = require('url');//使用 url.resolve
var eventproxy = require('eventproxy');//控制并发
/*1.获取所有cnodejs.org所有话题的href,在此基础上面得到完整的url*/
var cnodeUrl="https://cnodejs.org/";
superagent.get(cnodeUrl)
    .end(function(err,res){
        if(err){
            console.error(err);
        }
        var cnodeUrls = [];
        var $ = cheerio.load(res.text);
        $('#topic_list .topic_title').each(function(i,element) {
            //console.log($(element).attr('href'));//$(element).attr('href') 获取到href的所有内容
            if(i<5){//防止过度请求
            var href = url.resolve(cnodeUrl,$(element).attr('href'));//整合成整个URL
            cnodeUrls.push(href);
            }
        });
        /*2.获取每个URL中的页面,*/
        var ep = new eventproxy();
        ep.after('topic_html',cnodeUrls.length,function(topics) {
                topics = topics.map(function(em) {
                    var topicUrl = em[0];
                    var score0 = em[1];
                    var title = em[2];
                    var comment0 = em[3];
                    var author0 = em[4];
                    return ({
                        title: title,
                        href: topicUrl,
                        comment0: comment0,
                        author0: author0,
                        score0: score0
                    });
                });
                console.log(topics);
        });
        cnodeUrls.forEach(function(topicUrl) {
           superagent.get(topicUrl)//针对每一个页面进行get
               .end(function(error,tres){
                        console.log('fetch ' + topicUrl + '-sucessful' );
                   var $ = cheerio.load(tres.text);
                   /*获取author的链接*/
                   var score0 = null;
                   var title = $('.topic_full_title').text().trim();
                   var comment0 = $('.reply_content').eq(0).text().trim();
                   var author0 = $('.reply_author').eq(0).text().trim();
                   var author0href = $('.reply_author').eq(0).attr('href');
                   //获取页面过多会存在undefined的
                   if(typeof(author0href)!='undefined'){
                       var author0Url = url.resolve(cnodeUrl, $('.reply_author').eq(0).attr('href'));
                       console.log('|-author fetch ' + author0Url + '-successful');

                       superagent.get(author0Url)
                           .end(function(auerr,aures) {
                               var $ = cheerio.load(aures.text);
                               score0 = $('.floor').text().trim();
                               ep.emit('topic_html',[topicUrl,score0,title,comment0,author0]);
                           });
                   }
            });
        });
    });`

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

4 participants