bda-research / node-webcrawler Goto Github PK

TypeError: $ is not a function

npm WARN notice [SECURITY] lodash has the following vulnerability: 1 low. Go here for more details: https://nodesecurity.io/advisories?search=lodash&version=3.8.0 - Run `npm i npm@latest -g` to upgrade your npm version, and then `npm audit` to get more info.

var Crawler = require("node-webcrawler");
var url = require('url');
var jsdom = require('jsdom');
var c = new Crawler({
    maxConnections : 10,
    jQuery: jsdom,
    // This will be called for each crawled page
    callback : function (error, result, $) {
        // $ is Cheerio by default
        //a lean implementation of core jQuery designed specifically for the server
        if(error){
            console.log(error);
        }else{

            try {
                console.log($("title").text());
                $('a').each(function (index, a) {
                    var toQueueUrl = $(a).prop('href');
                    //console.log(toQueueUrl);
                    c.queue(toQueueUrl);
                });

            } catch (e) {
                console.log(e);
            }
        }
    }
});

c.queue('http://www.wandoujia.com/');

var Crawler = require("node-webcrawler");
var url = require('url');
var urls = [];
var c = new Crawler({
    maxConnections : 1,

    userAgent: 'Mozilla/5.0 (Windows; U; Windows NT 5.1; pt-BR; rv:1.8.1.17) Gecko/20080829 Firefox/2.0.0.17',
    //cache:true,
    // This will be called for each crawled page 
    callback : function (error, result, $) {
        // $ is Cheerio by default 
        //a lean implementation of core jQuery designed specifically for the server 
        if(error){
            console.log(error);
        }else{
             //console.log($("title").text());

            $( "a" ).each(function() {
             var url = $( this ).attr( "href" );
                spider(url);



        });
        }
    }
});

 var spider = function(url){
     //console.log(urls.indexOf(url));
     if(!url){
         return false;
     }
     if((urls.indexOf(url)<0) && (url.indexOf("#")<0) && (url.indexOf("javascript:")<0)){
     urls.push(url);
     console.log(url);
     c.queue(url);
     }  
 }
// Queue just one URL, with default callback 
c.queue('http://www.example.com');