CasperJS - How to open up all links in an array of links

I'm trying to make it so that CasperJS will open up every link in an array of links. I have it so that after I open a link, it will display the title of that page. Yet when I run it, nothing is displayed.

I can use a for loop to display the links and it works perfectly.

This is the code for what I just explained:

var x;

casper.start(URL, function() {

    x = links.split(" "); // now x is an array of links

    for (var i = 0; j < x.length; i++) // for every link...
    {
        casper.thenOpen(partialURL + x[i], function() { // open that link
            console.log(this.getTitle() + '\n'); // display the title of page
        });
    }

    this.exit();
});

casper.run();

This is another method I tried:

var x;

casper.start(URL, function() {
    x = links.split(" "); // now x is an array of links
    this.exit();
});

for (var i = 0; j < x.length; i++) // for every link...
{
    casper.thenOpen(partialURL + x[i], function() { // open that link
        console.log(this.getTitle() + '\n'); // display the title of page
    });
}

casper.run();

It says that 'x' in undefined. Notice that I set x to be a global variable though. Any modifications that you could make would be great. Thanks.

Answers:

Answer
var x; var i = -1;

casper.start(URL, function() {
    x = links.split(" "); // now x is an array of links
});

casper.then(function() {
    this.each(x, function() { 
        i++; // change the link being opened (has to be here specifically)
        this.thenOpen((partialURL + x[i]), function() {
            this.echo(this.getTitle()); // display the title of page
        });
    });
});

casper.run();
Answer
var i = 0;
var nTimes = x.length;

casper.repeat(nTimes, function() {
    //... do your stuff
    i++;
});

worked for me.

Answer
casper.start('about:blank');

var urls = ['http://google.fr', 'http://yahoo.fr', 'http://amazon.fr'];

casper.each(urls, function(casper, url) {
  casper.thenOpen(url, function() {
        this.echo("I'm in your " + url + ".");
    });
});
Answer

In my case, I had to scrape a site that had an unknown number of pages. Each page (except the last) had a <a class="next-page" href="/page/N">Next page</a> link (where N is the page number). There was no way for the scraper to know when it was finished except when the "Next Page" link was no longer present.

Of course you'll have to make adjustments depending on what type of pagination links might exist on your page.

Here's what I did. Ymmv.

// imports
var fs = require('fs');

// scraper state
var state = {page: 1, data: []};

// casper
var casper = require("casper").create();

// scraper function
function scrape() {
  this.echo('Scraping page ' + state.page + '...', 'INFO');

  state.data = state.data.concat(this.evaluate(function() {
    // get some stuff from the page
    return someData;
  });

  var nextUrl = this.evaluate(function() {
    var nextLink = document.querySelector("a.next-page");
    return nextLink && nextLink.href;
  });

  if (nextUrl) {
    state.page = state.page + 1;
    casper.thenOpen(nextUrl, scrape); // <- recursion
  }
});

// run
casper.run(function() {
  fs.write('./data.json', JSON.stringify(state.data, null, '\t'), 'w');
  this.echo('Done!', 'INFO');
});

Hope this helps someone. If you have other questions, I'll be happy to try to help.

Answer
casper.start();
casper.each(Object.keys(array), function(casper, array_elem) {
    this.thenOpen(partialURL+array[attay_item], function() {
        ...
};

And as to "undefined" error. Try not to use this too much. I experience this error with CasperJS to often, so I prefer to write casper instead of this.

Answer

Try something like this.

var x;

casper.start(URL, function() {
    x = links.split(" "); // now x is an array of links
});

casper.then(function() {
    this.eachThen(x, function(response) {
        this.thenOpen((partialURL + response.data), function() {
            this.echo(this.getTitle()); // display the title of page
        });
    });
});

casper.run();

x was undefined because the for loop was being executed before casper.start. In the above code, the eachThen() block is nested inside of a casper.then block in order to delay its execution.

Answer

I have solved the same issue with this code:

casper.then(function () {
    var i = -1;
    this.eachThen(locations, function () {
        i++;
        //Do stuff here like for example:
        this.thenOpen(YOUR_URL, function () {
            this.waitForSelector("MYSELECTOR", 
            function () {

            },                
            function () {

            })
        });
    })
});

Tags

Recent Questions

Top Questions

Home Tags Terms of Service Privacy Policy DMCA Contact Us

©2020 All rights reserved.