New to node. Blank of idea how to write it in order to avoid the hang up,
and properly get the whole thing done.
Searching around internet and found a few solutions, but all not working
for me.
==
arrtickers got around 300 values.
for each value, read through 9 websites to scrape.
Some website may not be accessible.
Some website may take longer time to load.
Memory is not a problem as I am using it in a server, I want to maximize
the scrapping speed.
==
Error sure happen during half way.
TypeError: Cannot read property 'statusCode' of undefined
==
when I do further debug, it say Error: socket hang up.
==
arrtickers.forEach(function(value) {
var sym= value;
(function(sym) {
url2=
"http://test2.com/quote.ashx?t="+sym+"&ty=c&ta=1&p=d&b=1";;
request({ uri:url2 }, function (error, response, body) {
console.log("error: " + error);
jsdom.env({
html: body,
scripts: [
jqlib
]
}, function (err, window) {
var $ = window.jQuery;
var data= $('body').html();
});
});
})(sym);
(function(sym) {
url=
"http://test3.com/quote.ashx?t="+sym+"&ty=c&ta=1&p=d&b=1";;
request({ uri:url3 }, function (error, response, body) {
console.log("error: " + error);
jsdom.env({
html: body,
scripts: [
jqlib
]
}, function (err, window) {
var $ = window.jQuery;
var data= $('body').html();
});
});
})(sym);
(function(sym) {
url=
"http://test4.com/quote.ashx?t="+sym+"&ty=c&ta=1&p=d&b=1";;
console.log("error: " + error);
request({ uri:url4 }, function (error, response, body) {
jsdom.env({
html: body,
scripts: [
jqlib
]
}, function (err, window) {
var $ = window.jQuery;
var data= $('body').html();
});
});
})(sym);
//same function repeat for test5.com, test6.com, test7.com,
test8.com, test9.com
});
--
Job Board: http://jobs.nodejs.org/
Posting guidelines:
https://github.com/joyent/node/wiki/Mailing-List-Posting-Guidelines
You received this message because you are subscribed to the Google
Groups "nodejs" group.
To post to this group, send email to [email protected]
To unsubscribe from this group, send email to
[email protected]
For more options, visit this group at
http://groups.google.com/group/nodejs?hl=en?hl=en
