|
|
|
|
|
|
|
|
let collection = dbo.collection('scrapes'); |
|
|
let collection = dbo.collection('scrapes'); |
|
|
let scrape = await collection.findOne({ _id: _id }); |
|
|
let scrape = await collection.findOne({ _id: _id }); |
|
|
|
|
|
|
|
|
for (var page = 1; page <= scrape.pageCount; page++) { |
|
|
|
|
|
|
|
|
// for (var page = 1; page <= scrape.pageCount; page++) { |
|
|
|
|
|
for (var page = 1; page <= 1; page++) { |
|
|
console.log("scrapping page " + page) |
|
|
console.log("scrapping page " + page) |
|
|
|
|
|
|
|
|
const filterPage = await axios(scrape.sourceUrl + `/${page}`); |
|
|
const filterPage = await axios(scrape.sourceUrl + `/${page}`); |
|
|
|
|
|
|
|
|
}).get(); |
|
|
}).get(); |
|
|
|
|
|
|
|
|
var properties = []; |
|
|
var properties = []; |
|
|
|
|
|
// var limit = 0; |
|
|
for (const link of propertyLinks) { |
|
|
for (const link of propertyLinks) { |
|
|
|
|
|
// if(limit == 1) break; |
|
|
try { |
|
|
try { |
|
|
var response = await axios(link); |
|
|
var response = await axios(link); |
|
|
|
|
|
|
|
|
var property = apartments.apartment(cheerio.load(response.data)); |
|
|
var property = apartments.apartment(cheerio.load(response.data)); |
|
|
|
|
|
property.url = link; |
|
|
properties.push(property); |
|
|
properties.push(property); |
|
|
console.log(`${link} scraped.`); |
|
|
console.log(`${link} scraped.`); |
|
|
|
|
|
limit++; |
|
|
}catch(err){ |
|
|
}catch(err){ |
|
|
console.error(`${link} scrape failed.`); |
|
|
console.error(`${link} scrape failed.`); |
|
|
} |
|
|
} |