Pārlūkot izejas kodu

Minor fxies

master
nikola pirms 4 gadiem
vecāks
revīzija
46ee12d7e0
1 mainītis faili ar 14 papildinājumiem un 4 dzēšanām
  1. 14
    4
      app.js

+ 14
- 4
app.js Parādīt failu

var apartments = require('./apartments.js'); var apartments = require('./apartments.js');
var houses = require('./houses.js'); var houses = require('./houses.js');


const distinct = (value, index, self) =>
{
return self.indexOf(value) === index;
}
// jobs // jobs
var mongoUrl = config.get("mongo"); var mongoUrl = config.get("mongo");
var agendaDb = config.get("agenda"); var agendaDb = config.get("agenda");
const propertyLinks = $('#placardContainer .property-link').map(function () { const propertyLinks = $('#placardContainer .property-link').map(function () {
return $(this).attr('href'); return $(this).attr('href');
}).get(); }).get();
const links = propertyLinks.filter(distinct);


var properties = []; var properties = [];
// var limit = 0; // var limit = 0;
for (const link of propertyLinks) {
for (const link of links) {
// if(limit == 1) break; // if(limit == 1) break;
try { try {
var response = await axios(link); var response = await axios(link);
var property = apartments.apartment(cheerio.load(response.data)); var property = apartments.apartment(cheerio.load(response.data));
property.url = link; property.url = link;
properties.push(property); properties.push(property);
console.log(`${link} scraped.`);
limit++;
console.log(`${link} scraped.`)
}catch(err){ }catch(err){
console.error(`${link} scrape failed.`); console.error(`${link} scrape failed.`);
} }
}); });
}); });


app.get("/", async (req, res) => {
return res.json("ok");
});

app.get("/scrapes", async (req, res) => { app.get("/scrapes", async (req, res) => {
try { try {
const dbo = client.db(database); const dbo = client.db(database);
count: resultCount, count: resultCount,
pageCount: pagesCount, pageCount: pagesCount,
estimate: dt, estimate: dt,
createDate: new Date(),
sourceUrl: query, sourceUrl: query,
location: location, location: location,
filters: [ filters: [
} }
}); });


const port = 3333;
const port = 5501;
app.listen(port, () => { app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`) console.log(`Example app listening at http://localhost:${port}`)
}); });

Notiek ielāde…
Atcelt
Saglabāt