Merge branch 'master' of http://176.104.105.124:3000/nikoladiligent/Scraper-api

app.js

@@ -9,6 +9,9 @@ const ObjectID = require('mongodb').ObjectID;
 var apartments = require('./apartments.js');
 var houses = require('./houses.js');
 
+const distinct = (value, index, self) => {
+  return self.indexOf(value) === index;
+}
 // jobs
 var mongoUrl = config.get("mongo");
 var agendaDb = config.get("agenda");
@@ -33,9 +36,11 @@ agenda.define('scrape', async function (job, done) {
         return $(this).attr('href');
       }).get();
 
+      const links = propertyLinks.filter(distinct);
+
       var properties = [];
       // var limit = 0;
-      for (const link of propertyLinks) {
+      for (const link of links) {
         // if(limit == 1) break;
         try {
           var response = await axios(link);
@@ -43,8 +48,7 @@ agenda.define('scrape', async function (job, done) {
           var property = apartments.apartment(cheerio.load(response.data));
           property.url = link;
           properties.push(property);
-          console.log(`${link} scraped.`);
-          limit++;
+          console.log(`${link} scraped.`)
         } catch (err) {
           console.error(`${link} scrape failed.`);
         }
@@ -89,6 +93,10 @@ MongoClient.connect(mongoUrl, function (err, db) {
   });
 });
 
+app.get("/", async (req, res) => {
+  return res.json("ok");
+});
+
 app.get("/scrapes", async (req, res) => {
   try {
     const dbo = client.db(database);
@@ -177,6 +185,7 @@ app.post("/scrapes/", async (req, res) => {
       count: resultCount,
       pageCount: pagesCount,
       estimate: dt,
+      createDate: new Date(),
       sourceUrl: query,
       location: location,
       description: description,
@@ -213,7 +222,7 @@ app.patch("/scrapes/:id/execute", async (req, res) => {
   }
 });
 
-const port = 3333;
+const port = 5501;
 app.listen(port, () => {
   console.log(`Example app listening at http://localhost:${port}`)
 });