scrapper-api/app.js

const express = require('express');
var cors = require('cors');

const app = express();
// app.use(bodyParser.urlencoded({ extended: false }));
app.use(express.json());
app.use(cors());
// app.use(bodyParser.raw());

// const cron = require('node-cron');
// const {MongoClient} = require('mongodb');

// const uri = "mongodb+srv://Nikola:Nikola@cluster0.jvqzh.mongodb.net/admin?authSource=admin&replicaSet=atlas-7exvp3-shard-0&w=majority&readPreference=primary&appname=MongoDB%20Compass&retryWrites=true&ssl=true";
// const client = new MongoClient(uri);
// client.connect();
// createListing =  async function (client, data){
//   const result = await client.db("scraper").collection("scraping").insertOne(data);
//   console.log(`New scraping created with the following id: ${result.insertedId}`);
// }
// var apartments = require('./apartments.js');
// var houses = require('./houses.js');


// app.set('json spaces', 2);

// const axios = require('axios');
// const cheerio = require('cheerio');


// const url = 'https://www.apartments.com/two-west-chicago-il/jqn1nf6/';
// app.get('/', (req, res) => {
//   axios(url).then(response => {
//     const html = response.data;
//     const $ = cheerio.load(html);

//     var data = apartments.apartment($);
//     res.json(data);
//   });
// });
// app.get('/houses/*', (req, res) => {
//   var url = req.params[0];
//   axios(url).then(response => {
//     const html = response.data;
//     const $ = cheerio.load(html);

//     var data = houses.house($);
//     res.json(data);
//   });
// });
// app.get('/filters/*', async (req, res) => {
//   var url = req.params[0];
//   const filterPage = await axios(url);
//   const html = filterPage.data;
//   const $ = cheerio.load(html);

//   const propertyLins = $('#placardContainer .property-link').map(function () {
//     return $(this).attr('href');
//   }).get();

//   var properties = [];
//   for (const link of propertyLins){
//     var response = await axios(link);

//     var property = apartments.apartment(cheerio.load(response.data));
//     properties.push(property);
//   }

//   res.json(properties);
// });
// app.get('/apartments/*', (req, res) => {
//   var url = req.params[0];
//   axios(url).then(response => {
//     const html = response.data;
//     const $ = cheerio.load(html);

//     var data = apartments.apartment($);

//     createListing(client, data);
//     res.json(data);


//   });
// });

app.get("/scrapes", (req, res) => {
  res.json(
    [
      { 
        id: 1,
        location: "Chicago, IL",
        count: 21,
        estimate: Date.now(),
        sourceUrl: "https://www.apartments.com",
        filters: [
          { name: "price", value: "1000"},
          { name: "beds", value: "2"},
        ],
        status: "requested"
      },
      { 
        id: 2,
        location: "New York, NY",
        count: 21,
        estimate: Date.now(),
        sourceUrl: "https://www.apartments.com",
        filters: [
          { name: "lifestyle", value: "2"},
        ],
        status: "pending"
      },{ 
        id: 3,
        location: "Los Angeles, CA",
        count: 21,
        estimate: Date.now(),
        sourceUrl: "https://www.apartments.com",
        filters: [
          { name: "type", value: "apartments"},
        ],
        status: "done"
      }
    ]
  )
});
app.get("/scrapes/:id", (req, res) => {
  const id = req.params.id;
  //todo: get data from mongo

  res.json(id);
});
app.post("/scrapes/", (req, res) => {
  const location = req.body.location;
  console.log(req.body)
  const price = req.body.price;
  const beds = req.body.beds;
  const type = req.body.type;
  const lifestyle = req.body.lifestyle;

  // query builder

  //todo: save data into the database
  res.json({
    id: 1,
    location: location,
    filters:[
      { name: 'price', value: price },
      { name: 'beds', value: beds },
      { name: 'type', value: type },
      { name: 'lifestyle', value: lifestyle },
    ]
  });
});
app.patch("/scrapes/:id/execute", (req, res) => {
  const id = req.params.id;
  //todo: get scrape data from db
  //todo: mark scrape for execution in the job queue
  res.status(200).json(id);
});

const port = 3333;
// var task = cron.schedule('* * * * *', function() {
//   console.log(`Runned job...`)
// });
//   var options = {
//     host: 'http://localhost',
//     port:port,
//     path: '/apartments/https://www.apartments.com/essex-on-the-park-chicago-il/begd58b/',
//     method: 'GET'
//   };
  

// task.start()

// task.stop();

app.listen(port, () => {
  console.log(`Example app listening at http://localhost:${port}`)
});