From fc54a3eafc1a99c7cd2ea1e22a02bd5c54792bd8 Mon Sep 17 00:00:00 2001 From: nicomigueles Date: Fri, 6 Sep 2019 21:00:52 -0300 Subject: [PATCH 1/5] Added `admin/fetch-videos` endpoint to manually fetch the yt-api --- src/api/admin/admin.controller.js | 13 +++++++++++++ src/api/admin/admin.routes.js | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/api/admin/admin.controller.js b/src/api/admin/admin.controller.js index fd0fae7..3ed008a 100644 --- a/src/api/admin/admin.controller.js +++ b/src/api/admin/admin.controller.js @@ -1,4 +1,5 @@ const { mainSeed } = require('../../tasks/seed/seed'); +const { fetchVideosJob } = require('../../helpers/fetchData'); async function seed(req, res, next) { try { @@ -9,6 +10,18 @@ async function seed(req, res, next) { } } +async function fetchVideos(req, res, next) { + try { + // eslint-disable-next-line no-console + console.log('Manual fetch.'); + await fetchVideosJob(); + res.sendStatus(200); + } catch (error) { + next(error); + } +} + module.exports = { seed, + fetchVideos, }; diff --git a/src/api/admin/admin.routes.js b/src/api/admin/admin.routes.js index 5a30cd3..48a1bf6 100644 --- a/src/api/admin/admin.routes.js +++ b/src/api/admin/admin.routes.js @@ -1,6 +1,7 @@ const router = require('express').Router(); -const { seed } = require('./admin.controller'); +const { seed, fetchVideos } = require('./admin.controller'); router.post('/seed', seed); +router.post('/fetch-videos', fetchVideos); module.exports = router; From a50c122944b8122241f4796deecc73033da9d792 Mon Sep 17 00:00:00 2001 From: nicomigueles Date: Sun, 8 Sep 2019 02:58:57 -0300 Subject: [PATCH 2/5] endpoint working! --- src/api/admin/admin.controller.js | 13 ++-- src/helpers/fetchData.js | 108 ++++++++++++++++++++---------- src/tasks/pollYoutube.js | 12 +++- 3 files changed, 92 insertions(+), 41 deletions(-) diff --git a/src/api/admin/admin.controller.js b/src/api/admin/admin.controller.js index 3ed008a..4ee4bc0 100644 --- a/src/api/admin/admin.controller.js +++ b/src/api/admin/admin.controller.js @@ -12,10 +12,15 @@ async function seed(req, res, next) { async function fetchVideos(req, res, next) { try { - // eslint-disable-next-line no-console - console.log('Manual fetch.'); - await fetchVideosJob(); - res.sendStatus(200); + const { message: info, error } = await fetchVideosJob(); + + const status = error ? 500 : 200; + const message = error || info; + + res.status(status).json({ + status, + message, + }); } catch (error) { next(error); } diff --git a/src/helpers/fetchData.js b/src/helpers/fetchData.js index 775d7d0..332386a 100644 --- a/src/helpers/fetchData.js +++ b/src/helpers/fetchData.js @@ -1,6 +1,5 @@ /* eslint-disable no-console */ const fetch = require('node-fetch'); -const { red, green } = require('colors/safe'); const Video = require('../api/video/video.model'); const { YOUTUBE_API_KEY, YOUTUBE_CHANNEL_ID } = require('../config'); @@ -14,53 +13,92 @@ async function fetchLatestYoutubeVideos({ maxResults, publishedAfter }) { const { items, error } = await resp.json(); if (error) { - error.errors.forEach(({ reason }) => { - console.error(`[fetch-error] ${reason}`); - }); - return []; + // Test this line! + const errors = error.errors.map(err => err.reason).join(', '); + return { + error: errors, + }; } - if (items) { - return items.map(({ id: { videoId: videoID }, snippet: { title: name, publishedAt: date, description, thumbnails: { high: { url: thumbnail } } } }) => { - return { - name, - date, - description, - url: `https://www.youtube.com/watch?v=${videoID}`, - videoID, - thumbnail, - }; - }); + if (!items) { + return []; } - return []; - } catch (err) { - console.error(`[error]: ${err}`); - return []; + + return items.map(({ id: { videoId: videoID }, snippet: { title: name, publishedAt: date, description, thumbnails: { high: { url: thumbnail } } } }) => { + return { + name, + date, + description, + url: `https://www.youtube.com/watch?v=${videoID}`, + videoID, + thumbnail, + }; + }); + } catch (error) { + return { + error, + }; } } async function fetchVideosJob() { try { - const video = await Video.findOne({}).sort({ date: -1 }); + const video = await Video.findOne({}).sort({ + date: -1, + }); // Check if db has at least one video - if (video) { - // Transforms date format to the Youtube-API standard. - const lastDate = video.date.toISOString(); + if (!video) { + return { + message: `Video database is empty.`, + }; + } + + // Transforms date format to the Youtube-API standard. + const lastDate = video.date.toISOString(); + const fetchedVideos = await fetchLatestYoutubeVideos({ + publishedAfter: lastDate, + }); + + if (fetchedVideos.error) { + return { + error: `Fetch error: ${fetchedVideos.error}.`, + }; + } + + if (fetchedVideos.length === 0) { + return { + message: `Youtube API returned an empty array.`, + }; + } - const fetchedVideos = await fetchLatestYoutubeVideos({ publishedAfter: lastDate }); + let savedVideos = 0; - if (fetchedVideos.length > 0) { - fetchedVideos.forEach(async newVideo => { - if (newVideo.date !== lastDate) { - const { name } = await new Video(newVideo).save(); - console.log(green(`[cron-job] Added new video from Youtube: ${name}, at ${new Date().toISOString()}`)); - } - }); + fetchedVideos.forEach(async newVideo => { + if (newVideo.date !== lastDate) { + savedVideos += 1; + try { + await new Video(newVideo).save(); + } catch (error) { + throw error; + } } + }); + if (savedVideos === 0) { + return { + message: `Video DB up to date.`, + }; } - } catch (err) { - console.error(red(`[cron-job-error] ${err}`)); + return { + message: `Saved ${savedVideos} new videos`, + }; + } catch (error) { + return { + error, + }; } } -module.exports = { fetchLatestYoutubeVideos, fetchVideosJob }; +module.exports = { + fetchLatestYoutubeVideos, + fetchVideosJob, +}; diff --git a/src/tasks/pollYoutube.js b/src/tasks/pollYoutube.js index 610aa4e..65f3e4a 100644 --- a/src/tasks/pollYoutube.js +++ b/src/tasks/pollYoutube.js @@ -1,5 +1,13 @@ +/* eslint-disable no-console */ const { schedule } = require('node-cron'); const { fetchVideosJob } = require('../helpers/fetchData'); -fetchVideosJob(); -schedule('59 * * * *', fetchVideosJob); +// Probably this won't be used anymore but works on dev now. +const runJob = async () => { + const { error, message } = await fetchVideosJob(); + if (error) console.error(error); + if (message) console.log(message); +}; + +runJob(); +schedule('59 * * * *', runJob); From 9330926dfa1ca312d10855704f57a53eecd504be Mon Sep 17 00:00:00 2001 From: nicomigueles Date: Sun, 8 Sep 2019 17:36:55 -0300 Subject: [PATCH 3/5] updated test --- src/api/admin/admin.test.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/admin/admin.test.js b/src/api/admin/admin.test.js index 24e2549..37e33a8 100644 --- a/src/api/admin/admin.test.js +++ b/src/api/admin/admin.test.js @@ -18,3 +18,11 @@ describe('POST /admin/seed', () => { .expect(200, done); }); }); + +describe('POST /admin/fetch-videos', () => { + it('Should respond with a 200 status code', done => { + request(app) + .post('/admin/fetch-videos') + .expect(200, done); + }); +}); From 507c2ab78b0c1a6569f3aca46771f8a713a29ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Migueles?= <35758739+NicoMigueles@users.noreply.github.com> Date: Wed, 18 Sep 2019 15:38:29 -0300 Subject: [PATCH 4/5] Solved conflict --- src/helpers/fetchData.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/helpers/fetchData.js b/src/helpers/fetchData.js index 99e5639..2eee235 100644 --- a/src/helpers/fetchData.js +++ b/src/helpers/fetchData.js @@ -1,6 +1,5 @@ /* eslint-disable no-console */ const fetch = require('node-fetch'); - const Video = require('../api/videos/videos.model'); const { YOUTUBE_API_KEY, YOUTUBE_CHANNEL_ID } = require('../config'); From 16a86e179d3b1d739945e9b2f9cb36247db8eefa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Migueles?= <35758739+NicoMigueles@users.noreply.github.com> Date: Thu, 19 Sep 2019 01:41:04 -0300 Subject: [PATCH 5/5] Update fetchData.js --- src/helpers/fetchData.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/helpers/fetchData.js b/src/helpers/fetchData.js index 2eee235..67f2b7f 100644 --- a/src/helpers/fetchData.js +++ b/src/helpers/fetchData.js @@ -76,11 +76,7 @@ async function fetchVideosJob() { fetchedVideos.forEach(async newVideo => { if (newVideo.date !== lastDate) { savedVideos += 1; - try { - await new Video(newVideo).save(); - } catch (error) { - throw error; - } + await new Video(newVideo).save(); } }); if (savedVideos === 0) {