diff --git a/firebase.json b/firebase.json index 88da25d..b1a3ed3 100644 --- a/firebase.json +++ b/firebase.json @@ -23,5 +23,32 @@ "function": "www-webApi" } ] + }, + "firestore": { + "rules": "firestore.rules", + "indexes": "firestore.indexes.json" + }, + "emulators": { + "functions": { + "port": 5001 + }, + "firestore": { + "port": 8080 + }, + "hosting": { + "port": 5000 + }, + "pubsub": { + "port": 8085 + }, + "ui": { + "enabled": true + }, + "database": { + "port": 9000 + } + }, + "storage": { + "rules": "storage.rules" } } diff --git a/functions/package-lock.json b/functions/package-lock.json index df547f1..b486623 100644 --- a/functions/package-lock.json +++ b/functions/package-lock.json @@ -474,6 +474,14 @@ "array-filter": "^1.0.0" } }, + "axios": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.2.tgz", + "integrity": "sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA==", + "requires": { + "follow-redirects": "1.5.10" + } + }, "babel-walk": { "version": "3.0.0-canary-5", "resolved": "https://registry.npmjs.org/babel-walk/-/babel-walk-3.0.0-canary-5.tgz", @@ -1159,6 +1167,29 @@ "lodash": "^4.17.5" } }, + "follow-redirects": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ==", + "requires": { + "debug": "=3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "foreach": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", diff --git a/functions/package.json b/functions/package.json index 128ac18..7b1e502 100644 --- a/functions/package.json +++ b/functions/package.json @@ -17,6 +17,7 @@ }, "main": "lib/bin/index.js", "dependencies": { + "axios": "^0.19.2", "body-parser": "^1.19.0", "cross-env": "^7.0.2", "express": "^4.17.1", diff --git a/functions/src/bin/daemon.ts b/functions/src/bin/daemon.ts index b96e737..8259abc 100644 --- a/functions/src/bin/daemon.ts +++ b/functions/src/bin/daemon.ts @@ -19,3 +19,5 @@ process.on('SIGINT', () => { exports.updater = functions.https.onRequest(async (req, resp) => { resp.sendStatus(200); }); + +exports.remoteConfigTrigger = functions.remoteConfig.onUpdate(updater.remoteConfigEventHandler) diff --git a/functions/src/models/updater.ts b/functions/src/models/updater.ts index d7bce2d..b4ad6ca 100644 --- a/functions/src/models/updater.ts +++ b/functions/src/models/updater.ts @@ -1,6 +1,9 @@ import {Updater} from '../updater'; import {RemoteConfigData} from "../rcdata"; import properties = require('../common/properties'); +import {TemplateVersion} from "firebase-functions/lib/providers/remoteConfig"; +import {languages} from "../common/properties"; +import {EventContext} from "firebase-functions"; const updaters: Record = {}; @@ -49,3 +52,11 @@ export async function stopScheduling() { clearInterval(timer); } } + +export async function remoteConfigEventHandler(event: TemplateVersion, _: EventContext) { + console.debug(`RemoteConfig values have changed - version ${event.versionNumber}`); + for (const language of languages) { + console.debug(`Updating search terms for language ${language}`); + remoteConfig.updaters[language].searchTerms = await remoteConfig.getSearchTermsForLanguage(language); + } +} diff --git a/functions/src/rcdata.ts b/functions/src/rcdata.ts index 86282ec..6dc3114 100644 --- a/functions/src/rcdata.ts +++ b/functions/src/rcdata.ts @@ -34,7 +34,9 @@ export class RemoteConfigData { } listenToRCChanges() { - functions.remoteConfig.onUpdate(_ => { + console.info('Listening to changes in RemoteConfig...'); + functions.remoteConfig.onUpdate((version, _) => { + console.debug(`RemoteConfig values have changed - version: ${version}`); return admin.credential.applicationDefault().getAccessToken() // tslint:disable-next-line:no-shadowed-variable .then(_ => { @@ -46,6 +48,7 @@ export class RemoteConfigData { template.parameters['search_terms'].conditionalValues[language]['value'] ); try { + console.debug("Updating updater search terms"); if (this.updaters[language].searchTerms.length !== terms.lenght) this.updaters[language].searchTerms = terms; } catch (e) { diff --git a/functions/src/updater.ts b/functions/src/updater.ts index 1716e1e..edf4dbd 100644 --- a/functions/src/updater.ts +++ b/functions/src/updater.ts @@ -1,6 +1,7 @@ import {NewsriverData} from "./newsriver"; import * as firebaseHelper from 'firebase-functions-helper'; -import * as fetch from 'node-fetch'; +import {AxiosInstance, default as axios} from "axios"; +import * as https from "https"; export class Updater { @@ -10,13 +11,14 @@ export class Updater { private _searchTerms: Array; private readonly language: string; private readonly auth: string; - private _url: string | undefined; + private _path: string | undefined; + private readonly network: AxiosInstance - get url(): Promise { - if (this._url === undefined) - return this.buildURL() - .then(url => this._url = url); - return Promise.resolve(this._url); + get path(): Promise { + if (this._path === undefined) + return this.buildPath() + .then(path => this._path = path); + return Promise.resolve(this._path); } get searchTerms(): Array { @@ -25,7 +27,7 @@ export class Updater { set searchTerms(value) { this._searchTerms = value; - this._url = undefined; + this._path = undefined; } constructor(db: FirebaseFirestore.Firestore | null, @@ -40,7 +42,18 @@ export class Updater { this.language = language; this.auth = auth; this.interval = intervalMins * 60 * 1000; - this.request() + this.network = axios.create({ + baseURL: 'https://api.newsriver.io/v2/', + headers: { + 'Authorization': this.auth, + 'Content-Type': 'application/json' + }, + withCredentials: true, + responseType: 'json', + httpsAgent: new https.Agent({ keepAlive: true }), + timeout: 500000 + }); + this.doRequest() .then(response => { this.updateData(response) // tslint:disable-next-line:no-empty @@ -53,8 +66,8 @@ export class Updater { schedule(): NodeJS.Timer { return setInterval(async () => { try { - const response = await this.request(); - await this.updateData(response); + const requestData = await this.doRequest(); + await this.updateData(requestData); } catch (e) { console.error(`Got error ${e} while querying data`); } @@ -65,11 +78,16 @@ export class Updater { try { for (const element of content) { try { - const exists = await firebaseHelper.firestore.checkDocumentExists(this.db, this.collectionName, element.id); - if (!exists) - await firebaseHelper.firestore.createDocumentWithID(this.db, this.collectionName, element.id, element); + const document = await this.db.collection(this.collectionName).doc(element.id).get(); + console.debug(`Item with id ${element.id} ${document.exists ? 'exists' : 'does not exist'}`) + if (!document.exists) + firebaseHelper.firestore.createDocumentWithID(this.db, this.collectionName, element.id, element) + .then(created => console.debug(`Item with ID: ${element.id} was ${created ? 'created' : 'not created'}`)) + .catch(err => console.error(`Error while creating document ${err}`)); else - await firebaseHelper.firestore.updateDocument(this.db, this.collectionName, element.id, element); + firebaseHelper.firestore.updateDocument(this.db, this.collectionName, element.id, element) + .then(updated => console.debug(`Item with ID ${element.id} was ${updated ? 'updated' : 'not updated'}`)) + .catch(err => console.error(`Error while updating document ${err}`)); } catch (err) { console.warn(`Error while creating/updating document - ${err}`); } @@ -80,23 +98,20 @@ export class Updater { } } - async request(): Promise> { - const requestUrl = await this.url; - const response = await fetch(requestUrl, { - method: 'GET', headers: new fetch.Headers({ - 'Authorization': this.auth, - 'Content-Type': 'application/json' - }) - }); - return await response.json() as Array; + async doRequest(): Promise> { + const response = await this.network.get(await this.path); + if (response.status === 200) + return response.data as Array; + else + throw new TypeError(`The response code is not valid - ${response.status}`); } - async buildURL(): Promise { - const parts = ['https://api.newsriver.io/v2/search?query=']; + async buildPath(): Promise { + const parts = ['/search?query='] this.searchTerms.forEach((term, i, _) => { if (i !== 0) parts.push(encodeURI(' OR ')); - parts.push(encodeURI(`title:${term} OR text:${term}`)); + parts.push(encodeURI(`title:${term} OR text:${term}`)); }); parts.push(encodeURI(` AND language:${this.language.toUpperCase()}`));