Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Updated Updater working method and RemoteConfig for listening to updates
  • Loading branch information
Javinator9889 committed Jun 29, 2020
1 parent 6d20163 commit 1572822
Show file tree
Hide file tree
Showing 7 changed files with 118 additions and 28 deletions.
27 changes: 27 additions & 0 deletions firebase.json
Expand Up @@ -23,5 +23,32 @@
"function": "www-webApi"
}
]
},
"firestore": {
"rules": "firestore.rules",
"indexes": "firestore.indexes.json"
},
"emulators": {
"functions": {
"port": 5001
},
"firestore": {
"port": 8080
},
"hosting": {
"port": 5000
},
"pubsub": {
"port": 8085
},
"ui": {
"enabled": true
},
"database": {
"port": 9000
}
},
"storage": {
"rules": "storage.rules"
}
}
31 changes: 31 additions & 0 deletions functions/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions functions/package.json
Expand Up @@ -17,6 +17,7 @@
},
"main": "lib/bin/index.js",
"dependencies": {
"axios": "^0.19.2",
"body-parser": "^1.19.0",
"cross-env": "^7.0.2",
"express": "^4.17.1",
Expand Down
2 changes: 2 additions & 0 deletions functions/src/bin/daemon.ts
Expand Up @@ -19,3 +19,5 @@ process.on('SIGINT', () => {
exports.updater = functions.https.onRequest(async (req, resp) => {
resp.sendStatus(200);
});

exports.remoteConfigTrigger = functions.remoteConfig.onUpdate(updater.remoteConfigEventHandler)
11 changes: 11 additions & 0 deletions functions/src/models/updater.ts
@@ -1,6 +1,9 @@
import {Updater} from '../updater';
import {RemoteConfigData} from "../rcdata";
import properties = require('../common/properties');
import {TemplateVersion} from "firebase-functions/lib/providers/remoteConfig";
import {languages} from "../common/properties";
import {EventContext} from "firebase-functions";


const updaters: Record<string, Updater> = {};
Expand Down Expand Up @@ -49,3 +52,11 @@ export async function stopScheduling() {
clearInterval(timer);
}
}

export async function remoteConfigEventHandler(event: TemplateVersion, _: EventContext) {
console.debug(`RemoteConfig values have changed - version ${event.versionNumber}`);
for (const language of languages) {
console.debug(`Updating search terms for language ${language}`);
remoteConfig.updaters[language].searchTerms = await remoteConfig.getSearchTermsForLanguage(language);
}
}
5 changes: 4 additions & 1 deletion functions/src/rcdata.ts
Expand Up @@ -34,7 +34,9 @@ export class RemoteConfigData {
}

listenToRCChanges() {
functions.remoteConfig.onUpdate(_ => {
console.info('Listening to changes in RemoteConfig...');
functions.remoteConfig.onUpdate((version, _) => {
console.debug(`RemoteConfig values have changed - version: ${version}`);
return admin.credential.applicationDefault().getAccessToken()
// tslint:disable-next-line:no-shadowed-variable
.then(_ => {
Expand All @@ -46,6 +48,7 @@ export class RemoteConfigData {
template.parameters['search_terms'].conditionalValues[language]['value']
);
try {
console.debug("Updating updater search terms");
if (this.updaters[language].searchTerms.length !== terms.lenght)
this.updaters[language].searchTerms = terms;
} catch (e) {
Expand Down
69 changes: 42 additions & 27 deletions functions/src/updater.ts
@@ -1,6 +1,7 @@
import {NewsriverData} from "./newsriver";
import * as firebaseHelper from 'firebase-functions-helper';
import * as fetch from 'node-fetch';
import {AxiosInstance, default as axios} from "axios";
import * as https from "https";


export class Updater {
Expand All @@ -10,13 +11,14 @@ export class Updater {
private _searchTerms: Array<string>;
private readonly language: string;
private readonly auth: string;
private _url: string | undefined;
private _path: string | undefined;
private readonly network: AxiosInstance

get url(): Promise<string> {
if (this._url === undefined)
return this.buildURL()
.then(url => this._url = url);
return Promise.resolve(this._url);
get path(): Promise<string> {
if (this._path === undefined)
return this.buildPath()
.then(path => this._path = path);
return Promise.resolve(this._path);
}

get searchTerms(): Array<string> {
Expand All @@ -25,7 +27,7 @@ export class Updater {

set searchTerms(value) {
this._searchTerms = value;
this._url = undefined;
this._path = undefined;
}

constructor(db: FirebaseFirestore.Firestore | null,
Expand All @@ -40,7 +42,18 @@ export class Updater {
this.language = language;
this.auth = auth;
this.interval = intervalMins * 60 * 1000;
this.request()
this.network = axios.create({
baseURL: 'https://api.newsriver.io/v2/',
headers: {
'Authorization': this.auth,
'Content-Type': 'application/json'
},
withCredentials: true,
responseType: 'json',
httpsAgent: new https.Agent({ keepAlive: true }),
timeout: 500000
});
this.doRequest()
.then(response => {
this.updateData(response)
// tslint:disable-next-line:no-empty
Expand All @@ -53,8 +66,8 @@ export class Updater {
schedule(): NodeJS.Timer {
return setInterval(async () => {
try {
const response = await this.request();
await this.updateData(response);
const requestData = await this.doRequest();
await this.updateData(requestData);
} catch (e) {
console.error(`Got error ${e} while querying data`);
}
Expand All @@ -65,11 +78,16 @@ export class Updater {
try {
for (const element of content) {
try {
const exists = await firebaseHelper.firestore.checkDocumentExists(this.db, this.collectionName, element.id);
if (!exists)
await firebaseHelper.firestore.createDocumentWithID(this.db, this.collectionName, element.id, element);
const document = await this.db.collection(this.collectionName).doc(element.id).get();
console.debug(`Item with id ${element.id} ${document.exists ? 'exists' : 'does not exist'}`)
if (!document.exists)
firebaseHelper.firestore.createDocumentWithID(this.db, this.collectionName, element.id, element)
.then(created => console.debug(`Item with ID: ${element.id} was ${created ? 'created' : 'not created'}`))
.catch(err => console.error(`Error while creating document ${err}`));
else
await firebaseHelper.firestore.updateDocument(this.db, this.collectionName, element.id, element);
firebaseHelper.firestore.updateDocument(this.db, this.collectionName, element.id, element)
.then(updated => console.debug(`Item with ID ${element.id} was ${updated ? 'updated' : 'not updated'}`))
.catch(err => console.error(`Error while updating document ${err}`));
} catch (err) {
console.warn(`Error while creating/updating document - ${err}`);
}
Expand All @@ -80,23 +98,20 @@ export class Updater {
}
}

async request(): Promise<Array<NewsriverData>> {
const requestUrl = await this.url;
const response = await fetch(requestUrl, {
method: 'GET', headers: new fetch.Headers({
'Authorization': this.auth,
'Content-Type': 'application/json'
})
});
return await response.json() as Array<NewsriverData>;
async doRequest(): Promise<Array<NewsriverData>> {
const response = await this.network.get(await this.path);
if (response.status === 200)
return response.data as Array<NewsriverData>;
else
throw new TypeError(`The response code is not valid - ${response.status}`);
}

async buildURL(): Promise<string> {
const parts = ['https://api.newsriver.io/v2/search?query='];
async buildPath(): Promise<string> {
const parts = ['/search?query=']
this.searchTerms.forEach((term, i, _) => {
if (i !== 0)
parts.push(encodeURI(' OR '));
parts.push(encodeURI(`title:${term} OR text:${term}`));
parts.push(encodeURI(`title:${term} OR text:${term}`));
});

parts.push(encodeURI(` AND language:${this.language.toUpperCase()}`));
Expand Down

0 comments on commit 1572822

Please sign in to comment.