-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
94 lines (80 loc) · 3.08 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
var kue = require('kue'),
async = require('async'),
mongoose = require('mongoose'),
jobs = kue.createQueue(),
_config = null;
module.exports.startUpdateService = function (config) {
_config = config;
var express = require('express'),
basicAuth = require('basic-auth-connect');
mongoose.connect('mongodb://' + config.mongodb.host + ':' + config.mongodb.port + '/' + config.mongodb.dbname);
mongoose.model('PostModel', require('./lib/models/post').PostModel);
mongoose.model('CommentModel', require('./lib/models/comment').CommentModel);
mongoose.model('AnalysisModel', require('./lib/models/analysis').AnalysisModel);
if (config.kue.admin.active) {
var app = express();
app.use(basicAuth(config.kue.admin.login, config.kue.admin.password));
kue.app.set('title', 'Cesspoll Turd Wrangler');
app.use(kue.app);
app.listen(config.kue.admin.port);
}
jobs.on('job complete', function (id) {
kue.Job.get(id, function (err, job) {
if (job) job.remove();
});
});
jobs.on('job complete', function (id) {
kue.Job.get(id, function (err, job) {
if (err) {
return console.log('error getting job on complete', err, job);
}
job.remove(function (err) {
if (err) {
console.log('error removing job', err, job);
return;
}
});
});
});
jobs.process('updateIndex', config.job_concurrency, function (job, done) {
var worker = require('./lib/workers/' + job.data.items);
worker.job(job, done);
});
// start digging for turds!!!
module.exports.updateIndex();
if (config.analysis.active) {
module.exports.analyzeNextComment();
}
};
module.exports.updateIndex = function () {
for (var index in _config.crawlers) {
jobs.create('updateIndex', { title: 'Fetch new posts', source: _config.crawlers[index], items: 'posts', config: _config }).save();
jobs.create('updateIndex', { title: 'Fetch new comments', source: _config.crawlers[index], items: 'comments', config: _config }).save();
}
setTimeout(function () {
module.exports.updateIndex();
}, _config.update_delay * 60 * 1000);
};
module.exports.analyzeNextComment = function () {
var worker = require('./lib/workers/hyphenation'),
path = require('path');
mongoose.model('CommentModel').findOne({ is_analyzed: false }, function (err, comment) {
if (err || !comment) {
setTimeout(function () {
module.exports.analyzeNextComment();
}, 1000);
} else {
worker.job({
data: {
hyphenation: _config.analysis.hyphenation,
comment: comment,
tmpPath: path.resolve(_config.tmp_folder)
}
}, function (err) {
setTimeout(function () {
module.exports.analyzeNextComment();
}, 0);
});
}
});
};