diff --git a/.gitignore b/.gitignore index 33e6301bd..4073e3213 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,4 @@ config/cert.pem config/key.pem -docs/_build - +docs/_build \ No newline at end of file diff --git a/config/crowdtangle_list.json b/config/crowdtangle_list.json new file mode 100644 index 000000000..08bd273b8 --- /dev/null +++ b/config/crowdtangle_list.json @@ -0,0 +1,3 @@ +{ + "crowdtangle_list_account_pairs":{} +} diff --git a/config/secrets.json.example b/config/secrets.json.example index 0f494f56d..66fafefcd 100644 --- a/config/secrets.json.example +++ b/config/secrets.json.example @@ -89,4 +89,4 @@ "experimentFile": "test/end-to-end/fixtures/experiment_reports.json", "adminPassword": "letmein1", "adminEmail": "aggie-admin@example.com" -} +} \ No newline at end of file diff --git a/lib/fetching/bot.js b/lib/fetching/bot.js index 2804b3116..d9501e8df 100644 --- a/lib/fetching/bot.js +++ b/lib/fetching/bot.js @@ -105,6 +105,9 @@ Bot.prototype._reportListener = function(reportData) { if (reportData._media === 'facebook') { reportData.metadata.isComment ? reportData.tags.push('FBComment') : reportData.tags.push('FBPost'); } + if (reportData._media === 'crowdtangle') { + true ? reportData.tags.push(reportData.metadata.ct_tag) : reportData.tags.push('Untagged'); + } } var drops = this.queue.drops; diff --git a/lib/fetching/bots/pull-bot.js b/lib/fetching/bots/pull-bot.js index 0bc79563f..e98b6358c 100644 --- a/lib/fetching/bots/pull-bot.js +++ b/lib/fetching/bots/pull-bot.js @@ -8,7 +8,7 @@ var logger = require('../../logger'); // options.source - The source to pull from. // options.contentService - The contentService to control. var PullBot = function(options) { - this.interval = options.interval || 120000; + this.interval = options.contentService.interval || 120000; Bot.call(this, options); }; diff --git a/lib/fetching/content-service-factory.js b/lib/fetching/content-service-factory.js index 74b1ca0e9..fa12464f2 100644 --- a/lib/fetching/content-service-factory.js +++ b/lib/fetching/content-service-factory.js @@ -15,11 +15,13 @@ contentServices['dummy-fast'] = require('./content-services/dummy-fast-content-s contentServices.smsgh = require('./content-services/smsgh-content-service'); contentServices.whatsapp = require('./content-services/whatsapp-content-service'); contentServices.replay = require('./content-services/replay-content-service'); +contentServices.crowdtangle = require('./content-services/crowd-tangle-content-service'); function ContentServiceFactory() { /* empty constructor */ } // Creates a new content service to match the given source. ContentServiceFactory.prototype.create = function(source) { + var Service = contentServices[source.media]; // Content services use only the lastReportDate, url, and keywords params. var basicSource = _.pick(source, 'lastReportDate', 'url', 'keywords'); diff --git a/lib/fetching/content-services/crowd-tangle-content-service.js b/lib/fetching/content-services/crowd-tangle-content-service.js new file mode 100644 index 000000000..1e1d147a8 --- /dev/null +++ b/lib/fetching/content-services/crowd-tangle-content-service.js @@ -0,0 +1,155 @@ +var request = require('request'); +var url = require('url'); + +var ContentService = require('../content-service'); +var util = require('util'); +var config = require('../../../config/secrets'); +var crowdtangle_lists = require('../../../config/crowdtangle_list'); + +var request = require('request'); +var _ = require('underscore'); + + +//options.lastReportDate is passed here through the content-service-factory and will be utilised in calling the api, but its actually only maintained by the parent content service, it is only utilised by the child +var CrowdTangleContentService = function(options) { + this._baseUrl = config.get().crowdtangle.baseUrl; + this._pathName = config.get().crowdtangle.pathName; + this._count = config.get().crowdtangle.count; + this._language = config.get().crowdtangle.language; + this._sortParameter = config.get().crowdtangle.sortParam; + this._apiToken = config.get().crowdtangle.apiToken; + this._keywords = options.keywords; + this._lastReportDate = options._lastReportDate; + this._listIds = parseInt(options.tags); + this.fetchType = 'pull'; + this.interval = 30000; + ContentService.call(this, options); //associates the child service with its parent, which has the notion of lastreportdate +} + + +util.inherits(CrowdTangleContentService, ContentService); + +//this method overwrites the _doFetch method in the content-service (which is the parent class) +//and then inside the content-service, we have a fetch method that is called by the pull-bot and is responsible for emitting (writing) the reports and updating the final value of lastReportDate +//options.maxCount (present but not used, used by the parent content service) +//what about types? -- discuss with Michael +CrowdTangleContentService.prototype._doFetch = function(options, callback) { + var self = this; + //handle errors using process.nextTick + // this will depend mostly on the options that we expect to receive, and if there is issue with that + + // if (!this._url) { + // process.nextTick(function() { self.emit('error', new Error('Missing URL')); }); + // return callback([]); + // } + + //now we need to submit the request + this._httpRequest( {url: this._completeUrl(options)}, function(err, res, body) { + if (err) { + self.emit('error', new Error('HTTP error: ' + err.message)); + return callback([]); + } else if (res.statusCode != 200) { + self.emit('error', new Error.HTTP(res.statusCode)); + return callback([]); + } + + //if no errors, parse the body.. + var responses; + try { + responses = JSON.parse(body).result.posts; + if (!(responses instanceof Array)) { + self.emit('error', new Error('Wrong data')); + return callback([]); + } + // any other error handling wrt the structure of responses + } catch (e) { + self.emit('error', new Error('Parse error: ' + e.message)); + return callback([]); + } + // the responses will be sorted by the content-service (parent method) + // Parse response data and return them. + var reportData = responses.map(function(x) { return self._parse(x); }); + callback(reportData); + }) +}; + +CrowdTangleContentService.prototype._httpRequest = function(params, callback) { + request(params, callback); +}; + +CrowdTangleContentService.prototype._completeUrl = function() { + // add one second to last report date to start fetching from + var startDate = new Date(this._lastReportDate.getTime() + 1000).toISOString(); + return url.format({ + protocol: 'https', + hostname: this._baseUrl, + pathname: this._pathName, + query: { + token: this._apiToken, + count: this._count, + language: this._language, + listIds: this._listIds, + searchTerm: this._keywords, + sortBy: this._sortParameter, + startDate: startDate, + endDate: new Date().toISOString() + } + }); +} + +CrowdTangleContentService.prototype._parse = function(data) { + + var metadata = { + sponsor: data.brandedContentSponsor || null, + caption: data.caption || null, + description: data.description || null, + title: data.title || null, + crowdtangleId: data.id || null, + externalUrl: data.link || null, + platform: data.platform || null, + type: data.type || null, + accountVerified: data.account ? data.account.verified : false, + accountHandle: data.account ? data.account.handle : null, + subscriberCount: data.account ? data.account.subscriberCount : 0, + accountUrl: data.account ? data.account.url : null, + mediaUrl: data.media? data.media.map(function(medium) { + return {type: medium.type, url: medium.url} + }) : null, + actualStatistics: data.statistics.actual, + expectedStatistics: data.statistics.expected, + rawAPIResponse: data + }; + var text = data.message || data.description || data.title || data.caption || "[No Content]"; // TODO need to revisit, what if there is no text? what about youtube case + var author = data.account ? data.account.name || data.account.handle : null; + + // This code deals specifically with matching a crowdtangle list to a report's account id + this.crowdtangle_lists = crowdtangle_lists.crowdtangle_list_account_pairs; + // If the list is found and matched, then the ct_tag is the list name + if (this.crowdtangle_lists[data.account.id]) { + metadata.ct_tag = this.crowdtangle_lists[data.account.id]; + } else { + // If the list is not found and not matched, make the ct_tag the account.id so we can identify it later. + metadata.ct_tag = data.account.id; + } + + return { + authoredAt: new Date(data.date + " UTC") || new Date(), + fetchedAt: new Date(), + content: text, + author: author, + metadata: metadata, + url: data.postUrl, + //_sources: '', //need to get this info from somewhere + //_sourceNicknames: '' + }; +} + + +CrowdTangleContentService.prototype.reloadSettings = function() { + this._baseUrl = config.get().crowdtangle.baseUrl; + this._pathName = config.get().crowdtangle.pathName; + this._count = config.get().crowdtangle.count; + this._language = config.get().crowdtangle.language; +}; + +module.exports = CrowdTangleContentService; \ No newline at end of file diff --git a/lib/fetching/content-services/rss-content-service.js b/lib/fetching/content-services/rss-content-service.js index 0636537c0..df2a34f50 100644 --- a/lib/fetching/content-services/rss-content-service.js +++ b/lib/fetching/content-services/rss-content-service.js @@ -8,6 +8,7 @@ var util = require('util'); var _ = require('underscore'); var logger = require('../../logger'); + // options.url - The URL of the RSS feed. // options.lastReportDate - The fetchedAt time of the last already fetched report (optional). var RSSContentService = function(options) { diff --git a/lib/fetching/content-services/twitter-content-service.js b/lib/fetching/content-services/twitter-content-service.js index c981f1503..3397fc9c7 100644 --- a/lib/fetching/content-services/twitter-content-service.js +++ b/lib/fetching/content-services/twitter-content-service.js @@ -122,7 +122,8 @@ TwitterContentService.prototype._parse = function(data) { latitude: data.coordinates ? data.coordinates[0] : 0, longitude: data.coordinates ? data.coordinates[1] : 0, retweetCount: data.retweet_count ? data.retweet_count : 0, - favouriteCount: data.favorite_count ? data.favorite_count : 0 + favouriteCount: data.favorite_count ? data.favorite_count : 0, + rawAPIResponse: data }; return { authoredAt: new Date(data.created_at), diff --git a/models/source.js b/models/source.js index d0c045b96..b84faab91 100644 --- a/models/source.js +++ b/models/source.js @@ -25,7 +25,7 @@ var urlValidator = function(url) { ) } -var mediaValues = ['facebook', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'dummy-pull', 'dummy-fast']; +var mediaValues = ['facebook', 'crowdtangle', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'dummy-pull', 'dummy-fast']; var sourceSchema = new mongoose.Schema({ media: { type: String, enum: mediaValues }, diff --git a/package-lock.json b/package-lock.json index f4d238631..b4d612796 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2117,11 +2117,6 @@ "domelementtype": "1" } }, - "dotenv": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-4.0.0.tgz", - "integrity": "sha1-hk7xN5rO1Vzm+V3r7NzhefegzR0=" - }, "duplexer2": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", @@ -5612,58 +5607,6 @@ "to-regex": "^3.0.2" } }, - "migrate": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/migrate/-/migrate-1.6.2.tgz", - "integrity": "sha512-XAFab+ArPTo9BHzmihKjsZ5THKRryenA+lwob0R+ax0hLDs7YzJFJT5YZE3gtntZgzdgcuFLs82EJFB/Dssr+g==", - "requires": { - "chalk": "^1.1.3", - "commander": "^2.9.0", - "dateformat": "^2.0.0", - "dotenv": "^4.0.0", - "inherits": "^2.0.3", - "minimatch": "^3.0.3", - "mkdirp": "^0.5.1", - "slug": "^0.9.2" - }, - "dependencies": { - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - }, - "mkdirp": { - "version": "0.5.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", - "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", - "requires": { - "minimist": "^1.2.5" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, "miller-rabin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", @@ -8544,14 +8487,6 @@ "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", "integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" }, - "slug": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/slug/-/slug-0.9.4.tgz", - "integrity": "sha512-3YHq0TeJ4+AIFbJm+4UWSQs5A1mmeWOTQqydW3OoPmQfNKxlO96NDRTIrp+TBkmvEsEFrd+Z/LXw8OD/6OlZ5g==", - "requires": { - "unicode": ">= 0.3.1" - } - }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -9648,11 +9583,6 @@ "resolved": "https://registry.npmjs.org/undertaker-registry/-/undertaker-registry-1.0.1.tgz", "integrity": "sha1-XkvaMI5KiirlhPm5pDWaSZglzFA=" }, - "unicode": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/unicode/-/unicode-12.1.0.tgz", - "integrity": "sha512-Ty6+Ew21DiYTWLYtd05RF/X4c1ekOvOgANyHbBj0h3MaXpfaGr2Rdmc0hMFuGQLyPLb9cU4ArNxl0bTF5HSzXw==" - }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", diff --git a/package.json b/package.json index 09c38f080..12e4f9d7f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,6 @@ "locale": "^0.1.0", "lodash": "^4.17.15", "merge-stream": "^2.0.0", - "migrate": "^1.6.2", "mkdirp": "^1.0.4", "moment": "^2.25.3", "mongoose": "^5.9.16", diff --git a/public/angular/images/crowdtangle.png b/public/angular/images/crowdtangle.png new file mode 100644 index 000000000..5de9d5393 Binary files /dev/null and b/public/angular/images/crowdtangle.png differ diff --git a/public/angular/js/config.js b/public/angular/js/config.js index 8c24a98df..ae42c0815 100644 --- a/public/angular/js/config.js +++ b/public/angular/js/config.js @@ -1,6 +1,6 @@ angular.module('Aggie') -.value('mediaOptions', ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp']) +.value('mediaOptions', ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp', 'crowdtangle']) .value('apiSettingsOptions', ['twitter', 'facebook', 'elmo', 'gplaces']) diff --git a/public/angular/js/controllers/incidents/show.js b/public/angular/js/controllers/incidents/show.js index 2a1fcb4cb..43b37d6ae 100644 --- a/public/angular/js/controllers/incidents/show.js +++ b/public/angular/js/controllers/incidents/show.js @@ -91,14 +91,20 @@ angular.module('Aggie') // Pick one of the sources that has a media type. For now, it happens that // if a report has multiple sources, they all have the same type, or are // deleted - for (var i = 0; i < report._sources.length; i++) { - var sourceId = report._sources[i]; - var source = $scope.sourcesById[sourceId]; - if (source && $scope.mediaOptions[source.media] !== -1) { - return source.media + '-source'; + + if (report.metadata.platform === "Facebook") { + // set Facebook as source for CrowdTangle reports + return 'facebook-source'; + } else { + for (var i = 0; i < report._sources.length; i++) { + var sourceId = report._sources[i]; + var source = $scope.sourcesById[sourceId]; + if (source && $scope.mediaOptions[source.media] !== -1) { + return source.media + '-source'; + } } + return 'unknown-source'; } - return 'unknown-source'; }; $scope.delete = function() { diff --git a/public/angular/js/controllers/reports/index.js b/public/angular/js/controllers/reports/index.js index 53aa78af3..6673a5d54 100644 --- a/public/angular/js/controllers/reports/index.js +++ b/public/angular/js/controllers/reports/index.js @@ -53,6 +53,7 @@ angular.module('Aggie') end: 0 }; + var init = function() { $scope.reportsById = $scope.reports.reduce(groupById, {}); $scope.sourcesById = $scope.sources.reduce(groupById, {}); @@ -75,7 +76,9 @@ angular.module('Aggie') }; var linkify = function(report) { - report.content = Autolinker.link(report.content); + if (report.content !== null) { + report.content = Autolinker.link(report.content); + } return report; }; @@ -372,14 +375,20 @@ angular.module('Aggie') // Pick one of the sources that has a media type. For now, it happens that // if a report has multiple sources, they all have the same type, or are // deleted - for (var i = 0; i < report._sources.length; i++) { - var sourceId = report._sources[i]; - var source = $scope.sourcesById[sourceId]; - if (source && $scope.mediaOptions[source.media] !== -1) { - return source.media + '-source'; + + if (report.metadata.platform === "Facebook") { + // set Facebook as source for CrowdTangle reports + return 'facebook-source'; + } else { + for (var i = 0; i < report._sources.length; i++) { + var sourceId = report._sources[i]; + var source = $scope.sourcesById[sourceId]; + if (source && $scope.mediaOptions[source.media] !== -1) { + return source.media + '-source'; + } } + return 'unknown-source'; } - return 'unknown-source'; }; $scope.$on('$destroy', function() { diff --git a/public/angular/js/controllers/sources/index.js b/public/angular/js/controllers/sources/index.js index 9882a1e10..e62218093 100644 --- a/public/angular/js/controllers/sources/index.js +++ b/public/angular/js/controllers/sources/index.js @@ -19,11 +19,13 @@ angular.module('Aggie') }; $scope.target = function(source) { - return source.media == 'twitter' || source.media == 'smsgh' || source.media == 'whatsapp' ? source.keywords : source.url; + return source.media == 'twitter' || source.media == 'smsgh' || source.media == 'whatsapp' || source.media == 'crowdtangle' ? source.keywords : source.url; }; $scope.sourceClass = function(source) { - var mediaOptions = ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp']; + + var mediaOptions = ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp', 'crowdtangle']; + if (mediaOptions.indexOf(source.media) !== -1) { return source.media + '-source'; } else { diff --git a/public/angular/sass/components/tables.scss b/public/angular/sass/components/tables.scss index 5024955d9..160ff5a8a 100644 --- a/public/angular/sass/components/tables.scss +++ b/public/angular/sass/components/tables.scss @@ -174,6 +174,10 @@ tbody { background: url('/images/whatsapp.png') no-repeat 0 50%; } +.crowdtangle-source .icon-left { + background: url('/images/crowdtangle.png') no-repeat 0 50%; +} + .strong { font-weight: bold; } diff --git a/public/angular/templates/settings.html b/public/angular/templates/settings.html index c564cbb55..2fac91539 100644 --- a/public/angular/templates/settings.html +++ b/public/angular/templates/settings.html @@ -26,6 +26,7 @@

API Authentication

+
diff --git a/public/angular/templates/sources/modal.html b/public/angular/templates/sources/modal.html index 2ce07655f..c48f74c03 100644 --- a/public/angular/templates/sources/modal.html +++ b/public/angular/templates/sources/modal.html @@ -39,19 +39,19 @@