diff --git a/.gitignore b/.gitignore index 33e6301bd..4073e3213 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,4 @@ config/cert.pem config/key.pem -docs/_build - +docs/_build \ No newline at end of file diff --git a/config/crowdtangle_list.json b/config/crowdtangle_list.json new file mode 100644 index 000000000..08bd273b8 --- /dev/null +++ b/config/crowdtangle_list.json @@ -0,0 +1,3 @@ +{ + "crowdtangle_list_account_pairs":{} +} diff --git a/config/secrets.json.example b/config/secrets.json.example index 0f494f56d..66fafefcd 100644 --- a/config/secrets.json.example +++ b/config/secrets.json.example @@ -89,4 +89,4 @@ "experimentFile": "test/end-to-end/fixtures/experiment_reports.json", "adminPassword": "letmein1", "adminEmail": "aggie-admin@example.com" -} +} \ No newline at end of file diff --git a/lib/fetching/bot.js b/lib/fetching/bot.js index 2804b3116..d9501e8df 100644 --- a/lib/fetching/bot.js +++ b/lib/fetching/bot.js @@ -105,6 +105,9 @@ Bot.prototype._reportListener = function(reportData) { if (reportData._media === 'facebook') { reportData.metadata.isComment ? reportData.tags.push('FBComment') : reportData.tags.push('FBPost'); } + if (reportData._media === 'crowdtangle') { + true ? reportData.tags.push(reportData.metadata.ct_tag) : reportData.tags.push('Untagged'); + } } var drops = this.queue.drops; diff --git a/lib/fetching/bots/pull-bot.js b/lib/fetching/bots/pull-bot.js index 0bc79563f..e98b6358c 100644 --- a/lib/fetching/bots/pull-bot.js +++ b/lib/fetching/bots/pull-bot.js @@ -8,7 +8,7 @@ var logger = require('../../logger'); // options.source - The source to pull from. // options.contentService - The contentService to control. var PullBot = function(options) { - this.interval = options.interval || 120000; + this.interval = options.contentService.interval || 120000; Bot.call(this, options); }; diff --git a/lib/fetching/content-service-factory.js b/lib/fetching/content-service-factory.js index 74b1ca0e9..fa12464f2 100644 --- a/lib/fetching/content-service-factory.js +++ b/lib/fetching/content-service-factory.js @@ -15,11 +15,13 @@ contentServices['dummy-fast'] = require('./content-services/dummy-fast-content-s contentServices.smsgh = require('./content-services/smsgh-content-service'); contentServices.whatsapp = require('./content-services/whatsapp-content-service'); contentServices.replay = require('./content-services/replay-content-service'); +contentServices.crowdtangle = require('./content-services/crowd-tangle-content-service'); function ContentServiceFactory() { /* empty constructor */ } // Creates a new content service to match the given source. ContentServiceFactory.prototype.create = function(source) { + var Service = contentServices[source.media]; // Content services use only the lastReportDate, url, and keywords params. var basicSource = _.pick(source, 'lastReportDate', 'url', 'keywords'); diff --git a/lib/fetching/content-services/crowd-tangle-content-service.js b/lib/fetching/content-services/crowd-tangle-content-service.js new file mode 100644 index 000000000..1e1d147a8 --- /dev/null +++ b/lib/fetching/content-services/crowd-tangle-content-service.js @@ -0,0 +1,155 @@ +var request = require('request'); +var url = require('url'); + +var ContentService = require('../content-service'); +var util = require('util'); +var config = require('../../../config/secrets'); +var crowdtangle_lists = require('../../../config/crowdtangle_list'); + +var request = require('request'); +var _ = require('underscore'); + + +//options.lastReportDate is passed here through the content-service-factory and will be utilised in calling the api, but its actually only maintained by the parent content service, it is only utilised by the child +var CrowdTangleContentService = function(options) { + this._baseUrl = config.get().crowdtangle.baseUrl; + this._pathName = config.get().crowdtangle.pathName; + this._count = config.get().crowdtangle.count; + this._language = config.get().crowdtangle.language; + this._sortParameter = config.get().crowdtangle.sortParam; + this._apiToken = config.get().crowdtangle.apiToken; + this._keywords = options.keywords; + this._lastReportDate = options._lastReportDate; + this._listIds = parseInt(options.tags); + this.fetchType = 'pull'; + this.interval = 30000; + ContentService.call(this, options); //associates the child service with its parent, which has the notion of lastreportdate +} + + +util.inherits(CrowdTangleContentService, ContentService); + +//this method overwrites the _doFetch method in the content-service (which is the parent class) +//and then inside the content-service, we have a fetch method that is called by the pull-bot and is responsible for emitting (writing) the reports and updating the final value of lastReportDate +//options.maxCount (present but not used, used by the parent content service) +//what about types? -- discuss with Michael +CrowdTangleContentService.prototype._doFetch = function(options, callback) { + var self = this; + //handle errors using process.nextTick + // this will depend mostly on the options that we expect to receive, and if there is issue with that + + // if (!this._url) { + // process.nextTick(function() { self.emit('error', new Error('Missing URL')); }); + // return callback([]); + // } + + //now we need to submit the request + this._httpRequest( {url: this._completeUrl(options)}, function(err, res, body) { + if (err) { + self.emit('error', new Error('HTTP error: ' + err.message)); + return callback([]); + } else if (res.statusCode != 200) { + self.emit('error', new Error.HTTP(res.statusCode)); + return callback([]); + } + + //if no errors, parse the body.. + var responses; + try { + responses = JSON.parse(body).result.posts; + if (!(responses instanceof Array)) { + self.emit('error', new Error('Wrong data')); + return callback([]); + } + // any other error handling wrt the structure of responses + } catch (e) { + self.emit('error', new Error('Parse error: ' + e.message)); + return callback([]); + } + // the responses will be sorted by the content-service (parent method) + // Parse response data and return them. + var reportData = responses.map(function(x) { return self._parse(x); }); + callback(reportData); + }) +}; + +CrowdTangleContentService.prototype._httpRequest = function(params, callback) { + request(params, callback); +}; + +CrowdTangleContentService.prototype._completeUrl = function() { + // add one second to last report date to start fetching from + var startDate = new Date(this._lastReportDate.getTime() + 1000).toISOString(); + return url.format({ + protocol: 'https', + hostname: this._baseUrl, + pathname: this._pathName, + query: { + token: this._apiToken, + count: this._count, + language: this._language, + listIds: this._listIds, + searchTerm: this._keywords, + sortBy: this._sortParameter, + startDate: startDate, + endDate: new Date().toISOString() + } + }); +} + +CrowdTangleContentService.prototype._parse = function(data) { + + var metadata = { + sponsor: data.brandedContentSponsor || null, + caption: data.caption || null, + description: data.description || null, + title: data.title || null, + crowdtangleId: data.id || null, + externalUrl: data.link || null, + platform: data.platform || null, + type: data.type || null, + accountVerified: data.account ? data.account.verified : false, + accountHandle: data.account ? data.account.handle : null, + subscriberCount: data.account ? data.account.subscriberCount : 0, + accountUrl: data.account ? data.account.url : null, + mediaUrl: data.media? data.media.map(function(medium) { + return {type: medium.type, url: medium.url} + }) : null, + actualStatistics: data.statistics.actual, + expectedStatistics: data.statistics.expected, + rawAPIResponse: data + }; + var text = data.message || data.description || data.title || data.caption || "[No Content]"; // TODO need to revisit, what if there is no text? what about youtube case + var author = data.account ? data.account.name || data.account.handle : null; + + // This code deals specifically with matching a crowdtangle list to a report's account id + this.crowdtangle_lists = crowdtangle_lists.crowdtangle_list_account_pairs; + // If the list is found and matched, then the ct_tag is the list name + if (this.crowdtangle_lists[data.account.id]) { + metadata.ct_tag = this.crowdtangle_lists[data.account.id]; + } else { + // If the list is not found and not matched, make the ct_tag the account.id so we can identify it later. + metadata.ct_tag = data.account.id; + } + + return { + authoredAt: new Date(data.date + " UTC") || new Date(), + fetchedAt: new Date(), + content: text, + author: author, + metadata: metadata, + url: data.postUrl, + //_sources: '', //need to get this info from somewhere + //_sourceNicknames: '' + }; +} + + +CrowdTangleContentService.prototype.reloadSettings = function() { + this._baseUrl = config.get().crowdtangle.baseUrl; + this._pathName = config.get().crowdtangle.pathName; + this._count = config.get().crowdtangle.count; + this._language = config.get().crowdtangle.language; +}; + +module.exports = CrowdTangleContentService; \ No newline at end of file diff --git a/lib/fetching/content-services/rss-content-service.js b/lib/fetching/content-services/rss-content-service.js index 0636537c0..df2a34f50 100644 --- a/lib/fetching/content-services/rss-content-service.js +++ b/lib/fetching/content-services/rss-content-service.js @@ -8,6 +8,7 @@ var util = require('util'); var _ = require('underscore'); var logger = require('../../logger'); + // options.url - The URL of the RSS feed. // options.lastReportDate - The fetchedAt time of the last already fetched report (optional). var RSSContentService = function(options) { diff --git a/lib/fetching/content-services/twitter-content-service.js b/lib/fetching/content-services/twitter-content-service.js index c981f1503..3397fc9c7 100644 --- a/lib/fetching/content-services/twitter-content-service.js +++ b/lib/fetching/content-services/twitter-content-service.js @@ -122,7 +122,8 @@ TwitterContentService.prototype._parse = function(data) { latitude: data.coordinates ? data.coordinates[0] : 0, longitude: data.coordinates ? data.coordinates[1] : 0, retweetCount: data.retweet_count ? data.retweet_count : 0, - favouriteCount: data.favorite_count ? data.favorite_count : 0 + favouriteCount: data.favorite_count ? data.favorite_count : 0, + rawAPIResponse: data }; return { authoredAt: new Date(data.created_at), diff --git a/models/source.js b/models/source.js index d0c045b96..b84faab91 100644 --- a/models/source.js +++ b/models/source.js @@ -25,7 +25,7 @@ var urlValidator = function(url) { ) } -var mediaValues = ['facebook', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'dummy-pull', 'dummy-fast']; +var mediaValues = ['facebook', 'crowdtangle', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'dummy-pull', 'dummy-fast']; var sourceSchema = new mongoose.Schema({ media: { type: String, enum: mediaValues }, diff --git a/package-lock.json b/package-lock.json index f4d238631..b4d612796 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2117,11 +2117,6 @@ "domelementtype": "1" } }, - "dotenv": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-4.0.0.tgz", - "integrity": "sha1-hk7xN5rO1Vzm+V3r7NzhefegzR0=" - }, "duplexer2": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", @@ -5612,58 +5607,6 @@ "to-regex": "^3.0.2" } }, - "migrate": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/migrate/-/migrate-1.6.2.tgz", - "integrity": "sha512-XAFab+ArPTo9BHzmihKjsZ5THKRryenA+lwob0R+ax0hLDs7YzJFJT5YZE3gtntZgzdgcuFLs82EJFB/Dssr+g==", - "requires": { - "chalk": "^1.1.3", - "commander": "^2.9.0", - "dateformat": "^2.0.0", - "dotenv": "^4.0.0", - "inherits": "^2.0.3", - "minimatch": "^3.0.3", - "mkdirp": "^0.5.1", - "slug": "^0.9.2" - }, - "dependencies": { - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" - }, - "mkdirp": { - "version": "0.5.5", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", - "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", - "requires": { - "minimist": "^1.2.5" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, "miller-rabin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", @@ -8544,14 +8487,6 @@ "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", "integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" }, - "slug": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/slug/-/slug-0.9.4.tgz", - "integrity": "sha512-3YHq0TeJ4+AIFbJm+4UWSQs5A1mmeWOTQqydW3OoPmQfNKxlO96NDRTIrp+TBkmvEsEFrd+Z/LXw8OD/6OlZ5g==", - "requires": { - "unicode": ">= 0.3.1" - } - }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -9648,11 +9583,6 @@ "resolved": "https://registry.npmjs.org/undertaker-registry/-/undertaker-registry-1.0.1.tgz", "integrity": "sha1-XkvaMI5KiirlhPm5pDWaSZglzFA=" }, - "unicode": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/unicode/-/unicode-12.1.0.tgz", - "integrity": "sha512-Ty6+Ew21DiYTWLYtd05RF/X4c1ekOvOgANyHbBj0h3MaXpfaGr2Rdmc0hMFuGQLyPLb9cU4ArNxl0bTF5HSzXw==" - }, "union-value": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", diff --git a/package.json b/package.json index 09c38f080..12e4f9d7f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,6 @@ "locale": "^0.1.0", "lodash": "^4.17.15", "merge-stream": "^2.0.0", - "migrate": "^1.6.2", "mkdirp": "^1.0.4", "moment": "^2.25.3", "mongoose": "^5.9.16", diff --git a/public/angular/images/crowdtangle.png b/public/angular/images/crowdtangle.png new file mode 100644 index 000000000..5de9d5393 Binary files /dev/null and b/public/angular/images/crowdtangle.png differ diff --git a/public/angular/js/config.js b/public/angular/js/config.js index 8c24a98df..ae42c0815 100644 --- a/public/angular/js/config.js +++ b/public/angular/js/config.js @@ -1,6 +1,6 @@ angular.module('Aggie') -.value('mediaOptions', ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp']) +.value('mediaOptions', ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp', 'crowdtangle']) .value('apiSettingsOptions', ['twitter', 'facebook', 'elmo', 'gplaces']) diff --git a/public/angular/js/controllers/incidents/show.js b/public/angular/js/controllers/incidents/show.js index 2a1fcb4cb..43b37d6ae 100644 --- a/public/angular/js/controllers/incidents/show.js +++ b/public/angular/js/controllers/incidents/show.js @@ -91,14 +91,20 @@ angular.module('Aggie') // Pick one of the sources that has a media type. For now, it happens that // if a report has multiple sources, they all have the same type, or are // deleted - for (var i = 0; i < report._sources.length; i++) { - var sourceId = report._sources[i]; - var source = $scope.sourcesById[sourceId]; - if (source && $scope.mediaOptions[source.media] !== -1) { - return source.media + '-source'; + + if (report.metadata.platform === "Facebook") { + // set Facebook as source for CrowdTangle reports + return 'facebook-source'; + } else { + for (var i = 0; i < report._sources.length; i++) { + var sourceId = report._sources[i]; + var source = $scope.sourcesById[sourceId]; + if (source && $scope.mediaOptions[source.media] !== -1) { + return source.media + '-source'; + } } + return 'unknown-source'; } - return 'unknown-source'; }; $scope.delete = function() { diff --git a/public/angular/js/controllers/reports/index.js b/public/angular/js/controllers/reports/index.js index 53aa78af3..6673a5d54 100644 --- a/public/angular/js/controllers/reports/index.js +++ b/public/angular/js/controllers/reports/index.js @@ -53,6 +53,7 @@ angular.module('Aggie') end: 0 }; + var init = function() { $scope.reportsById = $scope.reports.reduce(groupById, {}); $scope.sourcesById = $scope.sources.reduce(groupById, {}); @@ -75,7 +76,9 @@ angular.module('Aggie') }; var linkify = function(report) { - report.content = Autolinker.link(report.content); + if (report.content !== null) { + report.content = Autolinker.link(report.content); + } return report; }; @@ -372,14 +375,20 @@ angular.module('Aggie') // Pick one of the sources that has a media type. For now, it happens that // if a report has multiple sources, they all have the same type, or are // deleted - for (var i = 0; i < report._sources.length; i++) { - var sourceId = report._sources[i]; - var source = $scope.sourcesById[sourceId]; - if (source && $scope.mediaOptions[source.media] !== -1) { - return source.media + '-source'; + + if (report.metadata.platform === "Facebook") { + // set Facebook as source for CrowdTangle reports + return 'facebook-source'; + } else { + for (var i = 0; i < report._sources.length; i++) { + var sourceId = report._sources[i]; + var source = $scope.sourcesById[sourceId]; + if (source && $scope.mediaOptions[source.media] !== -1) { + return source.media + '-source'; + } } + return 'unknown-source'; } - return 'unknown-source'; }; $scope.$on('$destroy', function() { diff --git a/public/angular/js/controllers/sources/index.js b/public/angular/js/controllers/sources/index.js index 9882a1e10..e62218093 100644 --- a/public/angular/js/controllers/sources/index.js +++ b/public/angular/js/controllers/sources/index.js @@ -19,11 +19,13 @@ angular.module('Aggie') }; $scope.target = function(source) { - return source.media == 'twitter' || source.media == 'smsgh' || source.media == 'whatsapp' ? source.keywords : source.url; + return source.media == 'twitter' || source.media == 'smsgh' || source.media == 'whatsapp' || source.media == 'crowdtangle' ? source.keywords : source.url; }; $scope.sourceClass = function(source) { - var mediaOptions = ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp']; + + var mediaOptions = ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp', 'crowdtangle']; + if (mediaOptions.indexOf(source.media) !== -1) { return source.media + '-source'; } else { diff --git a/public/angular/sass/components/tables.scss b/public/angular/sass/components/tables.scss index 5024955d9..160ff5a8a 100644 --- a/public/angular/sass/components/tables.scss +++ b/public/angular/sass/components/tables.scss @@ -174,6 +174,10 @@ tbody { background: url('/images/whatsapp.png') no-repeat 0 50%; } +.crowdtangle-source .icon-left { + background: url('/images/crowdtangle.png') no-repeat 0 50%; +} + .strong { font-weight: bold; } diff --git a/public/angular/templates/settings.html b/public/angular/templates/settings.html index c564cbb55..2fac91539 100644 --- a/public/angular/templates/settings.html +++ b/public/angular/templates/settings.html @@ -26,6 +26,7 @@
diff --git a/public/angular/templates/sources/modal.html b/public/angular/templates/sources/modal.html
index 2ce07655f..c48f74c03 100644
--- a/public/angular/templates/sources/modal.html
+++ b/public/angular/templates/sources/modal.html
@@ -39,19 +39,19 @@
-
-
Keywords are required for twitter sources.
+ Keywords are required for Twitter sources.
+
diff --git a/public/angular/translations/locale-en.json b/public/angular/translations/locale-en.json
index ba984823b..1a4350363 100644
--- a/public/angular/translations/locale-en.json
+++ b/public/angular/translations/locale-en.json
@@ -20,6 +20,7 @@
"elmo": "Elmo",
"facebook": "Facebook",
"smsgh": "SMS GH",
+ "crowdtangle": "CrowdTangle",
"Smsgh": "SMS GH",
"new": "New",
"working": "Working",
@@ -328,9 +329,9 @@
"Close": "Close",
"Choose source media": "Choose source media",
"Media is required.": "Media is required.",
- "Nickname": "Nickname",
+ "Source Name": "Source Name",
"A short name for this source.": "A short name for this source.",
- "Nickname is required.": "Nickname is required.",
+ "Source Name is required.": "Source Name is required.",
"Keywords": "Keywords",
"Enter tags": "Enter tags",
"sources_modal": {
@@ -355,7 +356,7 @@
"Are you sure you want to delete this trend? (All historical data will be lost)": "Are you sure you want to delete this trend? (All historical data will be lost)",
"Recent Events": "Recent Events",
"Enter keywords": "Enter keywords",
- "Enter a nickname": "Enter a nickname",
+ "Enter a name": "Enter a name",
"Enter url": "Enter url",
"Enter author": "Enter author",
"Enter location": "Enter location",
@@ -398,5 +399,12 @@
"Unescalated": "Unescalated",
"Tags": "Tags",
"Assign To": "Assign To",
- "entry.created": "Created"
+ "entry.created": "Created",
+ "API Key" : "API Key",
+ "Enter dashboard key" : "Enter dashboard key",
+ "Dashboard API Key is required": "Dashboard API Key is required",
+ "The Dashboard API Key for this source.": "The Dashboard API Key for this source",
+ "Nickname": "Nickname",
+ "Nickname is required.": "Nickname is required.",
+ "Enter a nickname": "Enter a nickname"
}
diff --git a/public/angular/translations/locale-es.json b/public/angular/translations/locale-es.json
index 60e0769d2..11ec95a56 100644
--- a/public/angular/translations/locale-es.json
+++ b/public/angular/translations/locale-es.json
@@ -20,6 +20,7 @@
"elmo": "Elmo",
"facebook": "Facebook",
"smsgh": "SMS GH",
+ "crowdtangle": "CrowdTangle",
"Smsgh": "SMS GH",
"new": "Nuevo",
"working": "En proceso",
diff --git a/test/backend/fixtures/ct-0.json b/test/backend/fixtures/ct-0.json
new file mode 100644
index 000000000..0637a088a
--- /dev/null
+++ b/test/backend/fixtures/ct-0.json
@@ -0,0 +1 @@
+[]
\ No newline at end of file
diff --git a/test/backend/fixtures/ct-1.json b/test/backend/fixtures/ct-1.json
new file mode 100644
index 000000000..7f904b284
--- /dev/null
+++ b/test/backend/fixtures/ct-1.json
@@ -0,0 +1,138 @@
+{
+ "status": 200,
+ "result": {
+ "posts": [
+ {
+ "id": 1662400672,
+ "platformId": "93625750491_10156590943270492",
+ "platform": "Facebook",
+ "date": "2016-02-09 15:00:00",
+ "updated": "2016-03-01 02:04:41",
+ "type": "photo",
+ "message": "Happy Birthday to #Phillies Wall of Famer John Kruk!",
+ "expandedLinks": [
+ {
+ "original": "https://www.facebook.com/Phillies/photos/a.188899285491.269976.93625750491/10156590943270492/?type=3",
+ "expanded": "https://www.facebook.com/Phillies/photos/a.188899285491.269976.93625750491/10156590943270492/?type=3"
+ }
+ ],
+ "link": "https://www.facebook.com/Phillies/photos/a.188899285491.269976.93625750491/10156590943270492/?type=3",
+ "postUrl": "https://www.facebook.com/Phillies/posts/10156590943270492",
+ "subscriberCount": 1684347,
+ "score": 8.738522954091817,
+ "media": [
+ {
+ "type": "photo",
+ "url": "https://scontent.xx.fbcdn.net/hphotos-xfp1/v/t1.0-9/p720x720/12728987_10156590943270492_90073343638331316_n.jpg?oh=c7a16bfd8b41e13fdf54e62daa07795c&oe=576AD842",
+ "height": 720,
+ "width": 720,
+ "full": "https://scontent.xx.fbcdn.net/hphotos-xfp1/v/t1.0-9/p720x720/12728987_10156590943270492_90073343638331316_n.jpg?oh=c7a16bfd8b41e13fdf54e62daa07795c&oe=576AD842"
+ }
+ ],
+ "statistics": {
+ "actual": {
+ "likeCount": 11475,
+ "shareCount": 1197,
+ "commentCount": 465,
+ "loveCount": 0,
+ "wowCount": 0,
+ "hahaCount": 0,
+ "sadCount": 0,
+ "angryCount": 0
+ },
+ "expected": {
+ "likeCount": 1966,
+ "shareCount": 160,
+ "commentCount": 80,
+ "loveCount": 0,
+ "wowCount": 0,
+ "hahaCount": 0,
+ "sadCount": 0,
+ "angryCount": 0
+ }
+ },
+ "account": {
+ "id": 992,
+ "name": "Philadelphia Phillies",
+ "handle": "Phillies",
+ "profileImage": "https://scontent.xx.fbcdn.net/v/t1.0-1/p50x50/17554304_10158560861625492_845952810880537491_n.jpg?oh=79c8c102377f9cf19fafada807fcece4&oe=594D229D",
+ "subscriberCount": 1703396,
+ "url": "https://www.facebook.com/93625750491",
+ "platform": "Facebook",
+ "platformId": "93625750491",
+ "verified": true
+ }
+ },
+ {
+ "id": 1420254373,
+ "platformId": "19614945368_10153444965190369",
+ "platform": "Facebook",
+ "date": "2016-01-10 18:00:00",
+ "date": "2017-04-13 03:02:50",
+ "type": "native_video",
+ "message": "My cats are always coming up with new, innovative ways to sit around.",
+ "expandedLinks": [
+ {
+ "original": "https://www.facebook.com/TaylorSwift/videos/10153444965190369/",
+ "expanded": "https://www.facebook.com/TaylorSwift/videos/10153444965190369/"
+ }
+ ],
+ "link": "https://www.facebook.com/TaylorSwift/videos/10153444965190369/",
+ "postUrl": "https://www.facebook.com/TaylorSwift/posts/10153444966305369",
+ "subscriberCount": 73987186,
+ "score": 1.9512645914396887,
+ "media": [
+ {
+ "type": "video",
+ "url": "https://video.xx.fbcdn.net/hvideo-xpf1/v/t42.1790-2/12554346_488798491321974_655798697_n.mp4?efg=eyJybHIiOjY5MywicmxhIjo1MTIsInZlbmNvZGVfdGFnIjoic3ZlX3NkIn0%3D&rl=693&vabr=385&oh=c4b0d0a140392ed1e70467ad873dcfc2&oe=5695EC62",
+ "height": 0,
+ "width": 0
+ },
+ {
+ "type": "photo",
+ "url": "https://scontent.xx.fbcdn.net/hvthumb-xpf1/v/t15.0-10/12474268_154354148269681_1401415196_n.jpg?oh=b115918988612d0127afb88fc9aacfbe&oe=5707855C",
+ "height": 640,
+ "width": 640,
+ "full": "https://scontent.xx.fbcdn.net/hvthumb-xpf1/v/t15.0-10/12474268_154354148269681_1401415196_n.jpg?oh=b115918988612d0127afb88fc9aacfbe&oe=5707855C"
+ }
+ ],
+ "statistics": {
+ "actual": {
+ "likeCount": 311401,
+ "shareCount": 29805,
+ "commentCount": 13871,
+ "loveCount": 2225,
+ "wowCount": 50,
+ "hahaCount": 1168,
+ "sadCount": 4,
+ "angryCount": 10
+ },
+ "expected": {
+ "likeCount": 141778,
+ "shareCount": 4693,
+ "commentCount": 3434,
+ "loveCount": 13820,
+ "wowCount": 609,
+ "hahaCount": 239,
+ "sadCount": 24,
+ "angryCount": 49
+ }
+ },
+ "account": {
+ "id": 9083,
+ "name": "Taylor Swift",
+ "handle": "TaylorSwift",
+ "profileImage": "https://scontent.xx.fbcdn.net/v/t1.0-1/p50x50/12998749_10153659625475369_5264917631114672817_n.jpg?oh=7675ec737f0b429cbf669b6584fe7ebe&oe=5985F4B9",
+ "subscriberCount": 74585525,
+ "url": "https://www.facebook.com/19614945368",
+ "platform": "Facebook",
+ "platformId": "19614945368",
+ "verified": true
+ }
+ }
+ ],
+ "pagination": {
+ "nextPage": "https://api.crowdtangle.com/posts?token=[your-token-here]&filterBy=overperforming&endDate=2016-02-13T00:41&startDate=2016-02-12T18:41:02&count=10&offset=10"
+ }
+ }
+}
\ No newline at end of file
diff --git a/test/backend/fixtures/ct-2.json b/test/backend/fixtures/ct-2.json
new file mode 100644
index 000000000..39c04539c
--- /dev/null
+++ b/test/backend/fixtures/ct-2.json
@@ -0,0 +1,5 @@
+[
+ {
+ "id": 1,
+ "submitter": "Tester",
+ "created_at": "2019-11-07
\ No newline at end of file
diff --git a/test/backend/fixtures/ct-3.json b/test/backend/fixtures/ct-3.json
new file mode 100644
index 000000000..e6d5ffea3
--- /dev/null
+++ b/test/backend/fixtures/ct-3.json
@@ -0,0 +1,8 @@
+{
+ "author" : "tester",
+ "options" : {
+ "apiToken" : "",
+ "keywords" : [],
+ "listIds": []
+ }
+}
\ No newline at end of file
diff --git a/test/backend/lib.fetching.content-service.crowd-tangle-content-service.test.js b/test/backend/lib.fetching.content-service.crowd-tangle-content-service.test.js
new file mode 100644
index 000000000..794cbf3c9
--- /dev/null
+++ b/test/backend/lib.fetching.content-service.crowd-tangle-content-service.test.js
@@ -0,0 +1,104 @@
+var utils = require('./init');
+var expect = require('chai').expect;
+var fs = require('fs');
+var path = require('path');
+var CrowdTangleContentService = require('../../lib/fetching/content-services/crowd-tangle-content-service');
+var ContentService = require('../../lib/fetching/content-service');
+var contentServiceFactory = require('../../lib/fetching/content-service-factory');
+
+
+// Stubs the _httpRequest method of the content service to return the data in the given fixture file.
+function stubWithFixture(fixtureFile, service) {
+
+ // If service is null, creates a CrowdTangleContentService
+ service = service || new CrowdTangleContentService({});
+
+ // Make the stub function return the expected args (err, res, body).
+ fixtureFile = path.join('test', 'backend', 'fixtures', fixtureFile);
+ service._httpRequest = function(params, callback) {
+ callback(null, { statusCode: 200 }, fs.readFileSync(fixtureFile).toString());
+ };
+
+ return service;
+}
+
+describe('CrowdTangle content service', function() {
+
+ it('factory should instantiate correct CrowdTangle content service', function() {
+ var service = new CrowdTangleContentService({});
+ expect(service).to.be.instanceOf(ContentService);
+ expect(service).to.be.instanceOf(CrowdTangleContentService);
+ });
+
+ it('should fetch empty content', function(done) {
+ var service = stubWithFixture('ct-0.json');
+ utils.expectToNotEmitReport(service, done);
+ expect(service._lastReportDate).to.be.undefined;
+ service.once('error', function(err) {
+ done(err);
+ });
+ setTimeout(done, 500);
+ });
+
+ it('should return data from CrowdTangle', function(done) {
+ var service = stubWithFixture('ct-3.json');
+
+ service.once('error', function(err) { done(err); });
+
+ service.on('report', function(reportData) {
+ expect(reportData.metadata.crowdtangleId).to.not.be.undefined;
+ });
+ setTimeout(done, 500);
+ });
+
+
+ it('should fetch mock content from CrowdTangle', function(done) {
+ var service = stubWithFixture('ct-1.json');
+ var fetched = 0;
+
+ service.once('error', function(err) { done(err); });
+
+ service.on('report', function(reportData) {
+ expect(reportData).to.have.property('fetchedAt');
+ expect(reportData).to.have.property('authoredAt');
+ expect(reportData).to.have.property('content');
+ expect(reportData).to.have.property('author');
+ expect(reportData).to.have.property('url');
+ expect(reportData).to.have.property('metadata');
+ switch (++fetched) {
+ case 1:
+ expect(reportData.content).to.contain('Happy Birthday');
+ expect(reportData.author).to.equal('Philadelphia Phillies');
+ expect(reportData.url).to.contain('https');
+ expect(reportData.metadata.type).to.equal('photo');
+ break;
+ case 2:
+ expect(reportData.content).to.contain('innovative ways to sit around');
+ expect(reportData.author).to.equal('Taylor Swift');
+ expect(reportData.metadata.platform).to.equal('Facebook');
+ expect(reportData.metadata.type).to.equal('native_video');
+ break;
+ case 3:
+ return done(new Error('Unexpected report'));
+ }
+ });
+
+ // Give enough time for extra report to appear.
+ process.nextTick(function() { if (fetched == 2) done(); });
+
+ service.fetch({ maxCount: 50 }, function() {});
+ });
+
+ describe('Errors', function() {
+
+ // test for bad data
+ it('should emit json parse error', function(done) {
+ var service = stubWithFixture('ct-2.json');
+ utils.expectToNotEmitReport(service, done);
+ utils.expectToEmitError(service, 'Parse error: Unexpected end of input', done);
+ service.fetch({ maxCount: 50 }, function() {});
+ });
+ });
+
+ after(utils.expectModelsEmpty);
+});
|