CrowdTangle Integration

.gitignore

@@ -44,5 +44,4 @@ config/cert.pem
 
 config/key.pem
 
-docs/_build
-
+docs/_build

config/crowdtangle_list.json

@@ -0,0 +1,3 @@
+{
+	"crowdtangle_list_account_pairs":{}
+}

config/secrets.json.example

@@ -89,4 +89,4 @@
   "experimentFile": "test/end-to-end/fixtures/experiment_reports.json",
   "adminPassword": "letmein1",
   "adminEmail": "aggie-admin@example.com"
-}
+}

lib/fetching/bot.js

@@ -105,6 +105,9 @@ Bot.prototype._reportListener = function(reportData) {
       if (reportData._media === 'facebook') {
         reportData.metadata.isComment ? reportData.tags.push('FBComment') : reportData.tags.push('FBPost');
       }
+      if (reportData._media === 'crowdtangle') {
+        true ? reportData.tags.push(reportData.metadata.ct_tag) : reportData.tags.push('Untagged');
+      }
     }
 
     var drops = this.queue.drops;

lib/fetching/bots/pull-bot.js

@@ -8,7 +8,7 @@ var logger = require('../../logger');
 // options.source - The source to pull from.
 // options.contentService - The contentService to control.
 var PullBot = function(options) {
-  this.interval = options.interval || 120000;
+  this.interval = options.contentService.interval || 120000;
   Bot.call(this, options);
 };
 

lib/fetching/content-service-factory.js

@@ -15,11 +15,13 @@ contentServices['dummy-fast'] = require('./content-services/dummy-fast-content-s
 contentServices.smsgh = require('./content-services/smsgh-content-service');
 contentServices.whatsapp = require('./content-services/whatsapp-content-service');
 contentServices.replay = require('./content-services/replay-content-service');
+contentServices.crowdtangle = require('./content-services/crowd-tangle-content-service');
 
 function ContentServiceFactory() { /* empty constructor */ }
 
 // Creates a new content service to match the given source.
 ContentServiceFactory.prototype.create = function(source) {
+
   var Service = contentServices[source.media];
   // Content services use only the lastReportDate, url, and keywords params.
   var basicSource = _.pick(source, 'lastReportDate', 'url', 'keywords');

lib/fetching/content-services/crowd-tangle-content-service.js

@@ -0,0 +1,155 @@
+var request = require('request');
+var url = require('url');
+
+var ContentService = require('../content-service');
+var util = require('util');
+var config = require('../../../config/secrets');
+var crowdtangle_lists = require('../../../config/crowdtangle_list');
+
+var request = require('request');
+var _ = require('underscore');
+
+
+//options.lastReportDate is passed here through the content-service-factory and will be utilised in calling the api, but its actually only maintained by the parent content service, it is only utilised by the child
+var CrowdTangleContentService = function(options) {
+	this._baseUrl = config.get().crowdtangle.baseUrl;
+	this._pathName = config.get().crowdtangle.pathName;
+	this._count = config.get().crowdtangle.count;
+	this._language = config.get().crowdtangle.language;
+	this._sortParameter = config.get().crowdtangle.sortParam;
+	this._apiToken = config.get().crowdtangle.apiToken;
+	this._keywords = options.keywords;
+	this._lastReportDate = options._lastReportDate;
+	this._listIds = parseInt(options.tags);
+	this.fetchType = 'pull';
+	this.interval = 30000;
+	ContentService.call(this, options); //associates the child service with its parent, which has the notion of lastreportdate
+}
+
+
+util.inherits(CrowdTangleContentService, ContentService);
+
+//this method overwrites the _doFetch method in the content-service (which is the parent class) 
+//and then inside the content-service, we have a fetch method that is called by the pull-bot and is responsible for emitting (writing) the reports and updating the final value of lastReportDate
+//options.maxCount (present but not used, used by the parent content service)
+//what about types? -- discuss with Michael
+CrowdTangleContentService.prototype._doFetch = function(options, callback) {
+	var self = this;
+ 	//handle errors using process.nextTick
+ 	// this will depend mostly on the options that we expect to receive, and if there is issue with that
+
+	// if (!this._url) {
+	// 	process.nextTick(function() { self.emit('error', new Error('Missing URL')); });
+	// 	return callback([]);
+	// }
+
+	  //now we need to submit the request
+		this._httpRequest( {url: this._completeUrl(options)}, function(err, res, body) {
+			if (err) {
+				self.emit('error', new Error('HTTP error: ' + err.message));
+				return callback([]);
+			} else if (res.statusCode != 200) {
+				self.emit('error', new Error.HTTP(res.statusCode));
+				return callback([]);
+			}
+
+			//if no errors, parse the body..
+			var responses;
+			try {
+				responses = JSON.parse(body).result.posts;
+				if (!(responses instanceof Array)) {
+					self.emit('error', new Error('Wrong data'));
+					return callback([]);
+				}
+				// any other error handling wrt the structure of responses
+			} catch (e) {
+				self.emit('error', new Error('Parse error: ' + e.message));
+				return callback([]);
+			}
+			// the responses will be sorted by the content-service (parent method)
+			// Parse response data and return them.
+			var reportData = responses.map(function(x) { return self._parse(x); });
+			callback(reportData);
+		})
+};
+
+CrowdTangleContentService.prototype._httpRequest = function(params, callback) {
+	request(params, callback);
+};
+
+CrowdTangleContentService.prototype._completeUrl = function() {
+	// add one second to last report date to start fetching from
+	var startDate = new Date(this._lastReportDate.getTime() + 1000).toISOString();
+	return url.format({
+		protocol: 'https',
+		hostname: this._baseUrl,
+		pathname: this._pathName,
+		query: {
+			token: this._apiToken,
+	    	count: this._count,
+	    	language: this._language,
+	    	listIds: this._listIds, 
+	    	searchTerm: this._keywords,
+	    	sortBy: this._sortParameter,
+	    	startDate: startDate,
+			endDate: new Date().toISOString()
+		}
+	});
+}
+
+CrowdTangleContentService.prototype._parse = function(data) {
+
+	var metadata = {
+		sponsor: data.brandedContentSponsor	|| null,
+		caption: data.caption || null,
+		description: data.description || null,
+		title: data.title || null,
+		crowdtangleId: data.id || null,
+		externalUrl: data.link || null,
+		platform: data.platform || null,
+		type: data.type || null,
+		accountVerified: data.account ? data.account.verified : false,
+		accountHandle: data.account ? data.account.handle : null,
+		subscriberCount: data.account ? data.account.subscriberCount : 0,
+		accountUrl: data.account ? data.account.url : null,
+		mediaUrl: data.media? data.media.map(function(medium) {
+			return {type: medium.type, url: medium.url}
+		}) : null,
+		actualStatistics: data.statistics.actual,
+		expectedStatistics: data.statistics.expected,
+		rawAPIResponse: data
+	};
+	var text = data.message || data.description || data.title || data.caption || "[No Content]"; // TODO need to revisit, what if there is no text? what about youtube case
+	var author = data.account ? data.account.name || data.account.handle : null;
+
+	// This code deals specifically with matching a crowdtangle list to a report's account id
+	this.crowdtangle_lists = crowdtangle_lists.crowdtangle_list_account_pairs;
+	// If the list is found and matched, then the ct_tag is the list name
+	if (this.crowdtangle_lists[data.account.id]) {
+		metadata.ct_tag = this.crowdtangle_lists[data.account.id];
+	} else {
+		// If the list is not found and not matched, make the ct_tag the account.id so we can identify it later.
+		metadata.ct_tag = data.account.id;
+	}
+
+	return {
+	    authoredAt: new Date(data.date + " UTC") || new Date(),
+	    fetchedAt: new Date(),
+	    content: text,
+	    author: author,
+	    metadata: metadata,
+	    url: data.postUrl,
+	    //_sources: '', //need to get this info from somewhere
+	    //_sourceNicknames: ''
+  	};
+}
+
+
+CrowdTangleContentService.prototype.reloadSettings = function() {
+	this._baseUrl = config.get().crowdtangle.baseUrl;
+	this._pathName = config.get().crowdtangle.pathName;
+	this._count = config.get().crowdtangle.count;
+	this._language = config.get().crowdtangle.language;
+};
+
+module.exports = CrowdTangleContentService;

lib/fetching/content-services/rss-content-service.js

@@ -8,6 +8,7 @@ var util = require('util');
 var _ = require('underscore');
 var logger = require('../../logger');
 
+
 // options.url - The URL of the RSS feed.
 // options.lastReportDate - The fetchedAt time of the last already fetched report (optional).
 var RSSContentService = function(options) {

lib/fetching/content-services/twitter-content-service.js

@@ -122,7 +122,8 @@ TwitterContentService.prototype._parse = function(data) {
     latitude: data.coordinates ? data.coordinates[0] : 0,
     longitude: data.coordinates ? data.coordinates[1] : 0,
     retweetCount: data.retweet_count ? data.retweet_count : 0,
-    favouriteCount: data.favorite_count ? data.favorite_count : 0
+    favouriteCount: data.favorite_count ? data.favorite_count : 0,
+    rawAPIResponse: data
   };
   return {
     authoredAt: new Date(data.created_at),

models/source.js

@@ -25,7 +25,7 @@ var urlValidator = function(url) {
   )
 }
 
-var mediaValues = ['facebook', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'dummy-pull', 'dummy-fast'];
+var mediaValues = ['facebook', 'crowdtangle', 'elmo', 'twitter', 'rss', 'dummy', 'smsgh', 'whatsapp', 'dummy-pull', 'dummy-fast'];
 
 var sourceSchema = new mongoose.Schema({
   media: { type: String, enum: mediaValues },

package.json

@@ -54,7 +54,6 @@
     "locale": "^0.1.0",
     "lodash": "^4.17.15",
     "merge-stream": "^2.0.0",
-    "migrate": "^1.6.2",
     "mkdirp": "^1.0.4",
     "moment": "^2.25.3",
     "mongoose": "^5.9.16",

public/angular/js/config.js

@@ -1,6 +1,6 @@
 angular.module('Aggie')
 
-.value('mediaOptions', ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp'])
+.value('mediaOptions', ['twitter', 'facebook', 'rss', 'elmo', 'smsgh', 'whatsapp', 'crowdtangle'])
 
 .value('apiSettingsOptions', ['twitter', 'facebook', 'elmo', 'gplaces'])
 

public/angular/js/controllers/incidents/show.js

@@ -91,14 +91,20 @@ angular.module('Aggie')
       // Pick one of the sources that has a media type. For now, it happens that
       // if a report has multiple sources, they all have the same type, or are
       // deleted
-      for (var i = 0; i < report._sources.length; i++) {
-        var sourceId = report._sources[i];
-        var source = $scope.sourcesById[sourceId];
-        if (source && $scope.mediaOptions[source.media] !== -1) {
-          return source.media + '-source';
+
+      if (report.metadata.platform === "Facebook") {
+          // set Facebook as source for CrowdTangle reports
+          return 'facebook-source';
+      } else {
+        for (var i = 0; i < report._sources.length; i++) {
+          var sourceId = report._sources[i];
+          var source = $scope.sourcesById[sourceId];
+          if (source && $scope.mediaOptions[source.media] !== -1) {
+            return source.media + '-source';
+          }
         }
+        return 'unknown-source';
       }
-      return 'unknown-source';
     };
 
     $scope.delete = function() {