Add webserver, webhooks, and specifically GitHub webhook support

csbot.deploy.cfg

@@ -2,7 +2,7 @@
 nickname = Mathison
 auth_method = sasl_plain
 channels = #cs-york #cs-york-dev #compsoc-uk #hacksoc
-plugins = logger linkinfo hoogle imgur csyork usertrack auth topic helix calc mongodb termdates whois xkcd youtube last
+plugins = logger linkinfo hoogle imgur csyork usertrack auth topic helix calc mongodb termdates whois xkcd youtube last webserver webhook github
 
 [linkinfo]
 scan_limit = 2
@@ -37,3 +37,30 @@ end =
 start =
 sep = |
 end =
+
+[webserver]
+host = 0.0.0.0
+port = 80
+
+[github]
+# Re-usable format strings
+fmt.source = [{repository[name]}] {sender[login]}
+fmt.issue_num = issue #{issue[number]}
+fmt.issue_text = {issue[title]} ({issue[html_url]})
+fmt.pr_num = PR #{pull_request[number]}
+fmt.pr_text = {pull_request[title]} ({pull_request[html_url]})
+# Format strings for specific events
+fmt/create = {fmt.source} created {ref_type} {ref} ({repository[html_url]}/tree/{ref})
+fmt/delete = {fmt.source} deleted {ref_type} {ref}
+fmt/issues/* = {fmt.source} {event_subtype} {fmt.issue_num}: {fmt.issue_text}
+fmt/issues/assigned = {fmt.source} {event_subtype} {fmt.issue_num} to {assignee[login]}: {fmt.issue_text}
+fmt/pull_request/* = {fmt.source} {event_subtype} {fmt.pr_num}: {fmt.pr_text}
+fmt/pull_request/assigned = {fmt.source} {event_subtype} {fmt.pr_num} to {assignee[login]}: {fmt.pr_text}
+fmt/pull_request/review_requested = {fmt.source} requested review from {requested_reviewer[login]} on {fmt.pr_num}: {fmt.pr_text}
+fmt/pull_request_review/submitted = {fmt.source} reviewed {fmt.pr_num} ({review_state}): {review[html_url]}
+fmt/push/pushed = {fmt.source} pushed {count} new commit(s) to {short_ref}: {compare}
+fmt/push/forced = {fmt.source} updated {short_ref}: {compare}
+fmt/release/* = {fmt.source} {event_subtype} release {release[name]}: {release[html_url]}
+
+[github/HackSoc/csbot]
+notify = #cs-york-dev

csbot/events.py

@@ -180,6 +180,9 @@ def __init__(self, bot, event_type, data=None):
         self.event_type = event_type
         self.datetime = datetime.now()
 
+    def __str__(self):
+        return f'<Event {self.event_type!r} {self!r}>'
+
     @classmethod
     def extend(cls, event, event_type=None, data=None):
         """Create a new event by extending an existing event.

csbot/plugins/github.py

@@ -0,0 +1,158 @@
+import hmac
+import datetime
+import json
+import string
+from functools import partial
+
+from ..plugin import Plugin
+
+
+class GitHub(Plugin):
+    PLUGIN_DEPENDS = ['webhook']
+
+    CONFIG_DEFAULTS = {
+        'secret': '',
+        'notify': '',
+        'debug_payloads': False,
+        'fmt/*': None,
+    }
+
+    CONFIG_ENVVARS = {
+        'secret': ['GITHUB_WEBHOOK_SECRET'],
+    }
+
+    __sentinel = object()
+
+    def config_get(self, key, repo=None):
+        """A special implementation of :meth:`Plugin.config_get` which looks at
+        a repo-based configuration subsection before the plugin's
+        configuration section.
+        """
+        default = super().config_get(key)
+
+        if repo is None:
+            return default
+        else:
+            return self.subconfig(repo).get(key, default)
+
+    @Plugin.hook('webhook.github')
+    async def webhook(self, e):
+        self.log.info("Handling github webhook")
+        request = e['request']
+        github_event = request.headers['X-GitHub-Event'].lower()
+        payload = await request.read()
+        data = await request.json()
+        repo = data.get("repository", {}).get("full_name", None)
+
+        if self.config_getboolean('debug_payloads'):
+            try:
+                extra = f'-{data["action"]}'
+            except KeyError:
+                extra = ''
+            now = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+            with open(f'github-{github_event}{extra}-{now}.headers.json', 'w') as f:
+                json.dump(dict(request.headers), f, indent=2)
+            with open(f'github-{github_event}{extra}-{now}.payload.json', 'wb') as f:
+                f.write(payload)
+
+        secret = self.config_get('secret', repo)
+        if not secret:
+            self.log.warning('No secret set, not verifying X-Hub-Signature')
+        else:
+            digest = request.headers['X-Hub-Signature']
+            if not self._hmac_compare(secret, payload, digest):
+                self.log.warning('X-Hub-Signature verification failed')
+                return
+        method = getattr(self, f'handle_{github_event}', self.generic_handler)
+        await method(data, github_event)
+
+    async def generic_handler(self, data, event_type, event_subtype=None, event_subtype_key='action', context=None):
+        repo = data.get("repository", {}).get("full_name", None)
+        if event_subtype is None:
+            event_subtype = data.get(event_subtype_key, None)
+        # Build event matchers from least to most specific
+        matchers = ['*']
+        if event_subtype is None:
+            matchers.append(event_type)
+        else:
+            matchers.append(f'{event_type}/*')
+            matchers.append(f'{event_type}/{event_subtype}')
+        # Re-order from most to least specific
+        matchers.reverse()
+        # Most specific event name
+        event_name = matchers[0]
+
+        self.log.info(f'{event_name} event on {repo}')
+
+        fmt = self.find_by_matchers(['fmt/' + m for m in matchers], self.config, None)
+        if not fmt:
+            return
+        formatter = MessageFormatter(partial(self.config_get, repo=repo))
+        format_context = {
+            'event_type': event_type,
+            'event_subtype': event_subtype,
+            'event_name': event_name,
+        }
+        format_context.update(context or {})
+        format_context.update(data)
+        msg = formatter.format(fmt, **format_context)
+        try:
+            notify = self.config_get('notify', repo)
+        except KeyError:
+            return
+        for target in notify.split():
+            self.bot.reply(target, msg)
+
+    async def handle_pull_request(self, data, event_type):
+        if data['action'] == 'closed' and data['pull_request']['merged']:
+            event_subtype = 'merged'
+        else:
+            event_subtype = None
+        return await self.generic_handler(data, event_type, event_subtype)
+
+    async def handle_push(self, data, event_type):
+        context = {
+            'count': len(data['commits']),
+            'short_ref': data['ref'].rsplit('/')[-1],
+        }
+        if data['forced']:
+            event_subtype = 'forced'
+        else:
+            event_subtype = 'pushed'
+        return await self.generic_handler(data, event_type, event_subtype, context=context)
+
+    async def handle_pull_request_review(self, data, event_type):
+        context = {
+            'review_state': data['review']['state'].replace('_', ' '),
+        }
+        return await self.generic_handler(data, event_type, context=context)
+
+    @classmethod
+    def find_by_matchers(cls, matchers, d, default=__sentinel):
+        for m in matchers:
+            if m in d:
+                return d[m]
+        if default is not cls.__sentinel:
+            return default
+        raise KeyError(f'none of {matchers} found')
+
+    def _hmac_digest(self, secret, msg, algorithm):
+        return hmac.new(secret.encode('utf-8'), msg, algorithm).hexdigest()
+
+    def _hmac_compare(self, secret, msg, digest):
+        algorithm, _, signature = digest.partition('=')
+        return hmac.compare_digest(self._hmac_digest(secret, msg, algorithm), signature)
+
+
+class MessageFormatter(string.Formatter):
+    def __init__(self, config_get):
+        self.config_get = config_get
+
+    def get_field(self, field_name, args, kwargs):
+        if field_name.startswith('fmt.'):
+            fmt = self.config_get(field_name)
+            if not fmt:
+                raise KeyError('format not configured: ' + field_name)
+            return self.vformat(fmt, args, kwargs), field_name
+
+        return super().get_field(field_name, args, kwargs)

csbot/plugins/webhook.py

@@ -0,0 +1,30 @@
+from aiohttp import web
+
+from ..plugin import Plugin
+
+
+class Webhook(Plugin):
+    CONFIG_DEFAULTS = {
+        # Prefix for web application
+        'prefix': '/webhook',
+        # Secret for URLs
+        'url_secret': '',
+    }
+
+    CONFIG_ENVVARS = {
+        'url_secret': ['WEBHOOK_SECRET'],
+    }
+
+    @Plugin.hook('webserver.build')
+    def create_app(self, e):
+        with e['webserver'].create_subapp(self.config_get('prefix')) as app:
+            app.add_routes([web.post('/{service}/{url_secret}', self.request_handler)])
+
+    async def request_handler(self, request):
+        if self.config_get('url_secret') != request.match_info['url_secret']:
+            return web.HTTPUnauthorized()
+        event_name = f'webhook.{request.match_info["service"]}'
+        await self.bot.emit_new(event_name, {
+            'request': request,
+        })
+        return web.Response(text="OK")

csbot/plugins/webserver.py

@@ -0,0 +1,46 @@
+from contextlib import contextmanager
+
+from aiohttp import web
+
+from ..plugin import Plugin
+
+
+class WebServer(Plugin):
+    CONFIG_DEFAULTS = {
+        'host': 'localhost',
+        'port': 1337,
+    }
+
+    def setup(self):
+        # Setup server
+        self.bot.loop.run_until_complete(self._build_app())
+        self.bot.loop.run_until_complete(self._start_app())
+
+    async def _build_app(self):
+        self.app = web.Application()
+        await self.bot.emit_new('webserver.build', {
+            'webserver': self,
+        })
+
+    async def _start_app(self):
+        self.app_runner = web.AppRunner(self.app)
+        await self.app_runner.setup()
+        self.site = web.TCPSite(self.app_runner, self.config_get('host'), self.config_get('port'))
+        await self.site.start()
+
+    async def _stop_app(self):
+        await self.app_runner.cleanup()
+        self.app_runner = None
+        self.site = None
+
+    def teardown(self):
+        self.bot.loop.run_until_complete(self._stop_app())
+
+        super().teardown()
+
+    @contextmanager
+    def create_subapp(self, prefix):
+        self.log.info(f'Registering web application at {prefix}')
+        app = web.Application()
+        yield app
+        self.app.add_subapp(prefix, app)

csbot/test/fixtures/github/github-create-20190129-215300.headers.json

@@ -0,0 +1,10 @@
+{
+  "Host": "home.hexi.co:8888",
+  "Accept": "*/*",
+  "User-Agent": "GitHub-Hookshot/640910a",
+  "X-GitHub-Event": "create",
+  "X-GitHub-Delivery": "53cb4d60-2410-11e9-84c8-1fcca054d18f",
+  "Content-Type": "application/json",
+  "X-Hub-Signature": "sha1=d6863fe8773b072e5b11eb1849426481b31388c2",
+  "Content-Length": "6349"
+}

csbot/test/fixtures/github/github-create-20190129-215300.payload.json

@@ -0,0 +1 @@
+{"ref":"alanbriolat-patch-2","ref_type":"branch","master_branch":"master","description":null,"pusher_type":"user","repository":{"id":167863409,"node_id":"MDEwOlJlcG9zaXRvcnkxNjc4NjM0MDk=","name":"csbot-webhook-test","full_name":"alanbriolat/csbot-webhook-test","private":true,"owner":{"login":"alanbriolat","id":12193,"node_id":"MDQ6VXNlcjEyMTkz","avatar_url":"https://avatars0.githubusercontent.com/u/12193?v=4","gravatar_id":"","url":"https://api.github.com/users/alanbriolat","html_url":"https://github.com/alanbriolat","followers_url":"https://api.github.com/users/alanbriolat/followers","following_url":"https://api.github.com/users/alanbriolat/following{/other_user}","gists_url":"https://api.github.com/users/alanbriolat/gists{/gist_id}","starred_url":"https://api.github.com/users/alanbriolat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alanbriolat/subscriptions","organizations_url":"https://api.github.com/users/alanbriolat/orgs","repos_url":"https://api.github.com/users/alanbriolat/repos","events_url":"https://api.github.com/users/alanbriolat/events{/privacy}","received_events_url":"https://api.github.com/users/alanbriolat/received_events","type":"User","site_admin":false},"html_url":"https://github.com/alanbriolat/csbot-webhook-test","description":null,"fork":false,"url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test","forks_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/forks","keys_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/teams","hooks_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/hooks","issue_events_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/issues/events{/number}","events_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/events","assignees_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/assignees{/user}","branches_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/branches{/branch}","tags_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/tags","blobs_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/statuses/{sha}","languages_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/languages","stargazers_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/stargazers","contributors_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/contributors","subscribers_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/subscribers","subscription_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/subscription","commits_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/contents/{+path}","compare_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/merges","archive_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/downloads","issues_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/issues{/number}","pulls_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/pulls{/number}","milestones_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/milestones{/number}","notifications_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/labels{/name}","releases_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/releases{/id}","deployments_url":"https://api.github.com/repos/alanbriolat/csbot-webhook-test/deployments","created_at":"2019-01-27T21:57:19Z","updated_at":"2019-01-29T21:52:57Z","pushed_at":"2019-01-29T21:53:35Z","git_url":"git://github.com/alanbriolat/csbot-webhook-test.git","ssh_url":"git@github.com:alanbriolat/csbot-webhook-test.git","clone_url":"https://github.com/alanbriolat/csbot-webhook-test.git","svn_url":"https://github.com/alanbriolat/csbot-webhook-test","homepage":null,"size":0,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"},"sender":{"login":"alanbriolat","id":12193,"node_id":"MDQ6VXNlcjEyMTkz","avatar_url":"https://avatars0.githubusercontent.com/u/12193?v=4","gravatar_id":"","url":"https://api.github.com/users/alanbriolat","html_url":"https://github.com/alanbriolat","followers_url":"https://api.github.com/users/alanbriolat/followers","following_url":"https://api.github.com/users/alanbriolat/following{/other_user}","gists_url":"https://api.github.com/users/alanbriolat/gists{/gist_id}","starred_url":"https://api.github.com/users/alanbriolat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alanbriolat/subscriptions","organizations_url":"https://api.github.com/users/alanbriolat/orgs","repos_url":"https://api.github.com/users/alanbriolat/repos","events_url":"https://api.github.com/users/alanbriolat/events{/privacy}","received_events_url":"https://api.github.com/users/alanbriolat/received_events","type":"User","site_admin":false}}

csbot/test/fixtures/github/github-create-20190130-101054.headers.json

@@ -0,0 +1,10 @@
+{
+  "Host": "home.hexi.co:8888",
+  "Accept": "*/*",
+  "User-Agent": "GitHub-Hookshot/640910a",
+  "X-GitHub-Event": "create",
+  "X-GitHub-Delivery": "68fd2700-2477-11e9-9943-fd3ebe75d7ac",
+  "Content-Type": "application/json",
+  "X-Hub-Signature": "sha1=7d72f5d3ca735c5579410a75032fbf6c9011660a",
+  "Content-Length": "6333"
+}