Add Tag field to Run table and controller

paddles/controllers/runs.py

@@ -1,6 +1,6 @@
 import logging
 import datetime
-from sqlalchemy import Date, cast
+from sqlalchemy import Date, cast, UnicodeText
 from sqlalchemy.exc import InvalidRequestError, OperationalError
 
 from pecan import abort, conf, expose, request
@@ -15,8 +15,17 @@
 date_format = '%Y-%m-%d'
 
 
-def latest_runs(fields=None, count=conf.default_latest_runs_count, page=1):
-    query = Run.query.order_by(Run.posted.desc())
+def latest_runs(fields=None, count=conf.default_latest_runs_count, page=1, tag=None):
+    query = Run.query
+    if tag:
+        # JSONType uses process_bind_param (json.dumps) for bind params and
+        # process_result_value (json.loads) when reading rows. The read path
+        # is fine; the filter RHS here is a bind param, so it would be
+        # JSON-encoded unless we compare as plain text:
+        tags_text = cast(Run.tags, UnicodeText)
+        for t in tag.split(','):
+            query = query.filter(tags_text.contains('"%s"' % t.strip()))
+    query = query.order_by(Run.posted.desc())
     query = offset_query(query, page_size=count, page=page)
     runs = list(query)
     if fields:
@@ -71,6 +80,26 @@ def index(self):
         json_run['jobs'] = self.run.get_jobs()
         return json_run
 
+    @index.when(method='PUT', template='json')
+    def index_put(self):
+        if not self.run:
+            error('/errors/not_found/',
+                  'attempted to update a non-existent run')
+        try:
+            data = request.json
+        except ValueError:
+            rollback()
+            error('/errors/invalid/', 'could not decode JSON body')
+
+        if 'tags' in data:
+            tags = data['tags']
+            if not isinstance(tags, list):
+                error('/errors/invalid/', "'tags' must be a list of strings")
+            self.run.tags = tags
+
+        log.info("Updated run: %r", self.run)
+        return {}
+
     @index.when(method='DELETE', template='json')
     def index_delete(self):
         if not self.run:
@@ -150,6 +179,25 @@ def get_lookup_controller(self):
         return SuiteController
 
 
+class TagsController(RunFilterIndexController):
+    def get_subquery(self, query):
+        return query.values(Run.tags)
+
+    @expose('json')
+    @retryOperation
+    def index(self):
+        query = request.context.get('query', Run.query)
+        all_tags = set()
+        for (tags_value,) in self.get_subquery(query):
+            if tags_value:
+                for tag in tags_value:
+                    all_tags.add(tag)
+        return sorted(all_tags)
+
+    def get_lookup_controller(self):
+        return TagController
+
+
 class UsersController(RunFilterIndexController):
     def get_subquery(self, query):
         return query.values(Run.user)
@@ -202,6 +250,8 @@ def get_lookup_controller(self, field):
             return StatusesController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'user':
             return UsersController()
         if field == 'flavor':
@@ -232,6 +282,8 @@ def get_lookup_controller(self, field):
             return Sha1sController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'user':
             return UsersController()
         if field == 'flavor':
@@ -253,6 +305,8 @@ def get_lookup_controller(self, field):
             return Sha1sController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'user':
             return UsersController()
         if field == 'flavor':
@@ -274,6 +328,8 @@ def get_lookup_controller(self, field):
             return Sha1sController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'user':
             return UsersController()
         if field == 'flavor':
@@ -295,6 +351,34 @@ def get_lookup_controller(self, field):
             return Sha1sController()
         if field == 'status':
             return StatusesController()
+        if field == 'tag':
+            return TagsController()
+        if field == 'user':
+            return UsersController()
+        if field == 'flavor':
+            return FlavorsController()
+
+
+class TagController(RunFilterController):
+    def get_subquery(self, query):
+        # Same as latest_runs: compare as UnicodeText so the filter bind param
+        # is not run through JSONType.process_bind_param (json.dumps).
+        return query.filter(
+            cast(Run.tags, UnicodeText).contains('"%s"' % self.value))
+
+    def get_lookup_controller(self, field):
+        if field == 'branch':
+            return BranchesController()
+        if field == 'date':
+            return DatesController()
+        if field == 'machine_type':
+            return MachineTypesController()
+        if field == 'sha1':
+            return Sha1sController()
+        if field == 'status':
+            return StatusesController()
+        if field == 'suite':
+            return SuitesController()
         if field == 'user':
             return UsersController()
         if field == 'flavor':
@@ -318,6 +402,8 @@ def get_lookup_controller(self, field):
             return StatusesController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'flavor':
             return FlavorsController()
 
@@ -339,6 +425,8 @@ def get_lookup_controller(self, field):
             return StatusesController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'user':
             return UsersController()
 
@@ -393,6 +481,8 @@ def get_lookup_controller(self, field):
             return StatusesController()
         if field == 'suite':
             return SuitesController()
+        if field == 'tag':
+            return TagsController()
         if field == 'user':
             return UsersController()
         if field == 'flavor':
@@ -402,31 +492,36 @@ def get_lookup_controller(self, field):
 
 class RunsController(object):
     @expose(generic=True, template='json')
-    def index(self, fields='', count=conf.default_latest_runs_count, page=1):
-        return latest_runs(fields=fields, count=count, page=page)
+    def index(self, fields='', count=conf.default_latest_runs_count, page=1, tag=None):
+        return latest_runs(fields=fields, count=count, page=page, tag=tag)
 
     @index.when(method='POST', template='json')
     def index_post(self):
-        # save to DB here
         try:
             name = request.json.get('name')
+            tags = request.json.get('tags')
         except ValueError:
             rollback()
             error('/errors/invalid/', 'could not decode JSON body')
+
         if not name:
             error('/errors/invalid/', "could not find required key: 'name'")
+
+        if tags is not None and not isinstance(tags, list):
+            error('/errors/invalid/', "'tags' must be a list of strings")
+
         if not Run.query.filter_by(name=name).first():
-            self._create_run(name)
+            self._create_run(name, tags=tags)
             return dict()
         else:
             error('/errors/invalid/', "run with name %s already exists" % name)
 
     @classmethod
     @retryOperation
-    def _create_run(cls, name):
-        log.info("Creating run: %s", name)
+    def _create_run(cls, name, tags=None):
+        log.info("Creating run: %s with tags: %r", name, tags)
         Session.flush()
-        return Run(name)
+        return Run(name, tags=tags)
 
     branch = BranchesController()
 
@@ -438,6 +533,8 @@ def _create_run(cls, name):
 
     suite = SuitesController()
 
+    tag = TagsController()
+
     queued = QueuedRunsController()
 
     sha1 = Sha1sController()

paddles/models/runs.py

@@ -8,6 +8,7 @@
 from paddles.util import local_datetime_to_utc
 from paddles.models import Base, TEUTHOLOGY_TIMESTAMP_FMT
 from paddles.models.jobs import Job
+from paddles.models.types import JSONType
 
 suite_names = ['big',
                'ceph-deploy',
@@ -110,6 +111,7 @@ class Run(Base):
     machine_type = Column(String(32), index=True)
     posted = Column(DateTime, index=True)
     started = Column(DateTime, index=True)
+    tags = Column(JSONType(), nullable=True)
     updated = Column(DateTime, index=True)
     jobs = relationship('Job',
                         backref=backref('run'),
@@ -128,8 +130,9 @@ class Run(Base):
                         'finished fail',
                         )
 
-    def __init__(self, name):
+    def __init__(self, name, tags=None):
         self.name = name
+        self.tags = tags or []
         self.posted = datetime.utcnow()
         parsed_name = self.parse_name()
         self.user = parsed_name.get('user', '')
@@ -154,6 +157,7 @@ def __json__(self):
         status = self.set_status(results)
         return dict(
             name=self.name,
+            tags=self.tags or [],
             href=self.href,
             user=self.user,
             status=status,

paddles/tests/controllers/test_runs.py

@@ -222,6 +222,112 @@ def test_queued_partially(self):
         result_names = sorted([r['name'] for r in result])
         assert result_names == sorted(run_names)
 
+    def test_create_run_with_tags(self):
+        self.app.post_json('/runs/', dict(name="tagged_run",
+                                          tags=["squid"]))
+        new_run = Run.get(1)
+        assert new_run.tags == ['squid']
+
+    def test_create_run_with_multiple_tags(self):
+        self.app.post_json('/runs/', dict(name="multi_tag",
+                                          tags=["squid", "tracker-123"]))
+        new_run = Run.get(1)
+        assert sorted(new_run.tags) == ['squid', 'tracker-123']
+
+    def test_create_run_without_tags(self):
+        self.app.post_json('/runs/', dict(name="untagged_run"))
+        new_run = Run.get(1)
+        assert new_run.tags == []
+
+    def test_create_run_tags_must_be_list(self):
+        response = self.app.post_json('/runs/',
+                                      dict(name="bad", tags="not-a-list"),
+                                      expect_errors=True)
+        assert response.status_int == 400
+
+    def test_tags_in_json_response(self):
+        self.app.post_json('/runs/', dict(name="foo",
+                                          tags=["reef", "squid"]))
+        response = self.app.get('/runs/')
+        assert sorted(response.json[0]['tags']) == ['reef', 'squid']
+
+    def test_filter_by_tag_query_param(self):
+        self.app.post_json('/runs/', dict(name="run_a", tags=["reef"]))
+        self.app.post_json('/runs/', dict(name="run_b", tags=["squid"]))
+        self.app.post_json('/runs/', dict(name="run_c"))
+        response = self.app.get('/runs/?tag=reef')
+        assert len(response.json) == 1
+        assert response.json[0]['name'] == 'run_a'
+
+    def test_filter_by_tag_query_param_multi(self):
+        self.app.post_json('/runs/', dict(name="run_a",
+                                          tags=["reef", "squid"]))
+        self.app.post_json('/runs/', dict(name="run_b", tags=["squid"]))
+        self.app.post_json('/runs/', dict(name="run_c", tags=["tentacle"]))
+        response = self.app.get('/runs/?tag=squid')
+        assert len(response.json) == 2
+        got_names = sorted(r['name'] for r in response.json)
+        assert got_names == ['run_a', 'run_b']
+
+    def test_filter_by_comma_separated_tags(self):
+        self.app.post_json('/runs/', dict(name="run_a",
+                                          tags=["reef", "squid"]))
+        self.app.post_json('/runs/', dict(name="run_b", tags=["squid"]))
+        self.app.post_json('/runs/', dict(name="run_c", tags=["tentacle"]))
+        response = self.app.get('/runs/?tag=reef,squid')
+        assert len(response.json) == 1
+        assert response.json[0]['name'] == 'run_a'
+
+    def test_filter_by_tag_url_path(self):
+        self.app.post_json('/runs/', dict(name="run_a", tags=["reef"]))
+        self.app.post_json('/runs/', dict(name="run_b", tags=["squid"]))
+        self.app.post_json('/runs/', dict(name="run_c"))
+        response = self.app.get('/runs/tag/reef/')
+        assert len(response.json) == 1
+        assert response.json[0]['name'] == 'run_a'
+
+    def test_filter_by_tag_url_path_multi(self):
+        self.app.post_json('/runs/', dict(name="run_a",
+                                          tags=["reef", "tentacle"]))
+        self.app.post_json('/runs/', dict(name="run_b",
+                                          tags=["squid", "tentacle"]))
+        response = self.app.get('/runs/tag/tentacle/')
+        assert len(response.json) == 2
+
+    def test_list_tags(self):
+        self.app.post_json('/runs/', dict(name="run_a",
+                                          tags=["reef", "tentacle"]))
+        self.app.post_json('/runs/', dict(name="run_b",
+                                          tags=["squid", "tentacle"]))
+        self.app.post_json('/runs/', dict(name="run_c"))
+        response = self.app.get('/runs/tag/')
+        assert sorted(response.json) == ['reef', 'squid', 'tentacle']
+
+    def test_put_tags(self):
+        self.app.post_json('/runs/', dict(name="foo"))
+        self.app.put_json('/runs/foo/', dict(tags=["new-tag"]))
+        response = self.app.get('/runs/foo/')
+        assert response.json['tags'] == ['new-tag']
+
+    def test_put_tags_append(self):
+        self.app.post_json('/runs/', dict(name="foo", tags=["old"]))
+        self.app.put_json('/runs/foo/', dict(tags=["old", "new"]))
+        response = self.app.get('/runs/foo/')
+        assert sorted(response.json['tags']) == ['new', 'old']
+
+    def test_put_tags_must_be_list(self):
+        self.app.post_json('/runs/', dict(name="foo"))
+        response = self.app.put_json('/runs/foo/',
+                                     dict(tags="not-a-list"),
+                                     expect_errors=True)
+        assert response.status_int == 400
+
+    def test_put_nonexistent_run(self):
+        response = self.app.put_json('/runs/nope/',
+                                     dict(tags=["x"]),
+                                     expect_errors=True)
+        assert response.status_int == 400
+
 
 class TestRunControllerDateFilters(TestApp):