diff --git a/paddles/controllers/runs.py b/paddles/controllers/runs.py index a3cdd93..3c5f7c9 100644 --- a/paddles/controllers/runs.py +++ b/paddles/controllers/runs.py @@ -1,6 +1,6 @@ import logging import datetime -from sqlalchemy import Date, cast +from sqlalchemy import Date, cast, UnicodeText from sqlalchemy.exc import InvalidRequestError, OperationalError from pecan import abort, conf, expose, request @@ -15,8 +15,17 @@ date_format = '%Y-%m-%d' -def latest_runs(fields=None, count=conf.default_latest_runs_count, page=1): - query = Run.query.order_by(Run.posted.desc()) +def latest_runs(fields=None, count=conf.default_latest_runs_count, page=1, tag=None): + query = Run.query + if tag: + # JSONType uses process_bind_param (json.dumps) for bind params and + # process_result_value (json.loads) when reading rows. The read path + # is fine; the filter RHS here is a bind param, so it would be + # JSON-encoded unless we compare as plain text: + tags_text = cast(Run.tags, UnicodeText) + for t in tag.split(','): + query = query.filter(tags_text.contains('"%s"' % t.strip())) + query = query.order_by(Run.posted.desc()) query = offset_query(query, page_size=count, page=page) runs = list(query) if fields: @@ -71,6 +80,26 @@ def index(self): json_run['jobs'] = self.run.get_jobs() return json_run + @index.when(method='PUT', template='json') + def index_put(self): + if not self.run: + error('/errors/not_found/', + 'attempted to update a non-existent run') + try: + data = request.json + except ValueError: + rollback() + error('/errors/invalid/', 'could not decode JSON body') + + if 'tags' in data: + tags = data['tags'] + if not isinstance(tags, list): + error('/errors/invalid/', "'tags' must be a list of strings") + self.run.tags = tags + + log.info("Updated run: %r", self.run) + return {} + @index.when(method='DELETE', template='json') def index_delete(self): if not self.run: @@ -150,6 +179,25 @@ def get_lookup_controller(self): return SuiteController +class TagsController(RunFilterIndexController): + def get_subquery(self, query): + return query.values(Run.tags) + + @expose('json') + @retryOperation + def index(self): + query = request.context.get('query', Run.query) + all_tags = set() + for (tags_value,) in self.get_subquery(query): + if tags_value: + for tag in tags_value: + all_tags.add(tag) + return sorted(all_tags) + + def get_lookup_controller(self): + return TagController + + class UsersController(RunFilterIndexController): def get_subquery(self, query): return query.values(Run.user) @@ -202,6 +250,8 @@ def get_lookup_controller(self, field): return StatusesController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'user': return UsersController() if field == 'flavor': @@ -232,6 +282,8 @@ def get_lookup_controller(self, field): return Sha1sController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'user': return UsersController() if field == 'flavor': @@ -253,6 +305,8 @@ def get_lookup_controller(self, field): return Sha1sController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'user': return UsersController() if field == 'flavor': @@ -274,6 +328,8 @@ def get_lookup_controller(self, field): return Sha1sController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'user': return UsersController() if field == 'flavor': @@ -295,6 +351,34 @@ def get_lookup_controller(self, field): return Sha1sController() if field == 'status': return StatusesController() + if field == 'tag': + return TagsController() + if field == 'user': + return UsersController() + if field == 'flavor': + return FlavorsController() + + +class TagController(RunFilterController): + def get_subquery(self, query): + # Same as latest_runs: compare as UnicodeText so the filter bind param + # is not run through JSONType.process_bind_param (json.dumps). + return query.filter( + cast(Run.tags, UnicodeText).contains('"%s"' % self.value)) + + def get_lookup_controller(self, field): + if field == 'branch': + return BranchesController() + if field == 'date': + return DatesController() + if field == 'machine_type': + return MachineTypesController() + if field == 'sha1': + return Sha1sController() + if field == 'status': + return StatusesController() + if field == 'suite': + return SuitesController() if field == 'user': return UsersController() if field == 'flavor': @@ -318,6 +402,8 @@ def get_lookup_controller(self, field): return StatusesController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'flavor': return FlavorsController() @@ -339,6 +425,8 @@ def get_lookup_controller(self, field): return StatusesController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'user': return UsersController() @@ -393,6 +481,8 @@ def get_lookup_controller(self, field): return StatusesController() if field == 'suite': return SuitesController() + if field == 'tag': + return TagsController() if field == 'user': return UsersController() if field == 'flavor': @@ -402,31 +492,36 @@ def get_lookup_controller(self, field): class RunsController(object): @expose(generic=True, template='json') - def index(self, fields='', count=conf.default_latest_runs_count, page=1): - return latest_runs(fields=fields, count=count, page=page) + def index(self, fields='', count=conf.default_latest_runs_count, page=1, tag=None): + return latest_runs(fields=fields, count=count, page=page, tag=tag) @index.when(method='POST', template='json') def index_post(self): - # save to DB here try: name = request.json.get('name') + tags = request.json.get('tags') except ValueError: rollback() error('/errors/invalid/', 'could not decode JSON body') + if not name: error('/errors/invalid/', "could not find required key: 'name'") + + if tags is not None and not isinstance(tags, list): + error('/errors/invalid/', "'tags' must be a list of strings") + if not Run.query.filter_by(name=name).first(): - self._create_run(name) + self._create_run(name, tags=tags) return dict() else: error('/errors/invalid/', "run with name %s already exists" % name) @classmethod @retryOperation - def _create_run(cls, name): - log.info("Creating run: %s", name) + def _create_run(cls, name, tags=None): + log.info("Creating run: %s with tags: %r", name, tags) Session.flush() - return Run(name) + return Run(name, tags=tags) branch = BranchesController() @@ -438,6 +533,8 @@ def _create_run(cls, name): suite = SuitesController() + tag = TagsController() + queued = QueuedRunsController() sha1 = Sha1sController() diff --git a/paddles/models/runs.py b/paddles/models/runs.py index 92d8757..00fc906 100644 --- a/paddles/models/runs.py +++ b/paddles/models/runs.py @@ -8,6 +8,7 @@ from paddles.util import local_datetime_to_utc from paddles.models import Base, TEUTHOLOGY_TIMESTAMP_FMT from paddles.models.jobs import Job +from paddles.models.types import JSONType suite_names = ['big', 'ceph-deploy', @@ -110,6 +111,7 @@ class Run(Base): machine_type = Column(String(32), index=True) posted = Column(DateTime, index=True) started = Column(DateTime, index=True) + tags = Column(JSONType(), nullable=True) updated = Column(DateTime, index=True) jobs = relationship('Job', backref=backref('run'), @@ -128,8 +130,9 @@ class Run(Base): 'finished fail', ) - def __init__(self, name): + def __init__(self, name, tags=None): self.name = name + self.tags = tags or [] self.posted = datetime.utcnow() parsed_name = self.parse_name() self.user = parsed_name.get('user', '') @@ -154,6 +157,7 @@ def __json__(self): status = self.set_status(results) return dict( name=self.name, + tags=self.tags or [], href=self.href, user=self.user, status=status, diff --git a/paddles/tests/controllers/test_runs.py b/paddles/tests/controllers/test_runs.py index c9867d9..80af12a 100644 --- a/paddles/tests/controllers/test_runs.py +++ b/paddles/tests/controllers/test_runs.py @@ -222,6 +222,112 @@ def test_queued_partially(self): result_names = sorted([r['name'] for r in result]) assert result_names == sorted(run_names) + def test_create_run_with_tags(self): + self.app.post_json('/runs/', dict(name="tagged_run", + tags=["squid"])) + new_run = Run.get(1) + assert new_run.tags == ['squid'] + + def test_create_run_with_multiple_tags(self): + self.app.post_json('/runs/', dict(name="multi_tag", + tags=["squid", "tracker-123"])) + new_run = Run.get(1) + assert sorted(new_run.tags) == ['squid', 'tracker-123'] + + def test_create_run_without_tags(self): + self.app.post_json('/runs/', dict(name="untagged_run")) + new_run = Run.get(1) + assert new_run.tags == [] + + def test_create_run_tags_must_be_list(self): + response = self.app.post_json('/runs/', + dict(name="bad", tags="not-a-list"), + expect_errors=True) + assert response.status_int == 400 + + def test_tags_in_json_response(self): + self.app.post_json('/runs/', dict(name="foo", + tags=["reef", "squid"])) + response = self.app.get('/runs/') + assert sorted(response.json[0]['tags']) == ['reef', 'squid'] + + def test_filter_by_tag_query_param(self): + self.app.post_json('/runs/', dict(name="run_a", tags=["reef"])) + self.app.post_json('/runs/', dict(name="run_b", tags=["squid"])) + self.app.post_json('/runs/', dict(name="run_c")) + response = self.app.get('/runs/?tag=reef') + assert len(response.json) == 1 + assert response.json[0]['name'] == 'run_a' + + def test_filter_by_tag_query_param_multi(self): + self.app.post_json('/runs/', dict(name="run_a", + tags=["reef", "squid"])) + self.app.post_json('/runs/', dict(name="run_b", tags=["squid"])) + self.app.post_json('/runs/', dict(name="run_c", tags=["tentacle"])) + response = self.app.get('/runs/?tag=squid') + assert len(response.json) == 2 + got_names = sorted(r['name'] for r in response.json) + assert got_names == ['run_a', 'run_b'] + + def test_filter_by_comma_separated_tags(self): + self.app.post_json('/runs/', dict(name="run_a", + tags=["reef", "squid"])) + self.app.post_json('/runs/', dict(name="run_b", tags=["squid"])) + self.app.post_json('/runs/', dict(name="run_c", tags=["tentacle"])) + response = self.app.get('/runs/?tag=reef,squid') + assert len(response.json) == 1 + assert response.json[0]['name'] == 'run_a' + + def test_filter_by_tag_url_path(self): + self.app.post_json('/runs/', dict(name="run_a", tags=["reef"])) + self.app.post_json('/runs/', dict(name="run_b", tags=["squid"])) + self.app.post_json('/runs/', dict(name="run_c")) + response = self.app.get('/runs/tag/reef/') + assert len(response.json) == 1 + assert response.json[0]['name'] == 'run_a' + + def test_filter_by_tag_url_path_multi(self): + self.app.post_json('/runs/', dict(name="run_a", + tags=["reef", "tentacle"])) + self.app.post_json('/runs/', dict(name="run_b", + tags=["squid", "tentacle"])) + response = self.app.get('/runs/tag/tentacle/') + assert len(response.json) == 2 + + def test_list_tags(self): + self.app.post_json('/runs/', dict(name="run_a", + tags=["reef", "tentacle"])) + self.app.post_json('/runs/', dict(name="run_b", + tags=["squid", "tentacle"])) + self.app.post_json('/runs/', dict(name="run_c")) + response = self.app.get('/runs/tag/') + assert sorted(response.json) == ['reef', 'squid', 'tentacle'] + + def test_put_tags(self): + self.app.post_json('/runs/', dict(name="foo")) + self.app.put_json('/runs/foo/', dict(tags=["new-tag"])) + response = self.app.get('/runs/foo/') + assert response.json['tags'] == ['new-tag'] + + def test_put_tags_append(self): + self.app.post_json('/runs/', dict(name="foo", tags=["old"])) + self.app.put_json('/runs/foo/', dict(tags=["old", "new"])) + response = self.app.get('/runs/foo/') + assert sorted(response.json['tags']) == ['new', 'old'] + + def test_put_tags_must_be_list(self): + self.app.post_json('/runs/', dict(name="foo")) + response = self.app.put_json('/runs/foo/', + dict(tags="not-a-list"), + expect_errors=True) + assert response.status_int == 400 + + def test_put_nonexistent_run(self): + response = self.app.put_json('/runs/nope/', + dict(tags=["x"]), + expect_errors=True) + assert response.status_int == 400 + class TestRunControllerDateFilters(TestApp): diff --git a/paddles/tests/models/test_runs.py b/paddles/tests/models/test_runs.py index 4a4efb7..4c696e2 100644 --- a/paddles/tests/models/test_runs.py +++ b/paddles/tests/models/test_runs.py @@ -255,3 +255,25 @@ def test_run_flavor(self): Job(dict(job_id=1, id=1, status='queued', flavor="blah"), new_run) run_result = new_run.get_results() assert run_result.get('flavor') == "blah" + + def test_run_tags(self): + new_run = Run("run_with_tags", tags=["tracker-456"]) + assert new_run.tags == ["tracker-456"] + + def test_run_multiple_tags(self): + new_run = Run("run_multi_tags", tags=["alpha", "beta"]) + assert sorted(new_run.tags) == ["alpha", "beta"] + + def test_run_tags_default_empty(self): + new_run = Run("run_without_tags") + assert new_run.tags == [] + + def test_run_tags_in_json(self): + new_run = Run("run_tags_json", tags=["my-tag"]) + run_json = new_run.__json__() + assert run_json['tags'] == ["my-tag"] + + def test_run_tags_empty_in_json(self): + new_run = Run("run_tags_empty_json") + run_json = new_run.__json__() + assert run_json['tags'] == []