diff --git a/config_sample.py b/config_sample.py index 34f30a469..99f609cc1 100755 --- a/config_sample.py +++ b/config_sample.py @@ -18,6 +18,9 @@ INSTALL_FOLDER = '/path/to/installation' SAMPLE_REPOSITORY = '/path/to/samples' SESSION_COOKIE_PATH = '/' +# Where the web console is served, if one is. Password reset links point +# there when it is set, and at these pages when it is not. +CONSOLE_URL = '' FTP_PORT = 21 MAX_CONTENT_LENGTH = 512 * 1024 * 1024 MIN_PWD_LEN = 10 diff --git a/mod_api/__init__.py b/mod_api/__init__.py index cf327dc70..74b2d4fa3 100644 --- a/mod_api/__init__.py +++ b/mod_api/__init__.py @@ -43,3 +43,4 @@ from mod_api.routes import runs as runs_routes # noqa: E402, F401 from mod_api.routes import samples as samples_routes # noqa: E402, F401 from mod_api.routes import system as system_routes # noqa: E402, F401 +from mod_api.routes import uploads as uploads_routes # noqa: E402, F401 diff --git a/mod_api/middleware/auth.py b/mod_api/middleware/auth.py index 0903c3d83..31e53ec74 100644 --- a/mod_api/middleware/auth.py +++ b/mod_api/middleware/auth.py @@ -24,6 +24,11 @@ _PUBLIC_ENDPOINTS = frozenset([ 'api.create_token', # POST /auth/tokens (uses email/password body) 'api.system_health', # GET /system/health (uptime monitoring) + # Account recovery: nobody holds a token at this point, which is the + # whole reason for asking. These are limited per IP instead. + 'api.signup', + 'api.request_password_reset', + 'api.complete_password_reset', ]) diff --git a/mod_api/middleware/rate_limit.py b/mod_api/middleware/rate_limit.py index 222dd0f5e..1b13f2652 100644 --- a/mod_api/middleware/rate_limit.py +++ b/mod_api/middleware/rate_limit.py @@ -60,9 +60,20 @@ def _get_client_ip(): return request.remote_addr +# Endpoints reached without a token, so there is nothing to key on but the +# address. All of them either hand out credentials or send mail to an address +# the caller names, which is worth rationing tightly. +_UNAUTHENTICATED_ENDPOINTS = frozenset([ + 'api.create_token', + 'api.signup', + 'api.request_password_reset', + 'api.complete_password_reset', +]) + + def _get_rate_limit_key(): """Build the rate-limit bucket key for this request.""" - if request.endpoint == 'api.create_token': + if request.endpoint in _UNAUTHENTICATED_ENDPOINTS: return f'ip:{_get_client_ip()}' token = getattr(g, 'api_token', None) if token: @@ -72,7 +83,7 @@ def _get_rate_limit_key(): def _get_limits(): """Return (max_requests, window_seconds) for the current endpoint.""" - if request.endpoint == 'api.create_token': + if request.endpoint in _UNAUTHENTICATED_ENDPOINTS: return 5, 900 if request.method in ('POST', 'DELETE', 'PUT', 'PATCH'): return 20, 60 diff --git a/mod_api/routes/auth.py b/mod_api/routes/auth.py index d397ec2b9..5c06e4664 100644 --- a/mod_api/routes/auth.py +++ b/mod_api/routes/auth.py @@ -1,16 +1,36 @@ """ -Token lifecycle, caller identity, and admin user management. +Token lifecycle, caller identity, account and admin user management. POST /auth/tokens Authenticate with email/password, get a token GET /auth/tokens List tokens (admin-only; ?all=true for all users) DELETE /auth/tokens/current Revoke the token you're currently using DELETE /auth/tokens/{id} Revoke a specific token by ID GET /auth/me Identity, role and scopes behind the token +PATCH /auth/me Change your own name, email or password +POST /auth/signup Send a registration link +POST /auth/password-reset Send a password reset link +POST /auth/password-reset/complete + Set a new password from a reset link +GET /auth/me/ftp-credentials + Your own FTP details for the ingest server +GET /auth/me/github Whether your account is connected to GitHub +DELETE /auth/me/github Forget this platform's copy of that connection GET /users List platform users (admin) +GET /users/{id} One platform user (admin) PATCH /users/{id} Change a user's role (admin) +POST /users/{id}/deactivate Anonymise an account (admin, or your own) +POST /users/{id}/password-reset + Send someone a reset link (admin, or your own) + +Signup and reset send the same emails as the classic pages and reuse their +signed links, so accounts are still created and passwords still set by one +implementation rather than two. """ -from flask import g, request +import hmac +import time + +from flask import g, request, url_for from passlib.apps import custom_app_context as pwd_context from sqlalchemy.exc import IntegrityError @@ -21,13 +41,74 @@ validate_offset_pagination, validate_path_id) from mod_api.models.api_token import DEFAULT_SCOPES, ApiToken, Scope -from mod_api.schemas.auth import (ApiTokenItemSchema, AuthTokenSchema, +from mod_api.schemas.auth import (AccountUpdateSchema, ApiTokenItemSchema, + AuthTokenSchema, EmailOnlySchema, + PasswordResetCompleteSchema, RoleUpdateSchema, TokenCreateRequestSchema) from mod_api.utils import paginated_response, single_response from mod_auth.models import Role, User _DUMMY_HASH = pwd_context.hash('__dummy__') +# Signup and reset links last a day, matching the classic pages. +_LINK_TTL = 86400 + + +def _send(to, subject, text): + """Send one email, logging rather than failing when the mailer says no.""" + if not g.mailer.send_simple_message( + {'to': to, 'subject': subject, 'text': text}): + g.log.error(f'could not send "{subject}" to {to}') + + +def _send_reset_link(user): + """ + Email a password reset link to one user. + + The signature and the template are the classic ones, so a link from + here and a link from the classic page are interchangeable. What cannot + be reused is mod_auth's own send_reset_email: it builds the URL with a + relative endpoint, which resolves against whichever blueprint is + handling the request and so cannot be built from inside this one. + + Where CONSOLE_URL names a web console, the link points there instead, + so somebody who started in the console is not handed to a second site + to finish. The console posts the same three values back to + /auth/password-reset/complete, so the link means the same thing either + way. Unset, which is every existing install, nothing changes. + """ + from mod_auth.controllers import generate_hmac_hash + from run import app + + expires = int(time.time()) + _LINK_TTL + mac = generate_hmac_hash( + app.config.get('HMAC_KEY', ''), + f'{user.id}|{expires}|{user.password}') + console = app.config.get('CONSOLE_URL', '') + if console: + url = (f"{console.rstrip('/')}/reset" + f'?uid={user.id}&expires={expires}&mac={mac}') + else: + url = url_for('auth.complete_reset', uid=user.id, expires=expires, + mac=mac, _external=True) + template = app.jinja_env.get_or_select_template('email/recovery_link.txt') + _send(user.email, + 'CCExtractor CI platform password recovery instructions', + template.render(url=url, name=user.name)) + + +def _invalid_reset_link(): + """ + Build the single reply every bad reset link gets. + + Expired, unknown user and bad signature deliberately share one message: + telling them apart would confirm which accounts exist. + """ + return make_error_response( + 'invalid_link', + 'This reset link is invalid or has expired. Request a new one.', + http_status=400) + @mod_api.route('/auth/tokens', methods=['POST']) @validate_body(TokenCreateRequestSchema) @@ -288,3 +369,315 @@ def update_user_role(user_id, validated_data=None): g.log.info(f'user {user.id} role {previous_role} -> {user.role.value} ' f'by admin {g.api_user.id}') return single_response(_serialize_user(user)) + + +@mod_api.route('/auth/signup', methods=['POST']) +@validate_body(EmailOnlySchema) +def signup(validated_data=None): + """ + Send a registration link to an email address. + + Answers the same way whether or not the address is already registered, + because a different reply here would tell an anonymous caller who has an + account. The link lands on the classic completion page, which is where + the account is actually created: two places able to mint accounts is a + surface worth not having. + """ + from mod_auth.controllers import generate_hmac_hash + from run import app + + email = validated_data['email'] + existing = User.query.filter_by(email=email).first() + + if existing is None: + expires = int(time.time()) + _LINK_TTL + mac = generate_hmac_hash( + app.config.get('HMAC_KEY', ''), f'{email}|{expires}') + url = url_for('auth.complete_signup', email=email, expires=expires, + mac=mac, _external=True) + template = 'email/registration_email.txt' + message = app.jinja_env.get_or_select_template(template).render(url=url) + else: + url = url_for('auth.reset', _external=True) + template = 'email/registration_existing.txt' + message = app.jinja_env.get_or_select_template(template).render( + url=url, name=existing.name) + + _send(email, 'CCExtractor CI platform registration', message) + return single_response({'sent': True}, http_status=202) + + +@mod_api.route('/auth/password-reset', methods=['POST']) +@validate_body(EmailOnlySchema) +def request_password_reset(validated_data=None): + """ + Send a password reset link. + + Silent about whether the address is registered, for the same reason + signup is. + """ + user = User.query.filter_by(email=validated_data['email']).first() + if user is not None: + _send_reset_link(user) + return single_response({'sent': True}, http_status=202) + + +@mod_api.route('/auth/password-reset/complete', methods=['POST']) +@validate_body(PasswordResetCompleteSchema) +def complete_password_reset(validated_data=None): + """ + Set a new password using a link from the reset email. + + The signature covers the current password hash, so a link stops working + the moment it is used or the password changes by any other route. The + caller is not signed in here: they still have to authenticate, which + proves the new password arrived intact. + """ + from mod_auth.controllers import generate_hmac_hash + from run import app + + data = validated_data + if int(time.time()) > data['expires']: + return _invalid_reset_link() + + user = User.query.filter_by(id=data['user_id']).first() + if user is None: + return _invalid_reset_link() + + expected = generate_hmac_hash( + app.config.get('HMAC_KEY', ''), + f"{data['user_id']}|{data['expires']}|{user.password}") + if not hmac.compare_digest(expected, data['mac']): + return _invalid_reset_link() + + user.password = User.generate_hash(data['password']) + g.db.commit() + + template = app.jinja_env.get_or_select_template('email/password_reset.txt') + _send(user.email, 'CCExtractor CI platform password reset', + template.render(name=user.name)) + + g.log.info(f'password reset completed via API for user {user.id}') + return single_response({'user_id': user.id, 'password_changed': True}) + + +@mod_api.route('/auth/me', methods=['PATCH']) +@validate_body(AccountUpdateSchema) +def update_account(validated_data=None): + """ + Change your own name, email or password. + + Touching the email or the password needs the current one as well: the + bearer token proves the request came from a signed-in session, not that + the person sending it knows the account's own credentials. + """ + data = validated_data + if not data or set(data) == {'current_password'}: + return make_error_response( + 'validation_error', 'No fields to update.', http_status=400) + + user = g.api_user + sensitive = {'email', 'new_password'} & set(data) + if sensitive: + current = data.get('current_password') + if not current or not user.is_password_valid(current): + return make_error_response( + 'forbidden', + 'current_password is required and must match to change ' + 'your email or password.', + http_status=403) + + if 'email' in data and data['email'] != user.email: + if User.query.filter_by(email=data['email']).first() is not None: + return make_error_response( + 'conflict', 'That email is already in use.', http_status=409) + user.email = data['email'] + + if 'name' in data: + user.name = data['name'] + if 'new_password' in data: + user.password = User.generate_hash(data['new_password']) + + try: + g.db.commit() + except IntegrityError: + # name and email are both unique, so a race lands here. + g.db.rollback() + return make_error_response( + 'conflict', 'That name or email is already in use.', + http_status=409) + + g.log.info(f'user {user.id} updated their own account: ' + f'{sorted(k for k in data if k != "current_password")}') + return single_response(_serialize_user(user)) + + +@mod_api.route('/users//deactivate', methods=['POST']) +@validate_path_id('user_id') +def deactivate_user(user_id): + """ + Anonymise an account and lock it out. + + The row stays so the uploads and runs it owns keep an author, which is + why this scrubs the identity instead of deleting. Open to admins and to + the account's owner, matching the classic page. + + Scope-free for the same reason revoking your own token is: closing your + own account cannot depend on tokens:manage, which no role below admin is + ever allowed to hold. Ownership is checked below instead. + + The account's own tokens are revoked as part of this. Scrambling the + password only stops new ones being minted, and an admin reaching for + this because somebody is abusing the platform means to end the access + they already have, not to leave it running for up to thirty days. + + Tokens belonging to the caller are untouched unless they are the same + account, so an admin doing this to somebody else keeps working. + """ + caller = g.api_user + if caller.role != Role.admin and caller.id != int(user_id): + return make_error_response( + 'forbidden', 'You can only deactivate your own account.', + http_status=403) + + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + + user.name = f'Anonymous {user.id}' + user.email = f'unknown{user.id}@ccextractor.org' + user.password = User.generate_hash(User.create_random_password(16)) + + revoked = 0 + for token in ApiToken.query.filter(ApiToken.user_id == user.id).all(): + if not token.is_revoked: + token.revoke() + revoked += 1 + g.db.commit() + + g.log.warning(f'user {user.id} deactivated via API by {caller.id}, ' + f'{revoked} token(s) revoked') + return single_response({ + 'user_id': user.id, + 'deactivated': True, + 'tokens_revoked': revoked, + }) + + +@mod_api.route('/users/', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.TOKENS_MANAGE) +@validate_path_id('user_id') +def get_user(user_id): + """Return one platform user.""" + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + return single_response(_serialize_user(user)) + + +@mod_api.route('/users//password-reset', methods=['POST']) +@validate_path_id('user_id') +def send_user_password_reset(user_id): + """ + Send a reset link to another account, or to your own. + + Unlike the public /auth/password-reset this names an account rather than + an address, so it says plainly when the id is unknown: the caller is + already signed in and can list users anyway. + + Scope-free for the same reason deactivation is: asking for a reset on + your own account cannot depend on tokens:manage. + """ + caller = g.api_user + if caller.role != Role.admin and caller.id != int(user_id): + return make_error_response( + 'forbidden', 'You can only request a reset for your own account.', + http_status=403) + + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + + _send_reset_link(user) + + g.log.info(f'password reset link sent to user {user.id} by {caller.id}') + return single_response({'user_id': user.id, 'sent': True}, + http_status=202) + + +@mod_api.route('/auth/me/ftp-credentials', methods=['GET']) +@require_scope(Scope.RUNS_WRITE) +def get_ftp_credentials(): + """ + Return the caller's FTP details, creating them on first ask. + + Only ever the caller's own: these are working credentials, so there is + no version of this that reads someone else's. The password is stored in + the clear by design (it is random and cannot be chosen), so this is the + only place it can come from. + + Behind runs:write rather than open to any token, unlike the rest of + /auth/me: FTP is another way to upload a sample, so a token narrowed to + reading has no business fetching a working credential for it. Every + role holds runs:write, so this narrows tokens without narrowing people. + """ + from mod_upload.controllers import retrieve_ftp_credentials + from run import config + + credentials = retrieve_ftp_credentials(g.api_user.id) + return single_response({ + 'host': config.get('SERVER_NAME', ''), + 'port': config.get('FTP_PORT', ''), + 'username': credentials.user_name, + 'password': credentials.password, + }) + + +@mod_api.route('/auth/me/github', methods=['GET']) +def get_github_link(): + """ + Say whether the caller's account is connected to GitHub. + + Connecting is a browser redirect, so this hands back the URL to send + somebody to rather than performing it. Trading the code GitHub returns + for a token stays on the classic callback: that needs the client + secret and the redirect registered with GitHub, and one place holding + those is better than two. + + The stored token is never part of this response. The URL carries only + the client id and the scope asked for, both of which are public. + """ + from run import config + + user = g.api_user + client_id = config.get('GITHUB_CLIENT_ID', '') + return single_response({ + 'linked': user.github_token is not None, + 'github_login': user.github_login, + 'authorize_url': ( + 'https://github.com/login/oauth/authorize' + f'?client_id={client_id}&scope=public_repo'), + }) + + +@mod_api.route('/auth/me/github', methods=['DELETE']) +def unlink_github(): + """ + Forget this platform's copy of the caller's GitHub connection. + + Only ever the caller's own, like the FTP details above. This drops the + token the platform holds; the authorisation itself is withdrawn from + GitHub's own applications page, which is the only place that can + really end it. + """ + user = g.api_user + user.github_token = None + user.github_login = None + g.db.commit() + + g.log.info(f'user {user.id} disconnected GitHub via API') + return single_response({'linked': False, 'github_login': None}) diff --git a/mod_api/routes/regression_tests.py b/mod_api/routes/regression_tests.py index a113d73d2..f5a645ac2 100644 --- a/mod_api/routes/regression_tests.py +++ b/mod_api/routes/regression_tests.py @@ -5,6 +5,14 @@ POST /regression-tests Create a test (inactive unless asked otherwise) PATCH /regression-tests/{id} Partially update a test DELETE /regression-tests/{id} Delete a test that has never run +GET /regression-tests/{id}/outputs/{oid}/download + Where a baseline lives in storage +GET /regression-tests/{id}/outputs/{oid}/variants/{vid}/download + The same, for an accepted variant +POST /regression-tests/{id}/outputs/{oid}/variants + Accept another output hash as correct +DELETE /regression-tests/{id}/outputs/{oid}/variants/{vid} + Stop accepting one GET /categories List categories with their test counts POST /categories Create a category PATCH /categories/{id} Rename or re-describe a category @@ -29,7 +37,9 @@ from mod_api.schemas.regression_tests import (CategoryCreateSchema, CategoryUpdateSchema, RegressionTestCreateSchema, - RegressionTestUpdateSchema) + RegressionTestUpdateSchema, + VariantCreateSchema) +from mod_api.services.storage import resolve_artifact from mod_api.utils import paginated_response, single_response from mod_auth.models import Role from mod_regression.models import (Category, InputType, OutputType, @@ -89,13 +99,165 @@ def get_regression_test(regression_test_id): 'correct_extension': output.correct_extension, 'expected_filename': output.expected_filename, 'ignore': output.ignore, - 'variants': [f.file_hashes for f in output.multiple_files], + # Ids as well as hashes: the download route addresses a variant + # by id, the same way the classic page does. + 'variants': [ + {'id': f.id, 'hash': f.file_hashes} + for f in output.multiple_files + ], } for output in test.output_files ] return single_response(data) +def _get_output(regression_test_id, output_id): + """ + Look up one expected output, matched against its parent test. + + Filtering on both ids means an output id belonging to a different test + reads as absent rather than resolving, so these URLs cannot be used to + walk the table. Returns (output, error_response). + """ + output = RegressionTestOutput.query.filter_by( + id=output_id, regression_id=regression_test_id).first() + if output is None: + return None, make_error_response( + 'not_found', + f'Output {output_id} not found on regression test ' + f'{regression_test_id}.', + http_status=404) + return output, None + + +def _located(filename): + """ + Report where a TestResults file is, in the artifact response shape. + + Like the sample and run-artifact routes, this hands back a signed URL + instead of the bytes, and says which backend holds the file when there + is no URL to give. + """ + url, status = resolve_artifact(f'TestResults/{filename}') + if status == 'missing': + return make_error_response( + 'not_found', f'{filename} is not present in storage.', + http_status=404) + return single_response({ + 'filename': filename, + 'download_url': url, + 'storage_status': status, + }) + + +@mod_api.route( + '/regression-tests//outputs//download', + methods=['GET'] +) +@require_scope(Scope.RUNS_READ) +@validate_path_id('regression_test_id') +def download_output(regression_test_id, output_id): + """Locate the baseline a regression test is expected to reproduce.""" + output, err = _get_output(regression_test_id, output_id) + if err: + return err + return _located(output.filename_correct) + + +@mod_api.route( + '/regression-tests//outputs/' + '/variants//download', + methods=['GET'] +) +@require_scope(Scope.RUNS_READ) +@validate_path_id('regression_test_id') +def download_variant(regression_test_id, output_id, variant_id): + """Locate one of the alternative outputs accepted for a baseline.""" + output, err = _get_output(regression_test_id, output_id) + if err: + return err + + variant = RegressionTestOutputFiles.query.filter_by( + id=variant_id, regression_test_output_id=output.id).first() + if variant is None: + return make_error_response( + 'not_found', + f'Variant {variant_id} not found on output {output_id}.', + http_status=404) + + return _located(f'{variant.file_hashes}{output.correct_extension}') + + +@mod_api.route( + '/regression-tests//outputs//variants', + methods=['POST'] +) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('regression_test_id') +@validate_body(VariantCreateSchema) +def create_variant(regression_test_id, output_id, validated_data=None): + """ + Accept another output hash as correct for a baseline. + + A test can legitimately produce different bytes on different platforms + or CCExtractor builds. Recording the hash here makes those runs pass + without overwriting the baseline everyone else is compared against, + which is what promoting to baseline would do. + """ + output, err = _get_output(regression_test_id, output_id) + if err: + return err + + file_hash = validated_data['hash'] + existing = RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=output.id, file_hashes=file_hash).first() + if existing is not None: + return make_error_response( + 'conflict', + f'Output {output_id} already accepts {file_hash}.', + http_status=409) + + variant = RegressionTestOutputFiles(file_hash, output.id) + g.db.add(variant) + g.db.commit() + + g.log.info(f'variant {file_hash} added to output {output_id} via API ' + f'by {g.api_user.id}') + return single_response( + {'id': variant.id, 'hash': variant.file_hashes}, http_status=201) + + +@mod_api.route( + '/regression-tests//outputs/' + '/variants/', + methods=['DELETE'] +) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('regression_test_id') +def delete_variant(regression_test_id, output_id, variant_id): + """Stop accepting one alternative output for a baseline.""" + output, err = _get_output(regression_test_id, output_id) + if err: + return err + + variant = RegressionTestOutputFiles.query.filter_by( + id=variant_id, regression_test_output_id=output.id).first() + if variant is None: + return make_error_response( + 'not_found', + f'Variant {variant_id} not found on output {output_id}.', + http_status=404) + + g.db.delete(variant) + g.db.commit() + + g.log.info(f'variant {variant_id} removed from output {output_id} via ' + f'API by {g.api_user.id}') + return single_response({'id': variant_id, 'deleted': True}) + + @mod_api.route('/regression-tests', methods=['POST']) @require_roles([Role.contributor, Role.admin]) @require_scope(Scope.RUNS_WRITE) diff --git a/mod_api/routes/runs.py b/mod_api/routes/runs.py index 1ff75a91d..3d44c6892 100644 --- a/mod_api/routes/runs.py +++ b/mod_api/routes/runs.py @@ -8,6 +8,7 @@ GET /runs/{id}/progress Progress event timeline GET /runs/{id}/config Run configuration and test matrix POST /runs/{id}/cancel Cancel a queued or running test +POST /runs/{id}/restart Clear a run's results so CI picks it up again """ from collections import defaultdict @@ -32,6 +33,7 @@ from mod_api.utils import get_sort_column, paginated_response, single_response from mod_auth.models import Role from mod_customized.models import CustomizedTest +from mod_home.models import CCExtractorVersion from mod_regression.models import RegressionTest, RegressionTestOutput from mod_test.models import (Fork, Test, TestPlatform, TestProgress, TestResult, TestResultFile, TestStatus, TestType) @@ -124,6 +126,20 @@ def _apply_run_filters(query, created_after, created_before): if commit_sha: query = query.filter(Test.commit == commit_sha) + # A release is recorded as the commit it was cut from, so filtering by + # version is the commit filter with one lookup in front of it. + ccx_version = request.args.get('ccx_version') + if ccx_version: + version = CCExtractorVersion.query.filter( + CCExtractorVersion.version == ccx_version).first() + if version is None: + return None, make_error_response( + 'validation_error', + f'Unknown CCExtractor version: {ccx_version}.', + http_status=400, + ) + query = query.filter(Test.commit == version.commit) + repository = request.args.get('repository') if repository: query, err = _apply_repository_filter(query, repository) @@ -675,3 +691,46 @@ def cancel_run(run_id): 'status': 'accepted', 'message': 'Run has been canceled.', }, http_status=202) + + +@mod_api.route('/runs//restart', methods=['POST']) +@require_roles([Role.admin, Role.contributor, Role.tester]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('run_id') +def restart_run(run_id): + """ + Queue a finished or stuck run to be executed again. + + Clearing the results and the progress trail is what makes the run + eligible again, because CI picks up tests that have no progress + recorded. The run keeps its id, so existing links stay valid, and the + old results are replaced rather than kept alongside the new ones. + + Like cancel, this is open to anyone holding runs:write rather than to + the run's owner, which suits a shared CI where a stuck VM blocks + everybody. + """ + # Locked the way cancel locks, so two restarts arriving together do not + # both go clearing the same results. + test = Test.query.with_for_update().filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + TestResultFile.query.filter( + TestResultFile.test_id == test.id).delete(synchronize_session=False) + TestResult.query.filter( + TestResult.test_id == test.id).delete(synchronize_session=False) + TestProgress.query.filter( + TestProgress.test_id == test.id).delete(synchronize_session=False) + g.db.commit() + + g.log.info(f'run {run_id} restarted via API by {g.api_user.id}') + return single_response({ + 'run_id': run_id, + 'action': 'restart', + 'status': 'accepted', + 'message': 'Run has been queued to run again.', + }, http_status=202) diff --git a/mod_api/routes/samples.py b/mod_api/routes/samples.py index 29169c3ad..e83e430aa 100644 --- a/mod_api/routes/samples.py +++ b/mod_api/routes/samples.py @@ -1,33 +1,50 @@ """ Sample and regression test routes. -GET /runs/{id}/samples Per-run regression test results -GET /runs/{id}/samples/{sid} Single result in a run -GET /samples Media sample catalog -GET /samples/{id} Single media sample -GET /samples/{id}/details Upload metadata, extra files, media info -GET /samples/{id}/history Cross-run history for a sample -GET /regression-tests Regression test definitions +GET /runs/{id}/samples Per-run regression test results +GET /runs/{id}/samples/{sid} Single result in a run +GET /samples Media sample catalog +GET /samples/{id} Single media sample +PATCH /samples/{id} Edit tags and upload metadata +DELETE /samples/{id} Delete a sample no test uses +GET /samples/{id}/details Upload metadata, extra files, media info +GET /samples/{id}/download Where the media file lives in storage +GET /samples/{id}/media-info/download + Where the MediaInfo XML lives +GET /samples/{id}/extra-files/{eid}/download + Where an accompanying file lives +DELETE /samples/{id}/extra-files/{eid} + Delete an accompanying file +GET /samples/{id}/history Cross-run history for a sample +GET /tags Tags a sample can be labelled with +POST /tags Create a tag +GET /regression-tests Regression test definitions """ +import os from collections import defaultdict from flask import g, request from sqlalchemy import func +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload, selectinload from mod_api import mod_api -from mod_api.middleware.auth import require_scope +from mod_api.middleware.auth import require_roles, require_scope from mod_api.middleware.error_handler import make_error_response -from mod_api.middleware.validation import (validate_date_range, +from mod_api.middleware.validation import (validate_body, validate_date_range, validate_offset_pagination, validate_path_id) from mod_api.models.api_token import Scope -from mod_api.schemas.samples import SampleHistoryEntrySchema +from mod_api.schemas.samples import (SampleHistoryEntrySchema, + SampleUpdateSchema, TagCreateSchema) from mod_api.services.status import (batch_get_run_data, derive_output_status, derive_sample_status, get_run_timestamps, is_dummy_row) -from mod_api.utils import paginated_response, single_response +from mod_api.services.storage import resolve_artifact +from mod_api.utils import paginated_response, safe_resolve, single_response +from mod_auth.models import Role +from mod_home.models import CCExtractorVersion from mod_regression.models import (Category, RegressionTest, RegressionTestOutput) from mod_sample.media_info_parser import (InvalidMediaInfoError, @@ -35,7 +52,7 @@ from mod_sample.models import ExtraFile, Sample, Tag from mod_test.models import (Test, TestPlatform, TestProgress, TestResult, TestResultFile) -from mod_upload.models import Upload +from mod_upload.models import Platform, Upload # Valid per-sample status values accepted by the ?status filter. Limited to the # statuses derive_sample_status can actually emit, so filtering can't silently @@ -445,6 +462,335 @@ def get_sample(sample_id): }) +@mod_api.route('/samples//download', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def download_sample(sample_id): + """ + Locate a sample's media file in storage. + + Returns a signed URL rather than the bytes. Samples run to several + gigabytes, so streaming one through the API would hold a worker for the + length of the transfer. Where only the local copy exists there is no URL + to hand out and storage_status reports that instead, matching how + /runs/{id}/artifacts describes the same two backends. + """ + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + url, status = resolve_artifact(f'TestFiles/{sample.filename}') + if status == 'missing': + return make_error_response( + 'not_found', + f'Sample {sample_id} is not present in storage.', + http_status=404) + + return single_response({ + 'sample_id': sample.id, + 'filename': sample.filename, + 'download_url': url, + 'storage_status': status, + }) + + +@mod_api.route('/tags', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_tags(limit=50, offset=0): + """List the tags a sample can be labelled with, alphabetically.""" + query = Tag.query.order_by(Tag.name.asc()) + total = query.count() + rows = query.offset(offset).limit(limit).all() + return paginated_response( + [{'id': t.id, 'name': t.name, 'description': t.description or ''} + for t in rows], total, limit, offset) + + +@mod_api.route('/tags', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_body(TagCreateSchema) +def create_tag(validated_data=None): + """Create a tag. Names are unique, so a duplicate is a 409.""" + name = validated_data['name'] + if Tag.query.filter(Tag.name == name).first() is not None: + return make_error_response( + 'conflict', f"Tag '{name}' already exists.", http_status=409) + + tag = Tag(name, validated_data['description']) + g.db.add(tag) + try: + g.db.commit() + except IntegrityError: + # name is unique, so a request that raced the check lands here. + g.db.rollback() + return make_error_response( + 'conflict', f"Tag '{name}' already exists.", http_status=409) + + g.log.info(f'tag {tag.id} created via API by {g.api_user.id}') + return single_response( + {'id': tag.id, 'name': tag.name, 'description': tag.description or ''}, + http_status=201) + + +#: Sample edit fields that live on the upload row rather than the sample. +_UPLOAD_FIELDS = frozenset(['notes', 'parameters', 'platform', 'version']) + + +def _apply_tags(sample, names): + """ + Replace a sample's tags, matching them by name. + + Returns an error response when a name is not a tag, so one typo rejects + the whole request instead of silently dropping a label. + """ + rows = Tag.query.filter(Tag.name.in_(names)).all() + found = {t.name for t in rows} + unknown = [n for n in names if n not in found] + if unknown: + return make_error_response( + 'validation_error', + f"Unknown tags: {', '.join(unknown)}", + details={'fields': {'tags': unknown}}, + http_status=400, + ) + sample.tags = rows + return None + + +def _apply_upload_fields(sample_id, data, requested): + """ + Write the parts of a sample edit that belong to its upload row. + + Returns an error response when there is no upload row to write to, or + when the version named is not one the platform knows. + """ + upload = Upload.query.filter(Upload.sample_id == sample_id).first() + if upload is None: + return make_error_response( + 'conflict', + f'Sample {sample_id} has no upload record, so ' + f"{', '.join(sorted(requested))} cannot be set on it.", + http_status=409, + ) + + if 'notes' in data: + upload.notes = data['notes'] + if 'parameters' in data: + upload.parameters = data['parameters'] + if 'platform' in data: + upload.platform = Platform.from_string(data['platform']) + if 'version' in data: + version = CCExtractorVersion.query.filter( + CCExtractorVersion.version == data['version']).first() + if version is None: + return make_error_response( + 'validation_error', + f"Unknown CCExtractor version: {data['version']}", + http_status=400) + upload.version_id = version.id + return None + + +@mod_api.route('/samples/', methods=['PATCH']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('sample_id') +@validate_body(SampleUpdateSchema) +def update_sample(sample_id, validated_data=None): + """ + Edit a sample's tags and the upload metadata recorded with it. + + Notes, parameters, platform and version live on the upload row rather + than the sample, so a sample that arrived without one (over FTP, or + seeded directly) can only have its tags changed. + """ + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + data = validated_data + if not data: + return make_error_response( + 'validation_error', 'No fields to update.', http_status=400) + + if 'tags' in data: + err = _apply_tags(sample, data['tags']) + if err: + return err + + requested = _UPLOAD_FIELDS & set(data) + if requested: + err = _apply_upload_fields(sample.id, data, requested) + if err: + return err + + g.db.commit() + + g.log.info(f'sample {sample.id} updated via API by {g.api_user.id}: ' + f'{sorted(data.keys())}') + return single_response({ + 'sample_id': sample.id, + 'tags': [t.name for t in sample.tags], + }) + + +def _remove_local(relative_path): + """ + Delete a file under the sample repository, tolerating its absence. + + In production the repository is a gcsfuse mount, so this removes the + stored object too. A path that no longer exists is not an error: the + row is going either way, and refusing would leave the database holding + a sample whose files are already gone. + """ + from run import config + + root = config.get('SAMPLE_REPOSITORY', '') + path = safe_resolve(root, relative_path) + if path is None: + g.log.warning(f'refusing to delete path outside the repository: ' + f'{relative_path}') + return + try: + os.remove(path) + except OSError as e: + g.log.warning(f'could not delete {relative_path}: {e}') + + +@mod_api.route('/samples/', methods=['DELETE']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('sample_id') +def delete_sample(sample_id): + """ + Delete a sample, its extra files and its media info. + + Refused while any regression test still points at it, because removing + the media would leave those tests unable to run and their history + describing a sample nobody can fetch. Detach or delete the tests first. + """ + sample = Sample.query.options(joinedload(Sample.extra_files)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + test_count = RegressionTest.query.filter_by(sample_id=sample.id).count() + if test_count: + return make_error_response( + 'conflict', + f'Sample {sample_id} is used by {test_count} regression ' + f'test(s). Delete those first.', + details={'regression_test_count': test_count}, + http_status=409, + ) + + # Read the paths off the row before it goes, then unlink only once the + # delete is committed: the same order finalize uses, so a failed commit + # cannot leave a sample row pointing at media that is already gone. + paths = [f'TestFiles/extra/{extra.filename}' for extra in + sample.extra_files] + paths.append(f'TestFiles/media/{sample.sha}.xml') + paths.append(f'TestFiles/{sample.filename}') + + g.db.delete(sample) + g.db.commit() + + for path in paths: + _remove_local(path) + + g.log.warning(f'sample {sample_id} deleted via API by {g.api_user.id}') + return single_response({'sample_id': int(sample_id), 'deleted': True}) + + +@mod_api.route( + '/samples//extra-files/', methods=['DELETE']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('sample_id') +def delete_extra_file(sample_id, extra_id): + """Delete one of the files uploaded alongside a sample.""" + extra, err = _get_extra_file(sample_id, extra_id) + if err: + return err + + _remove_local(f'TestFiles/extra/{extra.filename}') + g.db.delete(extra) + g.db.commit() + + g.log.warning(f'extra file {extra_id} of sample {sample_id} deleted via ' + f'API by {g.api_user.id}') + return single_response({'id': extra_id, 'deleted': True}) + + +def _located(sample_id, relative_path, filename, missing_message): + """Report where one of a sample's files is, in the download shape.""" + url, status = resolve_artifact(relative_path) + if status == 'missing': + return make_error_response( + 'not_found', missing_message, http_status=404) + return single_response({ + 'sample_id': sample_id, + 'filename': filename, + 'download_url': url, + 'storage_status': status, + }) + + +@mod_api.route('/samples//media-info/download', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def download_media_info(sample_id): + """ + Locate the MediaInfo XML written alongside a sample. + + The parsed tree is already on /samples/{id}/details; this is for anyone + who wants the original file MediaInfo produced. + """ + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + name = f'{sample.sha}.xml' + return _located( + sample.id, f'TestFiles/media/{name}', name, + f'No media info stored for sample {sample_id}.') + + +def _get_extra_file(sample_id, extra_id): + """Look up one extra file, matched against the sample that owns it.""" + extra = ExtraFile.query.filter_by( + id=extra_id, sample_id=sample_id).first() + if extra is None: + return None, make_error_response( + 'not_found', + f'Extra file {extra_id} not found on sample {sample_id}.', + http_status=404) + return extra, None + + +@mod_api.route( + '/samples//extra-files//download', + methods=['GET'] +) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def download_extra_file(sample_id, extra_id): + """Locate one of the files uploaded alongside a sample.""" + extra, err = _get_extra_file(sample_id, extra_id) + if err: + return err + + return _located( + int(sample_id), f'TestFiles/extra/{extra.filename}', extra.filename, + f'Extra file {extra_id} is not present in storage.') + + def _get_history_failure_signature(result, result_files, status): if status == 'fail': for rf in result_files: diff --git a/mod_api/routes/system.py b/mod_api/routes/system.py index d5b47ec0f..da0d3d6f4 100644 --- a/mod_api/routes/system.py +++ b/mod_api/routes/system.py @@ -4,6 +4,7 @@ GET /system/health Health check (unauthenticated) GET /system/queue Queue status — active + queued runs GET /runs/{id}/artifacts Run artifacts from GCS + local +GET /system/about Versions this platform runs against GET /system/maintenance Maintenance state per platform PATCH /system/maintenance/{platform} Pause or resume a platform GET /system/blocked-users CI users blocked from triggering @@ -45,6 +46,7 @@ from mod_api.utils import paginated_response, safe_resolve, single_response from mod_auth.models import Role from mod_ci.models import BlockedUsers, MaintenanceMode +from mod_home.models import CCExtractorVersion, GeneralData from mod_sample.models import ForbiddenExtension from mod_test.models import (Test, TestPlatform, TestProgress, TestResultFile, TestStatus) @@ -368,6 +370,33 @@ def list_artifacts(run_id, limit=50, offset=0): return paginated_response(paged, total, limit, offset) +@mod_api.route('/system/about', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +def system_about(): + """ + Report the versions this platform is running against. + + The classic about page is static prose. The parts of it worth reading + from a client are the CCExtractor release under test and the commit the + platform itself was deployed from, so those are what this returns. + """ + from run import app + + latest = CCExtractorVersion.query.order_by( + CCExtractorVersion.released.desc()).first() + last_commit = GeneralData.query.filter( + GeneralData.key == 'last_commit').first() + + return single_response({ + 'platform_commit': app.config.get('BUILD_COMMIT'), + 'ccextractor_version': latest.version if latest else None, + 'ccextractor_released': ( + latest.released.isoformat() + if latest and latest.released else None), + 'last_tested_commit': last_commit.value if last_commit else None, + }) + + def _maintenance_entry(platform, row): """Maintenance shape for one platform; no row means never paused.""" return { diff --git a/mod_api/routes/uploads.py b/mod_api/routes/uploads.py new file mode 100644 index 000000000..dd0a002f5 --- /dev/null +++ b/mod_api/routes/uploads.py @@ -0,0 +1,345 @@ +""" +Sample upload and the queue an upload sits in until it is described. + +POST /samples/upload Upload a media file into the queue +GET /queued-samples Uploads waiting to be described +GET /queued-samples/{id} One queued upload +POST /queued-samples/{id}/finalize + Turn a queued upload into a sample +POST /queued-samples/{id}/link Attach one to an existing sample +DELETE /queued-samples/{id} Discard a queued upload + +Uploading and describing are two steps here for the same reason they are on +the classic pages: the transfer is slow and the metadata needs a CCExtractor +version the uploader may have to go and look up, so the bytes are banked +first and the description follows. +""" + +import hashlib +import mimetypes +import os +from uuid import uuid4 + +import magic +from flask import g, request +from werkzeug.utils import secure_filename + +from mod_api import mod_api +from mod_api.middleware.auth import require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.schemas.uploads import QueueLinkSchema, UploadFinalizeSchema +from mod_api.utils import paginated_response, single_response +from mod_auth.models import Role +from mod_home.models import CCExtractorVersion +from mod_sample.models import (ExtraFile, ForbiddenExtension, + ForbiddenMimeType, Sample) +from mod_upload.models import Platform, QueuedSample, Upload + +# Enough of the file for libmagic to identify it, matching the classic form. +_SNIFF_BYTES = 1024 + + +def _repo(*parts): + """Build a path inside the sample repository.""" + from run import config + + return os.path.join(config.get('SAMPLE_REPOSITORY', ''), *parts) + + +def _serialize(queued): + """Public shape of one queued upload.""" + return { + 'id': queued.id, + 'sha': queued.sha, + 'extension': queued.extension, + 'original_name': queued.original_name, + 'user_id': queued.user_id, + } + + +def _forbidden_reason(filename, head): + """ + Say why an upload is not allowed, or None when it is. + + Runs the same three checks as the classic upload form: the extension on + the name, the mime type libmagic reads out of the file, and the + extension that mime type implies. The last one catches a banned format + renamed to get past the first. + """ + extension = os.path.splitext(filename)[1].lstrip('.').lower() + if extension and ForbiddenExtension.query.filter( + ForbiddenExtension.extension == extension).first() is not None: + return f"Files with the '{extension}' extension are not accepted." + + mimetype = magic.from_buffer(head, mime=True) + if ForbiddenMimeType.query.filter( + ForbiddenMimeType.mimetype == mimetype).first() is not None: + return f"Files of type '{mimetype}' are not accepted." + + implied = mimetypes.guess_extension(mimetype) + if implied and ForbiddenExtension.query.filter( + ForbiddenExtension.extension == implied.lstrip('.') + ).first() is not None: + return f"Files of type '{mimetype}' are not accepted." + + return None + + +@mod_api.route('/samples/upload', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +def upload_sample(): + """ + Upload a media file into the queue. + + The file is hashed as it is written, because the hash is both the + duplicate check and the name the sample is stored under, and reading a + multi-gigabyte upload twice to get it would double the cost of every + upload. + + An upload that turns out to be forbidden or already known is deleted + here rather than left in TempFiles for someone to clean up later. + """ + uploaded = request.files.get('file') + if uploaded is None or not uploaded.filename: + return make_error_response( + 'validation_error', + 'Attach the media file as the "file" part of a multipart form.', + http_status=400) + + filename = secure_filename(uploaded.filename) + if not filename: + # secure_filename empties a name made only of dots, separators or + # whitespace, and an empty name resolves to TempFiles itself. + return make_error_response( + 'validation_error', 'Unusable file name.', http_status=400) + + # The staging name belongs to this upload alone rather than to the + # caller's file name: two clients sending the same name at the same + # time would otherwise write into one file and produce a hash that + # describes neither. + temp_path = _repo('TempFiles', f'api-{uuid4().hex}') + + head = uploaded.stream.read(_SNIFF_BYTES) + uploaded.stream.seek(0) + reason = _forbidden_reason(filename, head) + if reason: + g.log.warning(f'user {g.api_user.id} tried to upload {filename}: ' + f'{reason}') + return make_error_response('forbidden', reason, http_status=403) + + digest = hashlib.sha256() + with open(temp_path, 'wb') as out: + for chunk in iter(lambda: uploaded.stream.read(8192), b''): + digest.update(chunk) + out.write(chunk) + sha = digest.hexdigest() + + stored = Sample.query.filter(Sample.sha == sha).first() + queued_already = QueuedSample.query.filter( + QueuedSample.sha == sha).first() + if stored is not None or queued_already is not None: + os.remove(temp_path) + return make_error_response( + 'conflict', + 'A sample with this content is already uploaded or queued.', + details={'sha': sha}, + http_status=409) + + extension = os.path.splitext(filename)[1] + queued = QueuedSample(sha, extension, os.path.splitext(filename)[0], + g.api_user.id) + g.db.add(queued) + g.db.commit() + + os.rename(temp_path, _repo('QueuedFiles', queued.filename)) + + g.log.info(f'sample {sha} queued via API by {g.api_user.id}') + return single_response(_serialize(queued), http_status=201) + + +def _visible_queue(): + """Queued uploads the caller may see: their own, or all for an admin.""" + query = QueuedSample.query.filter( + QueuedSample.user_id == g.api_user.id) + if g.api_user.role == Role.admin: + query = QueuedSample.query + return query.order_by(QueuedSample.id.asc()) + + +@mod_api.route('/queued-samples', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_queued_samples(limit=50, offset=0): + """List uploads waiting to be described. Admins see everyone's.""" + query = _visible_queue() + total = query.count() + rows = query.offset(offset).limit(limit).all() + return paginated_response( + [_serialize(q) for q in rows], total, limit, offset) + + +def _get_queued(queued_id): + """Look up a queued upload the caller is allowed to see.""" + queued = _visible_queue().filter(QueuedSample.id == queued_id).first() + if queued is None: + # Someone else's upload reads as absent rather than forbidden, so + # the id cannot be used to find out what others have queued. + return None, make_error_response( + 'not_found', f'Queued sample {queued_id} not found.', + http_status=404) + return queued, None + + +@mod_api.route('/queued-samples/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('queued_id') +def get_queued_sample(queued_id): + """Return one queued upload.""" + queued, err = _get_queued(queued_id) + if err: + return err + return single_response(_serialize(queued)) + + +@mod_api.route('/queued-samples//finalize', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('queued_id') +@validate_body(UploadFinalizeSchema) +def finalize_queued_sample(queued_id, validated_data=None): + """ + Turn a queued upload into a sample. + + The file only moves into TestFiles once the rows are committed, so a + failure part way through leaves the upload in the queue to retry rather + than a sample row pointing at a file that was never placed. + + Reporting the sample as a GitHub issue, which the classic page offers + here, is left to the classic page: it needs the platform's GitHub token + and has nothing to do with getting the sample into the library. + """ + queued, err = _get_queued(queued_id) + if err: + return err + + data = validated_data + version = CCExtractorVersion.query.filter( + CCExtractorVersion.version == data['version']).first() + if version is None: + return make_error_response( + 'validation_error', + f"Unknown CCExtractor version: {data['version']}", + http_status=400) + + source = _repo('QueuedFiles', queued.filename) + if not os.path.isfile(source): + return make_error_response( + 'conflict', + f'The uploaded file for queued sample {queued_id} is missing ' + f'from storage, so it cannot be finalized.', + http_status=409) + + destination = _repo('TestFiles', queued.filename) + sample = Sample(queued.sha, queued.extension.lstrip('.'), + queued.original_name) + g.db.add(sample) + g.db.flush([sample]) + g.db.add(Upload( + g.api_user.id, sample.id, version.id, + Platform.from_string(data['platform']), + data['parameters'], data['notes'], + )) + g.db.delete(queued) + g.db.commit() + + os.rename(source, destination) + + g.log.info(f'queued sample {queued_id} finalized as sample {sample.id} ' + f'via API by {g.api_user.id}') + return single_response({ + 'sample_id': sample.id, + 'sha': sample.sha, + 'original_name': sample.original_name, + }, http_status=201) + + +@mod_api.route('/queued-samples//link', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('queued_id') +@validate_body(QueueLinkSchema) +def link_queued_sample(queued_id, validated_data=None): + """ + Attach a queued upload to an existing sample as an extra file. + + Used for the material that belongs with a sample without being one, such + as a subtitle track to compare against. The classic page offered this + but never carried it out: its confirm step checks permissions and then + redirects without touching anything, so this is the behaviour it + described rather than the behaviour it had. + + As with finalize, the file moves only after the row is committed, since + the stored name is derived from the id the row is given. + """ + queued, err = _get_queued(queued_id) + if err: + return err + + sample_id = validated_data['sample_id'] + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + source = _repo('QueuedFiles', queued.filename) + if not os.path.isfile(source): + return make_error_response( + 'conflict', + f'The uploaded file for queued sample {queued_id} is missing ' + f'from storage, so it cannot be linked.', + http_status=409) + + extra = ExtraFile(sample.id, queued.extension.lstrip('.'), + queued.original_name) + g.db.add(extra) + g.db.flush([extra]) + filename = extra.filename + g.db.delete(queued) + g.db.commit() + + os.rename(source, _repo('TestFiles', 'extra', filename)) + + g.log.info(f'queued sample {queued_id} linked to sample {sample.id} as ' + f'extra file {extra.id} via API by {g.api_user.id}') + return single_response({ + 'id': extra.id, + 'sample_id': sample.id, + 'filename': filename, + }, http_status=201) + + +@mod_api.route('/queued-samples/', methods=['DELETE']) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('queued_id') +def delete_queued_sample(queued_id): + """Discard a queued upload and the file behind it.""" + queued, err = _get_queued(queued_id) + if err: + return err + + path = _repo('QueuedFiles', queued.filename) + try: + os.remove(path) + except OSError as e: + # The row goes either way: refusing would leave a queue entry + # nobody can clear. + g.log.warning(f'could not delete queued file {path}: {e}') + + g.db.delete(queued) + g.db.commit() + + g.log.warning(f'queued sample {queued_id} discarded via API by ' + f'{g.api_user.id}') + return single_response({'id': int(queued_id), 'deleted': True}) diff --git a/mod_api/schemas/auth.py b/mod_api/schemas/auth.py index b56ece783..b003adaea 100644 --- a/mod_api/schemas/auth.py +++ b/mod_api/schemas/auth.py @@ -1,6 +1,6 @@ """Request/response schemas for the token and user endpoints.""" -from marshmallow import RAISE, Schema, fields, validate +from marshmallow import RAISE, Schema, ValidationError, fields, validate from mod_api.models.api_token import VALID_SCOPES from mod_api.schemas.common import DATETIME_FORMAT @@ -81,3 +81,67 @@ class Meta: """Reject unknown fields.""" unknown = RAISE + + +class EmailOnlySchema(Schema): + """Validates the signup and password reset request bodies.""" + + email = fields.Email(required=True) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +def valid_password(value): + """ + Check a password against the configured length bounds. + + Written as a validator rather than a pre-built Length, because the + bounds come from config and reading those while the schema class is + being defined would import the app mid-blueprint-setup. The classic + forms read the same two keys, so the rules cannot drift apart. + """ + from run import config + + low = int(config.get('MIN_PWD_LEN', 10)) + high = int(config.get('MAX_PWD_LEN', 500)) + if not low <= len(value) <= high: + raise ValidationError( + f'Password must be between {low} and {high} characters.') + + +class PasswordResetCompleteSchema(Schema): + """Validates POST /auth/password-reset/complete bodies.""" + + user_id = fields.Integer(required=True, validate=validate.Range(min=1)) + expires = fields.Integer(required=True) + mac = fields.String( + required=True, validate=validate.Length(min=1, max=128)) + password = fields.String(required=True, validate=valid_password) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class AccountUpdateSchema(Schema): + """ + Validates PATCH /auth/me bodies; every field optional. + + current_password is demanded alongside any change to the credentials + themselves, so a leaked token cannot quietly become a stolen account. + """ + + name = fields.String(validate=validate.Length(min=1, max=50)) + email = fields.Email() + new_password = fields.String(validate=valid_password) + current_password = fields.String( + validate=validate.Length(min=1, max=500)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/regression_tests.py b/mod_api/schemas/regression_tests.py index 49734caba..0ee997696 100644 --- a/mod_api/schemas/regression_tests.py +++ b/mod_api/schemas/regression_tests.py @@ -18,6 +18,9 @@ _NAME_MAX = 64 _DESCRIPTION_MAX = 1024 +# Matches the maxLength the contract publishes for a variant hash. +_HASH_MAX = 128 + class RegressionTestCreateSchema(Schema): """Validates POST /regression-tests bodies.""" @@ -92,3 +95,26 @@ class Meta: """Reject unknown fields.""" unknown = RAISE + + +class VariantCreateSchema(Schema): + """Validates POST /regression-tests/{id}/outputs/{oid}/variants bodies.""" + + # The hash is joined to the baseline's extension to form a filename under + # TestResults, so it is restricted to characters that cannot climb out of + # that directory. Content hashes are hex, so this costs nothing real. + hash = fields.String( + required=True, + validate=[ + validate.Length(min=1, max=_HASH_MAX), + validate.Regexp( + r'^[A-Za-z0-9]+$', + error='hash must be alphanumeric', + ), + ], + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/samples.py b/mod_api/schemas/samples.py index 0b43b012c..7a2de00d7 100644 --- a/mod_api/schemas/samples.py +++ b/mod_api/schemas/samples.py @@ -1,8 +1,16 @@ """Schemas for sample endpoints.""" -from marshmallow import Schema, fields +from marshmallow import RAISE, Schema, fields, validate from mod_api.schemas.common import DATETIME_FORMAT +from mod_upload.models import Platform + +# Read off the model so a platform the database accepts can never be +# rejected here, the way the regression test schemas take their type lists. +_PLATFORMS = sorted(Platform.values()) + +_TAG_NAME_MAX = 64 +_TAG_DESCRIPTION_MAX = 1024 class SampleHistoryEntrySchema(Schema): @@ -16,3 +24,40 @@ class SampleHistoryEntrySchema(Schema): commit_sha = fields.String(required=True) tested_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) failure_signature = fields.String(allow_none=True) + + +class TagCreateSchema(Schema): + """Validates POST /tags bodies.""" + + name = fields.String( + required=True, validate=validate.Length(min=1, max=_TAG_NAME_MAX)) + description = fields.String( + load_default='', validate=validate.Length(max=_TAG_DESCRIPTION_MAX)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class SampleUpdateSchema(Schema): + """ + Validates PATCH /samples/{id} bodies; every field optional. + + Tags are given by name rather than id so a caller can work from the + names the sample payload already shows, instead of first fetching the + tag list to translate them. + """ + + tags = fields.List( + fields.String(validate=validate.Length(min=1, max=_TAG_NAME_MAX)), + validate=validate.Length(max=50)) + notes = fields.String(validate=validate.Length(max=1024)) + parameters = fields.String(validate=validate.Length(max=1024)) + platform = fields.String(validate=validate.OneOf(_PLATFORMS)) + version = fields.String(validate=validate.Length(min=1, max=10)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/uploads.py b/mod_api/schemas/uploads.py new file mode 100644 index 000000000..5de9a311b --- /dev/null +++ b/mod_api/schemas/uploads.py @@ -0,0 +1,40 @@ +"""Request schemas for the upload queue endpoints.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_upload.models import Platform + +# Read off the model so a platform the database accepts is never rejected +# here, the way the sample and regression test schemas do it. +_PLATFORMS = sorted(Platform.values()) + + +class UploadFinalizeSchema(Schema): + """Validates POST /queued-samples/{id}/finalize bodies.""" + + # The version string rather than its id: a client works from what + # /system/about and the sample payloads already show it. + version = fields.String( + required=True, validate=validate.Length(min=1, max=10)) + platform = fields.String( + required=True, validate=validate.OneOf(_PLATFORMS)) + parameters = fields.String( + load_default='', validate=validate.Length(max=1024)) + notes = fields.String( + load_default='', validate=validate.Length(max=1024)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class QueueLinkSchema(Schema): + """Validates POST /queued-samples/{id}/link bodies.""" + + sample_id = fields.Integer(required=True, validate=validate.Range(min=1)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/openapi-ci-api.yaml b/openapi-ci-api.yaml index c3c95c920..c958f6665 100644 --- a/openapi-ci-api.yaml +++ b/openapi-ci-api.yaml @@ -297,6 +297,128 @@ paths: default: $ref: "#/components/responses/Error" + /auth/signup: + post: + tags: [Auth] + summary: Send a registration link to an email address + operationId: signup + description: > + Answers the same way whether or not the address is already + registered, because a different reply would tell an anonymous caller + who has an account. The link lands on the classic completion page, + which is where the account is actually created. Unauthenticated and + limited to 5 requests per 15 minutes per IP. + parameters: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [email] + properties: + email: + type: string + format: email + responses: + "202": + description: Accepted; an email has been sent if appropriate + content: + application/json: + schema: + type: object + properties: + sent: + type: boolean + "400": + $ref: "#/components/responses/BadRequest" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/password-reset: + post: + tags: [Auth] + summary: Send a password reset link + operationId: requestPasswordReset + description: > + Silent about whether the address is registered, for the same reason + signup is. Unauthenticated and limited per IP. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [email] + properties: + email: + type: string + format: email + responses: + "202": + description: Accepted; an email has been sent if appropriate + content: + application/json: + schema: + type: object + properties: + sent: + type: boolean + "400": + $ref: "#/components/responses/BadRequest" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/password-reset/complete: + post: + tags: [Auth] + summary: Set a new password using a link from the reset email + operationId: completePasswordReset + description: > + The signature covers the current password hash, so a link stops + working the moment it is used or the password changes by any other + route. Expired, unknown and forged links share one reply. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [user_id, expires, mac, password] + properties: + user_id: + type: integer + minimum: 1 + expires: + type: integer + mac: + type: string + maxLength: 128 + password: + type: string + responses: + "200": + description: Password changed + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + password_changed: + type: boolean + "400": + $ref: "#/components/responses/BadRequest" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + /auth/me: get: tags: [Auth] @@ -326,6 +448,55 @@ paths: $ref: "#/components/responses/Error" # RUNS + patch: + tags: [Auth] + summary: Change your own name, email or password + operationId: updateAccount + description: > + Touching the email or the password needs the current one as well: + the bearer token proves the request came from a signed-in session, + not that the sender knows the account's own credentials. + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + maxLength: 50 + email: + type: string + format: email + new_password: + type: string + current_password: + type: string + responses: + "200": + description: Account updated + content: + application/json: + schema: + $ref: "#/components/schemas/PlatformUser" + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + /runs: get: @@ -345,6 +516,13 @@ paths: - $ref: "#/components/parameters/RunStatus" - $ref: "#/components/parameters/Branch" - $ref: "#/components/parameters/CommitSha" + - name: ccx_version + in: query + description: > + Filter to the commit a CCExtractor release was cut from. + schema: + type: string + maxLength: 10 - $ref: "#/components/parameters/Repository" - $ref: "#/components/parameters/Platform" - $ref: "#/components/parameters/CreatedAfter" @@ -540,6 +718,40 @@ paths: default: $ref: "#/components/responses/Error" + /runs/{run_id}/restart: + post: + tags: [Runs] + summary: Queue a finished or stuck run to be executed again + operationId: restartRun + description: > + Clears the run's results and progress trail, which is what makes CI + pick it up again. The run keeps its id, so existing links stay valid, + and the old results are replaced rather than kept alongside the new + ones. Requires runs:write scope. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, tester, contributor] + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "202": + description: Restart accepted + content: + application/json: + schema: + $ref: "#/components/schemas/RunActionResult" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + /runs/{run_id}/cancel: post: tags: [Runs] @@ -818,6 +1030,113 @@ paths: default: $ref: "#/components/responses/Error" + patch: + tags: [Samples] + summary: Edit a sample's tags and upload metadata + operationId: updateSample + description: > + Notes, parameters, platform and version live on the upload row + rather than the sample, so a sample that arrived without one (over + FTP, or seeded directly) can only have its tags changed and asking + for the rest returns 409. Tags are given by name. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/SampleId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + tags: + type: array + maxItems: 50 + items: + type: string + maxLength: 64 + notes: + type: string + maxLength: 1024 + parameters: + type: string + maxLength: 1024 + platform: + type: string + enum: [linux, windows, mac, bsd] + version: + type: string + maxLength: 10 + responses: + "200": + description: Sample updated + content: + application/json: + schema: + type: object + properties: + sample_id: + type: integer + tags: + type: array + items: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + delete: + tags: [Samples] + summary: Delete a sample, its extra files and its media info + operationId: deleteSample + description: > + Refused while any regression test still points at the sample, + because removing the media would leave those tests unable to run and + their history describing a sample nobody can fetch. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Sample deleted + content: + application/json: + schema: + type: object + properties: + sample_id: + type: integer + deleted: + type: boolean + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + /samples/{sample_id}/details: get: tags: [Samples] @@ -853,6 +1172,39 @@ paths: default: $ref: "#/components/responses/Error" + /samples/{sample_id}/download: + get: + tags: [Samples] + summary: Locate a sample's media file in storage + operationId: downloadSample + description: > + Returns a signed URL rather than the bytes. Samples run to several + gigabytes, so streaming one through the API would hold a worker for + the length of the transfer. Where only the local copy exists there is + no URL to hand out and storage_status reports that instead. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Where the sample file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + /samples/{sample_id}/history: get: tags: [Samples] @@ -991,14 +1343,883 @@ paths: schema: $ref: "#/components/schemas/RegressionTestCreateRequest" responses: - "201": - description: The created regression test + "201": + description: The created regression test + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTest" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /categories: + get: + tags: [Samples] + summary: List regression test categories + operationId: listCategories + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated categories, ordered by name + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Category" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Samples] + summary: Create a category + operationId: createCategory + description: Admin or contributor only. Names are unique. + security: + - bearerAuth: [] + x-required-scope: runs:write + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CategoryCreateRequest" + responses: + "201": + description: The created category + content: + application/json: + schema: + $ref: "#/components/schemas/Category" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /categories/{category_id}: + patch: + tags: [Samples] + summary: Rename or re-describe a category + operationId: updateCategory + description: Admin or contributor only. An empty body is rejected. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/CategoryId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CategoryUpdateRequest" + responses: + "200": + description: The updated category + content: + application/json: + schema: + $ref: "#/components/schemas/Category" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + delete: + tags: [Samples] + summary: Delete a category + operationId: deleteCategory + description: > + Admin or contributor only. A category still attached to regression + tests is refused with 409 and a count, because dropping it would + change which tests a suite selection picks up. Detach the tests + first by PATCHing their categories. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/CategoryId" + responses: + "200": + description: The category was deleted + content: + application/json: + schema: + $ref: "#/components/schemas/DeletedResource" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}: + get: + tags: [Samples] + summary: Get one regression test with its baselines + operationId: getRegressionTest + description: > + Adds the expected outputs and the alternative hashes accepted as + variants, which the list endpoint omits because they multiply the + payload for every row. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RegressionTestId" + responses: + "200": + description: Regression test detail + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTestDetail" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + delete: + tags: [Samples] + summary: Delete a regression test + operationId: deleteRegressionTest + description: > + Admin or contributor only. A test that has already run is refused + with 409 and the number of results referencing it, because deleting + it would erase evidence of past regressions. Retire such a test with + PATCH active=false instead. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/RegressionTestId" + responses: + "200": + description: The regression test was deleted + content: + application/json: + schema: + $ref: "#/components/schemas/DeletedResource" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + patch: + tags: [Samples] + summary: Update part of a regression test definition + operationId: updateRegressionTest + description: > + Admin or contributor only. Only the fields present in the body are + written, so two clients editing different fields cannot clobber each + other. An empty body is rejected. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/RegressionTestId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTestUpdateRequest" + responses: + "200": + description: The updated regression test + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTest" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/download: + get: + tags: [Regression Tests] + summary: Locate the baseline a regression test is expected to reproduce + operationId: downloadRegressionTestOutput + description: > + Matches the output against its parent test, so an id belonging to a + different test reads as absent rather than resolving. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + responses: + "200": + description: Where the baseline file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/variants/{variant_id}/download: + get: + tags: [Regression Tests] + summary: Locate one of the alternative outputs accepted for a baseline + operationId: downloadRegressionTestVariant + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + - name: variant_id + in: path + required: true + description: Id of the variant, from the test detail payload + schema: + type: integer + minimum: 1 + responses: + "200": + description: Where the variant file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/variants: + post: + tags: [Regression Tests] + summary: Accept another output hash as correct for a baseline + operationId: createRegressionTestVariant + description: > + A test can legitimately produce different bytes on different + platforms or CCExtractor builds. Recording the hash here makes those + runs pass without overwriting the baseline everyone else is compared + against, which is what promoting to baseline would do. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, contributor] + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [hash] + properties: + hash: + type: string + maxLength: 128 + pattern: '^[A-Za-z0-9]+$' + responses: + "201": + description: Variant accepted + content: + application/json: + schema: + type: object + required: [id, hash] + properties: + id: + type: integer + minimum: 1 + hash: + type: string + maxLength: 128 + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/variants/{variant_id}: + delete: + tags: [Regression Tests] + summary: Stop accepting one alternative output for a baseline + operationId: deleteRegressionTestVariant + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, contributor] + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + - name: variant_id + in: path + required: true + description: Id of the variant, from the test detail payload + schema: + type: integer + minimum: 1 + responses: + "200": + description: Variant removed + content: + application/json: + schema: + type: object + required: [id, deleted] + properties: + id: + type: integer + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/media-info/download: + get: + tags: [Samples] + summary: Locate the MediaInfo XML written alongside a sample + operationId: downloadSampleMediaInfo + description: > + The parsed tree is already on /samples/{sample_id}/details; this is + for anyone who wants the original file MediaInfo produced. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Where the media info file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/extra-files/{extra_id}/download: + get: + tags: [Samples] + summary: Locate one of the files uploaded alongside a sample + operationId: downloadSampleExtraFile + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + - name: extra_id + in: path + required: true + description: Id of the accompanying file, from the sample details payload + schema: + type: integer + minimum: 1 + responses: + "200": + description: Where the accompanying file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/extra-files/{extra_id}: + delete: + tags: [Samples] + summary: Delete one of the files uploaded alongside a sample + operationId: deleteSampleExtraFile + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/SampleId" + - name: extra_id + in: path + required: true + description: Id of the accompanying file, from the sample details payload + schema: + type: integer + minimum: 1 + responses: + "200": + description: Accompanying file deleted + content: + application/json: + schema: + type: object + properties: + id: + type: integer + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /tags: + get: + tags: [Samples] + summary: List the tags a sample can be labelled with + operationId: listTags + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated tags + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + type: object + properties: + id: + type: integer + name: + type: string + description: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + post: + tags: [Samples] + summary: Create a tag + operationId: createTag + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name] + properties: + name: + type: string + maxLength: 64 + description: + type: string + maxLength: 1024 + responses: + "201": + description: Tag created + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + description: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/about: + get: + tags: [System] + summary: Report the versions this platform runs against + operationId: systemAbout + description: > + The classic about page is static prose. The parts worth reading from + a client are the CCExtractor release under test and the commit the + platform itself was deployed from. + security: + - bearerAuth: [] + x-required-scope: system:read + responses: + "200": + description: Platform and CCExtractor versions + content: + application/json: + schema: + type: object + properties: + platform_commit: + type: string + nullable: true + ccextractor_version: + type: string + nullable: true + ccextractor_released: + type: string + nullable: true + last_tested_commit: + type: string + nullable: true + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /users/{user_id}/deactivate: + post: + tags: [Auth] + summary: Anonymise an account and lock it out + operationId: deactivateUser + description: > + The row stays so the uploads and runs it owns keep an author, which + is why this scrubs the identity instead of deleting. Open to admins + and to the account's owner. Scope-free for the same reason revoking + your own token is: closing your own account cannot depend on + tokens:manage, which no role below admin may hold. + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/UserId" + responses: + "200": + description: Account deactivated and its own tokens revoked + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + deactivated: + type: boolean + tokens_revoked: + type: integer + description: > + How many of the account's tokens were still live and + have now been revoked. Scrambling the password only + stops new ones being minted, so the ones already + issued are ended here as well. + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/upload: + post: + tags: [Uploads] + summary: Upload a media file into the queue + operationId: uploadSample + description: > + Uploading and describing are two steps, as on the classic pages: the + transfer is slow and the metadata needs a CCExtractor version the + uploader may have to look up, so the bytes are banked first. The file + is hashed as it is written, and content already in the library or the + queue is refused with 409. Extension and mime type are checked + against the forbidden lists. + security: + - bearerAuth: [] + x-required-scope: runs:write + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: + type: string + format: binary + responses: + "201": + description: Upload queued + content: + application/json: + schema: + type: object + properties: + id: + type: integer + sha: + type: string + extension: + type: string + original_name: + type: string + user_id: + type: integer + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "413": + description: File larger than the configured upload limit + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /queued-samples: + get: + tags: [Uploads] + summary: List uploads waiting to be described + operationId: listQueuedSamples + description: > + Admins see the whole queue; everyone else sees only their own + uploads. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated queued uploads + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + type: object + properties: + id: + type: integer + sha: + type: string + extension: + type: string + original_name: + type: string + user_id: + type: integer + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /queued-samples/{queued_id}: + get: + tags: [Uploads] + summary: Get one queued upload + operationId: getQueuedSample + description: > + Someone else's upload reads as absent rather than forbidden, so an + id cannot be used to find out what others have queued. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 + responses: + "200": + description: The queued upload content: application/json: schema: - $ref: "#/components/schemas/RegressionTest" - "400": - $ref: "#/components/responses/BadRequest" + type: object + properties: + id: + type: integer + sha: + type: string + extension: + type: string + original_name: + type: string + user_id: + type: integer "401": $ref: "#/components/responses/Unauthorized" "403": @@ -1009,244 +2230,307 @@ paths: $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" - - /categories: - get: - tags: [Samples] - summary: List regression test categories - operationId: listCategories + delete: + tags: [Uploads] + summary: Discard a queued upload + operationId: deleteQueuedSample security: - bearerAuth: [] - x-required-scope: runs:read + x-required-scope: runs:write parameters: - - $ref: "#/components/parameters/Limit" - - $ref: "#/components/parameters/Offset" + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 responses: "200": - description: Paginated categories, ordered by name + description: Queued upload discarded content: application/json: schema: - allOf: - - $ref: "#/components/schemas/Page" - - type: object - properties: - data: - type: array - items: - $ref: "#/components/schemas/Category" - "400": - $ref: "#/components/responses/BadRequest" + type: object + properties: + id: + type: integer + deleted: + type: boolean "401": $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" "429": $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" + /queued-samples/{queued_id}/finalize: post: - tags: [Samples] - summary: Create a category - operationId: createCategory - description: Admin or contributor only. Names are unique. + tags: [Uploads] + summary: Turn a queued upload into a sample + operationId: finalizeQueuedSample + description: > + The file only moves into TestFiles once the rows are committed, so a + failure part way through leaves the upload in the queue to retry + rather than a sample row pointing at a file that was never placed. + Reporting the sample as a GitHub issue stays on the classic page. security: - bearerAuth: [] x-required-scope: runs:write + parameters: + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/CategoryCreateRequest" + type: object + required: [version, platform] + properties: + version: + type: string + maxLength: 10 + description: CCExtractor release the sample was produced with + platform: + type: string + enum: [linux, windows, mac, bsd] + parameters: + type: string + maxLength: 1024 + notes: + type: string + maxLength: 1024 responses: "201": - description: The created category + description: Sample created from the upload content: application/json: schema: - $ref: "#/components/schemas/Category" + type: object + properties: + sample_id: + type: integer + sha: + type: string + original_name: + type: string "400": $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "409": - $ref: "#/components/responses/Conflict" + "404": + $ref: "#/components/responses/NotFound" "429": $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" - /categories/{category_id}: - patch: - tags: [Samples] - summary: Rename or re-describe a category - operationId: updateCategory - description: Admin or contributor only. An empty body is rejected. + /users/{user_id}/password-reset: + post: + tags: [Auth] + summary: Send a reset link to another account, or to your own + operationId: sendUserPasswordReset + description: > + Unlike the public /auth/password-reset this names an account rather + than an address, so it says plainly when the id is unknown: the + caller is already signed in and can list users anyway. Scope-free + for the same reason deactivation is. security: - bearerAuth: [] - x-required-scope: runs:write parameters: - - $ref: "#/components/parameters/CategoryId" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/CategoryUpdateRequest" + - $ref: "#/components/parameters/UserId" responses: - "200": - description: The updated category + "202": + description: Reset link sent content: application/json: schema: - $ref: "#/components/schemas/Category" - "400": - $ref: "#/components/responses/BadRequest" + type: object + properties: + user_id: + type: integer + sent: + type: boolean "401": $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" "404": $ref: "#/components/responses/NotFound" - "409": - $ref: "#/components/responses/Conflict" "429": $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" - delete: - tags: [Samples] - summary: Delete a category - operationId: deleteCategory + /auth/me/github: + get: + tags: [Auth] + summary: Whether your account is connected to GitHub + operationId: getGithubLink description: > - Admin or contributor only. A category still attached to regression - tests is refused with 409 and a count, because dropping it would - change which tests a suite selection picks up. Detach the tests - first by PATCHing their categories. + Connecting is a browser redirect, so this hands back the URL to send + somebody to rather than performing it. Trading the code GitHub + returns for a token stays on the classic callback, which already + holds the client secret and the registered redirect. The stored + token is never part of this response; the URL carries only the + client id and the scope asked for, both of which are public. security: - bearerAuth: [] - x-required-scope: runs:write - parameters: - - $ref: "#/components/parameters/CategoryId" responses: "200": - description: The category was deleted + description: Connection status and where to start one content: application/json: schema: - $ref: "#/components/schemas/DeletedResource" + type: object + properties: + linked: + type: boolean + github_login: + type: string + nullable: true + authorize_url: + type: string "401": $ref: "#/components/responses/Unauthorized" - "403": - $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" - "409": - $ref: "#/components/responses/Conflict" "429": $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" - - /regression-tests/{regression_test_id}: - get: - tags: [Samples] - summary: Get one regression test with its baselines - operationId: getRegressionTest + delete: + tags: [Auth] + summary: Forget this platform's copy of your GitHub connection + operationId: unlinkGithub description: > - Adds the expected outputs and the alternative hashes accepted as - variants, which the list endpoint omits because they multiply the - payload for every row. + Only ever the caller's own. This drops the token the platform + holds; the authorisation itself is withdrawn from GitHub's own + applications page, which is the only place that can really end it. security: - bearerAuth: [] - x-required-scope: runs:read - parameters: - - $ref: "#/components/parameters/RegressionTestId" responses: "200": - description: Regression test detail + description: The connection has been forgotten content: application/json: schema: - $ref: "#/components/schemas/RegressionTestDetail" + type: object + properties: + linked: + type: boolean + github_login: + type: string + nullable: true "401": $ref: "#/components/responses/Unauthorized" - "403": - $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "429": $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" - - delete: - tags: [Samples] - summary: Delete a regression test - operationId: deleteRegressionTest + /auth/me/ftp-credentials: + get: + tags: [Uploads] + summary: Get your own FTP details for the ingest server + operationId: getFtpCredentials description: > - Admin or contributor only. A test that has already run is refused - with 409 and the number of results referencing it, because deleting - it would erase evidence of past regressions. Retire such a test with - PATCH active=false instead. + Created on first ask and stable afterwards. Only ever the caller's + own: these are working credentials, so there is no version of this + that reads someone else's. The password is stored in the clear by + design, being random and unchangeable, so this is the only place it + can come from. security: - bearerAuth: [] - x-required-scope: runs:write - parameters: - - $ref: "#/components/parameters/RegressionTestId" responses: "200": - description: The regression test was deleted + description: FTP connection details content: application/json: schema: - $ref: "#/components/schemas/DeletedResource" + type: object + properties: + host: + type: string + port: + type: string + username: + type: string + password: + type: string "401": $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" "404": $ref: "#/components/responses/NotFound" - "409": - $ref: "#/components/responses/Conflict" "429": $ref: "#/components/responses/RateLimited" default: $ref: "#/components/responses/Error" - patch: - tags: [Samples] - summary: Update part of a regression test definition - operationId: updateRegressionTest + /queued-samples/{queued_id}/link: + post: + tags: [Uploads] + summary: Attach a queued upload to an existing sample + operationId: linkQueuedSample description: > - Admin or contributor only. Only the fields present in the body are - written, so two clients editing different fields cannot clobber each - other. An empty body is rejected. + For material that belongs with a sample without being one, such as a + subtitle track to compare against. The classic page offered this but + never carried it out: its confirm step checks permissions and then + redirects without touching anything, so this is the behaviour it + described rather than the behaviour it had. security: - bearerAuth: [] x-required-scope: runs:write parameters: - - $ref: "#/components/parameters/RegressionTestId" + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/RegressionTestUpdateRequest" + type: object + required: [sample_id] + properties: + sample_id: + type: integer + minimum: 1 responses: - "200": - description: The updated regression test + "201": + description: Upload attached as an extra file content: application/json: schema: - $ref: "#/components/schemas/RegressionTest" + type: object + properties: + id: + type: integer + sample_id: + type: integer + filename: + type: string "400": $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" "401": $ref: "#/components/responses/Unauthorized" "403": @@ -1599,6 +2883,33 @@ paths: $ref: "#/components/responses/Error" /users/{user_id}: + get: + tags: [Auth] + summary: Get one platform user + operationId: getUser + security: + - bearerAuth: [] + x-required-scope: tokens:manage + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/UserId" + responses: + "200": + description: The user + content: + application/json: + schema: + $ref: "#/components/schemas/PlatformUser" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" patch: tags: [Users] summary: Change a user's role @@ -3198,8 +4509,36 @@ components: type: array maxItems: 100 items: - type: string - maxLength: 128 + type: object + required: [id, hash] + properties: + id: + type: integer + minimum: 1 + description: Addresses the variant in its download route. + hash: + type: string + maxLength: 128 + + StoredFile: + type: object + required: [filename, download_url, storage_status] + description: > + Where a file lives rather than the file itself. Samples and baselines + are handed out as signed URLs so that large transfers do not run + through the API. A null download_url with storage_status=degraded + means the only copy is on the platform's own disk. + properties: + filename: + type: string + maxLength: 255 + download_url: + type: string + format: uri + nullable: true + storage_status: + type: string + enum: [ok, degraded] MaintenanceState: type: object diff --git a/tests/api/test_routes_auth.py b/tests/api/test_routes_auth.py index 33357a876..ce8806c4f 100644 --- a/tests/api/test_routes_auth.py +++ b/tests/api/test_routes_auth.py @@ -1,4 +1,5 @@ import json +import time from unittest.mock import patch from flask import g @@ -452,3 +453,316 @@ def test_update_user_role_unknown_user(self): res = self._patch_user(self._admin_token('usr_404'), 999999, {'role': 'user'}) self.assertEqual(res.status_code, 404) + + # ---- account: signup, reset, self-service, deactivate -------------- + + def _json(self, method, path, body, headers=None): + return getattr(self.client, method)( + f'/api/v1{path}', data=json.dumps(body), + content_type='application/json', headers=headers or {}) + + @patch('requests.post') + def test_signup_sends_a_link(self, post): + res = self._json('post', '/auth/signup', + {'email': 'brand_new@local.com'}) + + self.assertEqual(res.status_code, 202) + self.assertTrue(post.called) + + @patch('requests.post') + def test_signup_is_silent_about_existing_accounts(self, post): + known = self._json('post', '/auth/signup', + {'email': 'auth_user@local.com'}) + _rate_limit_store.clear() + unknown = self._json('post', '/auth/signup', + {'email': 'nobody@local.com'}) + + # Same status and body either way, or the reply becomes a way to + # ask whether an address has an account. + self.assertEqual(known.status_code, unknown.status_code) + self.assertEqual(known.json, unknown.json) + + @patch('requests.post') + def test_password_reset_request_is_silent_about_unknown_email(self, post): + res = self._json('post', '/auth/password-reset', + {'email': 'nobody@local.com'}) + + self.assertEqual(res.status_code, 202) + self.assertFalse(post.called) + + @patch('requests.post') + def test_password_reset_request_sends_to_a_known_address(self, post): + # Exercises the path that actually builds and sends the link, which + # an unknown-address test never reaches. + res = self._json('post', '/auth/password-reset', + {'email': 'auth_user@local.com'}) + + self.assertEqual(res.status_code, 202) + self.assertTrue(post.called) + + @patch('requests.post') + def test_reset_link_points_at_these_pages_without_a_console(self, post): + from run import app + + with patch.dict(app.config, {'CONSOLE_URL': ''}): + self._json('post', '/auth/password-reset', + {'email': 'auth_user@local.com'}) + + body = str(post.call_args) + self.assertIn('/account/reset/', body) + + @patch('requests.post') + def test_reset_link_points_at_the_console_when_one_is_configured( + self, post): + from run import app + + with patch.dict(app.config, + {'CONSOLE_URL': 'https://console.example.org/'}): + self._json('post', '/auth/password-reset', + {'email': 'auth_user@local.com'}) + + body = str(post.call_args) + # One slash, and the three values the console posts back. + self.assertIn('https://console.example.org/reset?uid=', body) + self.assertIn('expires=', body) + self.assertIn('mac=', body) + + @patch('requests.post') + def test_password_reset_completes_with_a_valid_link(self, post): + from mod_auth.controllers import generate_hmac_hash + from run import app + + expires = int(time.time()) + 600 + content = f'{self.user_id}|{expires}|{self.user.password}' + mac = generate_hmac_hash(app.config.get('HMAC_KEY', ''), content) + + res = self._json('post', '/auth/password-reset/complete', { + 'user_id': self.user_id, + 'expires': expires, + 'mac': mac, + 'password': 'a-brand-new-password', + }) + + self.assertEqual(res.status_code, 200) + changed = User.query.filter_by(id=self.user_id).first() + self.assertTrue(changed.is_password_valid('a-brand-new-password')) + + def test_password_reset_rejects_a_forged_mac(self): + res = self._json('post', '/auth/password-reset/complete', { + 'user_id': self.user_id, + 'expires': int(time.time()) + 600, + 'mac': 'not-the-real-signature', + 'password': 'a-brand-new-password', + }) + + self.assertEqual(res.status_code, 400) + unchanged = User.query.filter_by(id=self.user_id).first() + self.assertTrue(unchanged.is_password_valid('userpass123')) + + def test_password_reset_rejects_an_expired_link(self): + from mod_auth.controllers import generate_hmac_hash + from run import app + + expires = int(time.time()) - 1 + content = f'{self.user_id}|{expires}|{self.user.password}' + mac = generate_hmac_hash(app.config.get('HMAC_KEY', ''), content) + + res = self._json('post', '/auth/password-reset/complete', { + 'user_id': self.user_id, + 'expires': expires, + 'mac': mac, + 'password': 'a-brand-new-password', + }) + + self.assertEqual(res.status_code, 400) + + def _auth(self, email='auth_user@local.com', pwd='userpass123', + name='acct', scopes=None): + # get_token returns the response in this class, not the token. + token = self.get_token(email, pwd, name, scopes=scopes).json['token'] + return {'Authorization': f'Bearer {token}'} + + def test_update_own_name(self): + res = self._json('patch', '/auth/me', {'name': 'Renamed'}, + self._auth(name='acct1')) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['name'], 'Renamed') + + def test_changing_password_requires_the_current_one(self): + res = self._json('patch', '/auth/me', + {'new_password': 'something-else-entirely'}, + self._auth(name='acct2')) + + self.assertEqual(res.status_code, 403) + + def test_change_password_with_the_current_one(self): + res = self._json('patch', '/auth/me', { + 'current_password': 'userpass123', + 'new_password': 'something-else-entirely', + }, self._auth(name='acct3')) + + self.assertEqual(res.status_code, 200) + changed = User.query.filter_by(id=self.user_id).first() + self.assertTrue(changed.is_password_valid('something-else-entirely')) + + def test_change_email_to_one_already_taken(self): + res = self._json('patch', '/auth/me', { + 'current_password': 'userpass123', + 'email': 'auth_admin@local.com', + }, self._auth(name='acct4')) + + self.assertEqual(res.status_code, 409) + + def test_deactivate_own_account(self): + # A plain contributor can never hold tokens:manage, so closing your + # own account has to work without it. + res = self.client.post( + f'/api/v1/users/{self.user_id}/deactivate', + headers=self._auth(name='acct5')) + + self.assertEqual(res.status_code, 200) + gone = User.query.filter_by(id=self.user_id).first() + self.assertEqual(gone.name, f'Anonymous {self.user_id}') + + def test_deactivate_someone_else_needs_admin(self): + res = self.client.post( + f'/api/v1/users/{self.user_id}/deactivate', + headers=self._auth('auth_admin@local.com', 'adminpass123', + 'acct6')) + + self.assertEqual(res.status_code, 200) + gone = User.query.filter_by(id=self.user_id).first() + self.assertEqual(gone.name, f'Anonymous {self.user_id}') + self.assertFalse(gone.is_password_valid('userpass123')) + + def test_deactivate_requires_admin_or_ownership(self): + res = self.client.post( + f'/api/v1/users/{self.admin.id}/deactivate', + headers=self._auth(name='acct7')) + + self.assertEqual(res.status_code, 403) + + def test_deactivation_revokes_the_accounts_tokens(self): + # A token the account already holds, minted before anyone reaches + # for deactivation. + victim = self._auth(name='doomed') + + res = self.client.post( + f'/api/v1/users/{self.user_id}/deactivate', + headers=self._auth('auth_admin@local.com', 'adminpass123', + 'deact_admin')) + + self.assertEqual(res.status_code, 200) + self.assertGreaterEqual(res.json['tokens_revoked'], 1) + self.assertTrue(all( + t.is_revoked for t in + ApiToken.query.filter(ApiToken.user_id == self.user_id).all())) + # And the token stops working straight away, not at its expiry. + after = self.client.get('/api/v1/auth/me', headers=victim) + self.assertEqual(after.status_code, 401) + + def test_deactivating_someone_else_leaves_the_admin_working(self): + admin_headers = self._auth('auth_admin@local.com', 'adminpass123', + 'deact_admin2') + + self.client.post(f'/api/v1/users/{self.user_id}/deactivate', + headers=admin_headers) + + still_fine = self.client.get('/api/v1/auth/me', headers=admin_headers) + self.assertEqual(still_fine.status_code, 200) + + def test_ftp_credentials_need_a_write_scope(self): + # A working credential for the ingest server is not something a + # read-only token should be able to fetch. + res = self.client.get( + '/api/v1/auth/me/ftp-credentials', + headers=self._auth(name='ftp_ro', scopes=['results:read'])) + + self.assertEqual(res.status_code, 403) + + def test_get_single_user(self): + res = self.client.get( + f'/api/v1/users/{self.user_id}', + headers=self._auth('auth_admin@local.com', 'adminpass123', + 'one_user', scopes=['tokens:manage'])) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['user_id'], self.user_id) + + @patch('requests.post') + def test_admin_can_send_someone_a_reset_link(self, post): + res = self.client.post( + f'/api/v1/users/{self.user_id}/password-reset', + headers=self._auth('auth_admin@local.com', 'adminpass123', + 'reset_other')) + + self.assertEqual(res.status_code, 202) + self.assertTrue(post.called) + + @patch('requests.post') + def test_reset_link_for_someone_else_needs_admin(self, post): + res = self.client.post( + f'/api/v1/users/{self.admin.id}/password-reset', + headers=self._auth(name='reset_forbidden')) + + self.assertEqual(res.status_code, 403) + self.assertFalse(post.called) + + def test_ftp_credentials_are_created_on_first_ask(self): + # Asked for with the scope an uploader would hold; the default set + # is read-only and is refused, which the test below covers. + res = self.client.get( + '/api/v1/auth/me/ftp-credentials', + headers=self._auth(name='ftp1', scopes=['runs:write'])) + + self.assertEqual(res.status_code, 200) + self.assertTrue(res.json['username']) + self.assertTrue(res.json['password']) + + # Asking twice returns the same pair rather than rotating them. + again = self.client.get( + '/api/v1/auth/me/ftp-credentials', + headers=self._auth(name='ftp2', scopes=['runs:write'])) + self.assertEqual(again.json['username'], res.json['username']) + + def test_github_link_reports_status_without_the_token(self): + user = User.query.filter(User.id == self.user_id).first() + user.github_token = 'gho_secret_value' + user.github_login = 'someone' + g.db.commit() + + res = self.client.get( + '/api/v1/auth/me/github', headers=self._auth(name='gh1')) + + self.assertEqual(res.status_code, 200) + self.assertTrue(res.json['linked']) + self.assertEqual(res.json['github_login'], 'someone') + # The stored token must never travel back out, under any key. + self.assertNotIn('gho_secret_value', res.get_data(as_text=True)) + # The URL to start a connection carries nothing secret either. + self.assertIn('github.com/login/oauth/authorize', + res.json['authorize_url']) + self.assertNotIn('client_secret', res.json['authorize_url']) + + def test_github_unlink_forgets_the_connection(self): + user = User.query.filter(User.id == self.user_id).first() + user.github_token = 'gho_secret_value' + user.github_login = 'someone' + g.db.commit() + + res = self.client.delete( + '/api/v1/auth/me/github', headers=self._auth(name='gh2')) + + self.assertEqual(res.status_code, 200) + self.assertFalse(res.json['linked']) + + user = User.query.filter(User.id == self.user_id).first() + self.assertIsNone(user.github_token) + self.assertIsNone(user.github_login) + + def test_github_endpoints_need_a_token(self): + self.assertEqual(self.client.get('/api/v1/auth/me/github').status_code, + 401) + self.assertEqual( + self.client.delete('/api/v1/auth/me/github').status_code, 401) diff --git a/tests/api/test_routes_regression_tests.py b/tests/api/test_routes_regression_tests.py index 3e70cc4b3..d08a169dd 100644 --- a/tests/api/test_routes_regression_tests.py +++ b/tests/api/test_routes_regression_tests.py @@ -1,4 +1,5 @@ import json +from unittest import mock from flask import g @@ -210,8 +211,189 @@ def test_get_regression_test_detail(self): self.assertEqual(res.status_code, 200) self.assertEqual(res.json['regression_test_id'], self.existing_id) self.assertEqual(res.json['outputs'][0]['correct'], 'expected_hash') - # Alternative accepted hashes travel with the baseline. - self.assertEqual(res.json['outputs'][0]['variants'], ['variant_hash']) + # Alternative accepted hashes travel with the baseline, each carrying + # the id its download route is addressed by. + variants = res.json['outputs'][0]['variants'] + self.assertEqual(len(variants), 1) + self.assertEqual(variants[0]['hash'], 'variant_hash') + self.assertIsInstance(variants[0]['id'], int) + + # ---- regression tests: baseline downloads -------------------------- + + def _baseline(self): + """Give the existing test one baseline with one accepted variant.""" + output = RegressionTestOutput( + self.existing_id, 'basehash', '.srt', 'expected_name') + g.db.add(output) + g.db.commit() + variant = RegressionTestOutputFiles('varianthash', output.id) + g.db.add(variant) + g.db.commit() + return output.id, variant.id + + @mock.patch('mod_api.routes.regression_tests.resolve_artifact') + def test_download_output(self, resolve): + resolve.return_value = ('https://signed.url', 'ok') + output_id, _ = self._baseline() + + res = self.client.get( + f'/api/v1/regression-tests/{self.existing_id}' + f'/outputs/{output_id}/download', + headers=self._as('rtw_user@local.com', 'userpass123', 'rt_dl', + ['runs:read'])) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['download_url'], 'https://signed.url') + self.assertEqual(res.json['filename'], 'basehash.srt') + resolve.assert_called_once_with('TestResults/basehash.srt') + + @mock.patch('mod_api.routes.regression_tests.resolve_artifact') + def test_download_output_local_only_has_no_url(self, resolve): + resolve.return_value = (None, 'degraded') + output_id, _ = self._baseline() + + res = self.client.get( + f'/api/v1/regression-tests/{self.existing_id}' + f'/outputs/{output_id}/download', + headers=self._as('rtw_user@local.com', 'userpass123', 'rt_dl_deg', + ['runs:read'])) + + self.assertEqual(res.status_code, 200) + self.assertIsNone(res.json['download_url']) + self.assertEqual(res.json['storage_status'], 'degraded') + + @mock.patch('mod_api.routes.regression_tests.resolve_artifact') + def test_download_output_missing_from_storage(self, resolve): + resolve.return_value = (None, 'missing') + output_id, _ = self._baseline() + + res = self.client.get( + f'/api/v1/regression-tests/{self.existing_id}' + f'/outputs/{output_id}/download', + headers=self._as('rtw_user@local.com', 'userpass123', 'rt_dl_gone', + ['runs:read'])) + + self.assertEqual(res.status_code, 404) + + def test_download_output_of_another_test_is_not_found(self): + output_id, _ = self._baseline() + other = RegressionTest( + self.sample_id, 'other command', InputType.file, OutputType.file, + None, 0) + g.db.add(other) + g.db.commit() + + # The output id is real, just not this test's, so it must not resolve. + res = self.client.get( + f'/api/v1/regression-tests/{other.id}' + f'/outputs/{output_id}/download', + headers=self._as('rtw_user@local.com', 'userpass123', 'rt_dl_x', + ['runs:read'])) + + self.assertEqual(res.status_code, 404) + + @mock.patch('mod_api.routes.regression_tests.resolve_artifact') + def test_download_variant(self, resolve): + resolve.return_value = ('https://signed.url', 'ok') + output_id, variant_id = self._baseline() + + res = self.client.get( + f'/api/v1/regression-tests/{self.existing_id}/outputs/{output_id}' + f'/variants/{variant_id}/download', + headers=self._as('rtw_user@local.com', 'userpass123', 'rt_var', + ['runs:read'])) + + self.assertEqual(res.status_code, 200) + # The variant borrows its parent baseline's extension. + resolve.assert_called_once_with('TestResults/varianthash.srt') + + def test_download_variant_not_found(self): + output_id, _ = self._baseline() + + res = self.client.get( + f'/api/v1/regression-tests/{self.existing_id}/outputs/{output_id}' + f'/variants/999999/download', + headers=self._as('rtw_user@local.com', 'userpass123', 'rt_var_404', + ['runs:read'])) + + self.assertEqual(res.status_code, 404) + + # ---- regression tests: variant management -------------------------- + + def test_create_variant(self): + output_id, _ = self._baseline() + + res = self._write( + 'post', + f'/regression-tests/{self.existing_id}/outputs/{output_id}' + f'/variants', + self._admin('rt_var_add'), {'hash': 'abc123'}) + + self.assertEqual(res.status_code, 201) + self.assertEqual(res.json['hash'], 'abc123') + self.assertEqual( + RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=output_id, + file_hashes='abc123').count(), 1) + + def test_create_variant_rejects_duplicate(self): + output_id, _ = self._baseline() + + res = self._write( + 'post', + f'/regression-tests/{self.existing_id}/outputs/{output_id}' + f'/variants', + self._admin('rt_var_dup'), {'hash': 'varianthash'}) + + self.assertEqual(res.status_code, 409) + + def test_create_variant_rejects_path_characters(self): + output_id, _ = self._baseline() + + # The hash is joined to an extension to build a path under + # TestResults, so separators must never reach the filesystem. + res = self._write( + 'post', + f'/regression-tests/{self.existing_id}/outputs/{output_id}' + f'/variants', + self._admin('rt_var_bad'), {'hash': '../../etc/passwd'}) + + self.assertEqual(res.status_code, 400) + + def test_create_variant_requires_write_role(self): + output_id, _ = self._baseline() + + res = self._write( + 'post', + f'/regression-tests/{self.existing_id}/outputs/{output_id}' + f'/variants', + self._as('rtw_user@local.com', 'userpass123', 'rt_var_role', + ['runs:write']), + {'hash': 'abc123'}) + + self.assertEqual(res.status_code, 403) + + def test_delete_variant(self): + output_id, variant_id = self._baseline() + + res = self.client.delete( + f'/api/v1/regression-tests/{self.existing_id}' + f'/outputs/{output_id}/variants/{variant_id}', + headers=self._admin('rt_var_del')) + + self.assertEqual(res.status_code, 200) + self.assertIsNone(RegressionTestOutputFiles.query.filter_by( + id=variant_id).first()) + + def test_delete_variant_not_found(self): + output_id, _ = self._baseline() + + res = self.client.delete( + f'/api/v1/regression-tests/{self.existing_id}' + f'/outputs/{output_id}/variants/999999', + headers=self._admin('rt_var_del404')) + + self.assertEqual(res.status_code, 404) def test_get_regression_test_detail_not_found(self): res = self.client.get( diff --git a/tests/api/test_routes_runs.py b/tests/api/test_routes_runs.py index 78843c06a..117d9a405 100644 --- a/tests/api/test_routes_runs.py +++ b/tests/api/test_routes_runs.py @@ -199,6 +199,39 @@ def test_cancel_run(self): progs = TestProgress.query.filter_by(test_id=self.test_id).all() self.assertEqual(progs[-1].status, TestStatus.canceled) + def test_restart_run(self): + token = self.get_token('runs_admin@local.com', 'adminpass123', + 'restart1', scopes=['runs:write']) + res = self.client.post( + f'/api/v1/runs/{self.test_id}/restart', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 202) + self.assertEqual(res.json['status'], 'accepted') + # An empty progress trail is what makes CI pick the run up again. + self.assertEqual( + TestProgress.query.filter_by(test_id=self.test_id).count(), 0) + self.assertEqual( + TestResult.query.filter_by(test_id=self.test_id).count(), 0) + + def test_restart_run_not_found(self): + token = self.get_token('runs_admin@local.com', 'adminpass123', + 'restart2', scopes=['runs:write']) + res = self.client.post( + '/api/v1/runs/999999/restart', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 404) + + def test_restart_run_requires_write_role(self): + token = self.get_token('runs_user@local.com', 'userpass123', + 'restart3', scopes=['runs:write']) + res = self.client.post( + f'/api/v1/runs/{self.test_id}/restart', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 403) + def test_cancel_run_idempotency(self): token = self.get_token('runs_admin@local.com', 'adminpass123', 't10', scopes=['runs:write']) diff --git a/tests/api/test_routes_samples.py b/tests/api/test_routes_samples.py index 0458c6973..43e7f4886 100644 --- a/tests/api/test_routes_samples.py +++ b/tests/api/test_routes_samples.py @@ -1,4 +1,5 @@ -from unittest.mock import patch +import json +from unittest import mock from flask import g from sqlalchemy import event @@ -6,7 +7,7 @@ from mod_api.middleware.rate_limit import _rate_limit_store from mod_regression.models import (Category, InputType, OutputType, RegressionTest, RegressionTestOutput) -from mod_sample.models import Sample +from mod_sample.models import ExtraFile, Sample, Tag from mod_test.models import (Test, TestPlatform, TestResult, TestResultFile, TestType) from tests.api.base import ApiTestCase @@ -350,7 +351,7 @@ def test_get_sample_history_status_filter(self): # The whole history fit inside the scan, so the page is complete. self.assertNotIn('truncated', res.json['pagination']) - @patch('mod_api.routes.samples._HISTORY_STATUS_SCAN_LIMIT', 2) + @mock.patch('mod_api.routes.samples._HISTORY_STATUS_SCAN_LIMIT', 2) def test_get_sample_history_status_filter_scan_is_bounded(self): # status is derived in Python, so it can't be pushed into SQL. The # scan is capped instead, and a capped page says so rather than @@ -488,6 +489,217 @@ def test_get_sample_details(self): self.assertIsNone(res.json['upload']) self.assertIsNone(res.json['media_info']) + @mock.patch('mod_api.routes.samples.resolve_artifact') + def test_download_sample(self, resolve): + resolve.return_value = ('https://signed.url', 'ok') + token = self.get_token('samp_user@local.com', 'userpass123', + 'dl1', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/download', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['download_url'], 'https://signed.url') + # Spelled out rather than read back off self.sample, which the + # request teardown detaches from its session. + resolve.assert_called_once_with('TestFiles/test_sha.txt') + + @mock.patch('mod_api.routes.samples.resolve_artifact') + def test_download_sample_local_only_has_no_url(self, resolve): + resolve.return_value = (None, 'degraded') + token = self.get_token('samp_user@local.com', 'userpass123', + 'dl2', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/download', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 200) + self.assertIsNone(res.json['download_url']) + self.assertEqual(res.json['storage_status'], 'degraded') + + @mock.patch('mod_api.routes.samples.resolve_artifact') + def test_download_sample_missing_from_storage(self, resolve): + resolve.return_value = (None, 'missing') + token = self.get_token('samp_user@local.com', 'userpass123', + 'dl3', scopes=['runs:read']) + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/download', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 404) + + def test_download_sample_not_found(self): + token = self.get_token('samp_user@local.com', 'userpass123', + 'dl4', scopes=['runs:read']) + res = self.client.get( + '/api/v1/samples/999999/download', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 404) + + # ---- tags, sample edits and deletes -------------------------------- + + def _admin(self, name): + token = self.get_token('samp_admin@local.com', 'adminpass123', name, + scopes=['runs:read', 'runs:write']) + return {'Authorization': f'Bearer {token}'} + + def _post(self, path, headers, body): + return self.client.post( + f'/api/v1{path}', data=json.dumps(body), + content_type='application/json', headers=headers) + + def test_create_and_list_tags(self): + res = self._post('/tags', self._admin('tag1'), + {'name': 'dvb', 'description': 'DVB subtitles'}) + self.assertEqual(res.status_code, 201) + + listed = self.client.get('/api/v1/tags', headers=self._admin('tag2')) + self.assertEqual(listed.status_code, 200) + self.assertIn('dvb', [t['name'] for t in listed.json['data']]) + + def test_create_tag_rejects_duplicate(self): + self._post('/tags', self._admin('tag3'), {'name': 'dvb'}) + res = self._post('/tags', self._admin('tag4'), {'name': 'dvb'}) + self.assertEqual(res.status_code, 409) + + def test_create_tag_requires_admin(self): + token = self.get_token('samp_user@local.com', 'userpass123', 'tag5', + scopes=['runs:write']) + res = self._post('/tags', {'Authorization': f'Bearer {token}'}, + {'name': 'nope'}) + self.assertEqual(res.status_code, 403) + + def test_update_sample_tags(self): + g.db.add(Tag('dvb', 'DVB subtitles')) + g.db.commit() + + res = self.client.patch( + f'/api/v1/samples/{self.sample_id}', + data=json.dumps({'tags': ['dvb']}), + content_type='application/json', + headers=self._admin('edit1')) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['tags'], ['dvb']) + + def test_update_sample_rejects_unknown_tag(self): + res = self.client.patch( + f'/api/v1/samples/{self.sample_id}', + data=json.dumps({'tags': ['nosuchtag']}), + content_type='application/json', + headers=self._admin('edit2')) + + self.assertEqual(res.status_code, 400) + + def test_update_sample_without_upload_row_rejects_metadata(self): + # notes and friends live on the upload row, and this sample has none. + res = self.client.patch( + f'/api/v1/samples/{self.sample_id}', + data=json.dumps({'notes': 'a note'}), + content_type='application/json', + headers=self._admin('edit3')) + + self.assertEqual(res.status_code, 409) + + def test_delete_sample_in_use_is_refused(self): + res = self.client.delete( + f'/api/v1/samples/{self.sample_id}', + headers=self._admin('del1')) + + self.assertEqual(res.status_code, 409) + self.assertEqual(res.json['details']['regression_test_count'], 1) + + def test_delete_unused_sample(self): + spare = Sample('spare_sha', 'txt', 'spare') + g.db.add(spare) + g.db.commit() + spare_id = spare.id + + res = self.client.delete( + f'/api/v1/samples/{spare_id}', headers=self._admin('del2')) + + self.assertEqual(res.status_code, 200) + self.assertIsNone(Sample.query.filter(Sample.id == spare_id).first()) + + @mock.patch('mod_api.routes.samples._remove_local') + def test_delete_sample_unlinks_its_files_after_the_row(self, remove): + spare = Sample('spare2_sha', 'txt', 'spare2') + g.db.add(spare) + g.db.flush([spare]) + g.db.add(ExtraFile(spare.id, 'srt', 'companion')) + g.db.commit() + spare_id = spare.id + + # Record whether the row had already gone at each unlink: nothing + # may be removed from disk until the delete is committed. + row_gone = [] + remove.side_effect = lambda path: row_gone.append( + Sample.query.filter(Sample.id == spare_id).first() is None) + + res = self.client.delete( + f'/api/v1/samples/{spare_id}', headers=self._admin('del4')) + + self.assertEqual(res.status_code, 200) + paths = [call.args[0] for call in remove.call_args_list] + # The media file, its media info, and the file uploaded alongside it. + self.assertEqual(len(paths), 3) + self.assertTrue(any('TestFiles/extra/' in p for p in paths)) + self.assertTrue(any(p.endswith('.xml') for p in paths)) + self.assertTrue(all(row_gone)) + + def test_delete_sample_requires_admin(self): + token = self.get_token('samp_user@local.com', 'userpass123', 'del3', + scopes=['runs:write']) + res = self.client.delete( + f'/api/v1/samples/{self.sample_id}', + headers={'Authorization': f'Bearer {token}'}) + + self.assertEqual(res.status_code, 403) + + @mock.patch('mod_api.routes.samples.resolve_artifact') + def test_download_media_info(self, resolve): + resolve.return_value = ('https://signed.url', 'ok') + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/media-info/download', + headers=self._admin('mi1')) + + self.assertEqual(res.status_code, 200) + resolve.assert_called_once_with('TestFiles/media/test_sha.xml') + + @mock.patch('mod_api.routes.samples.resolve_artifact') + def test_download_extra_file(self, resolve): + resolve.return_value = ('https://signed.url', 'ok') + extra = ExtraFile(self.sample_id, 'srt', 'subs.srt') + g.db.add(extra) + g.db.commit() + extra_id = extra.id + + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/extra-files/{extra_id}' + f'/download', + headers=self._admin('ex1')) + + self.assertEqual(res.status_code, 200) + # ExtraFile.filename is _. + resolve.assert_called_once_with( + f'TestFiles/extra/test_sha_{extra_id}.srt') + + def test_download_extra_file_of_another_sample_is_not_found(self): + other = Sample('other_sha', 'txt', 'other') + g.db.add(other) + g.db.commit() + extra = ExtraFile(other.id, 'srt', 'subs.srt') + g.db.add(extra) + g.db.commit() + + res = self.client.get( + f'/api/v1/samples/{self.sample_id}/extra-files/{extra.id}' + f'/download', + headers=self._admin('ex2')) + + self.assertEqual(res.status_code, 404) + def test_get_sample_details_includes_upload_metadata(self): from mod_upload.models import Platform, Upload g.db.add(Upload(self.admin.id, self.sample_id, None, diff --git a/tests/api/test_routes_uploads.py b/tests/api/test_routes_uploads.py new file mode 100644 index 000000000..39955f222 --- /dev/null +++ b/tests/api/test_routes_uploads.py @@ -0,0 +1,252 @@ +import hashlib +import io +import json +from unittest import mock + +from flask import g + +from mod_api.middleware.rate_limit import _rate_limit_store +from mod_auth.models import Role, User +from mod_sample.models import ExtraFile, ForbiddenExtension, Sample +from mod_upload.models import QueuedSample +from tests.api.base import ApiTestCase + +CONTENT = b'\x00\x01two three four' * 64 +SHA = hashlib.sha256(CONTENT).hexdigest() + + +class TestRoutesUploads(ApiTestCase): + def setUp(self): + super().setUp() + self.setup_run_data('upl') + self.admin_id = self.admin.id + self.user_id = self.user.id + + self.bystander = User('testother_upl', Role.user, + 'upl_other@local.com', + User.generate_hash('otherpass123')) + g.db.add(self.bystander) + g.db.commit() + + _rate_limit_store.clear() + + def _admin(self, name): + token = self.get_token('upl_admin@local.com', 'adminpass123', name, + scopes=['runs:read', 'runs:write']) + return {'Authorization': f'Bearer {token}'} + + def _bystander(self, name): + token = self.get_token('upl_other@local.com', 'otherpass123', name, + scopes=['runs:read', 'runs:write']) + return {'Authorization': f'Bearer {token}'} + + def _upload(self, headers, name='clip.ts', content=CONTENT): + return self.client.post( + '/api/v1/samples/upload', + data={'file': (io.BytesIO(content), name)}, + content_type='multipart/form-data', + headers=headers) + + def _queue_row(self, user_id): + queued = QueuedSample(SHA, '.ts', 'clip', user_id) + g.db.add(queued) + g.db.commit() + return queued.id + + def test_upload_requires_a_file_part(self): + res = self.client.post( + '/api/v1/samples/upload', + data={}, content_type='multipart/form-data', + headers=self._admin('up1')) + + self.assertEqual(res.status_code, 400) + + @mock.patch('mod_api.routes.uploads.os.rename') + @mock.patch('mod_api.routes.uploads.open', new_callable=mock.mock_open) + def test_upload_queues_the_file(self, _open, _rename): + res = self._upload(self._admin('up2')) + + self.assertEqual(res.status_code, 201) + # The hash is both the duplicate check and the stored name, so it + # has to be the digest of what was actually read off the stream. + self.assertEqual(res.json['sha'], SHA) + self.assertIsNotNone( + QueuedSample.query.filter(QueuedSample.sha == SHA).first()) + + def test_upload_rejects_an_unusable_file_name(self): + # secure_filename empties this one, and an empty name used to + # resolve to the TempFiles directory and fail on open(). + res = self._upload(self._admin('up2b'), name='...') + + self.assertEqual(res.status_code, 400) + self.assertIsNone( + QueuedSample.query.filter(QueuedSample.sha == SHA).first()) + + @mock.patch('mod_api.routes.uploads.os.rename') + @mock.patch('mod_api.routes.uploads.open', new_callable=mock.mock_open) + def test_uploads_of_one_name_stage_to_separate_files(self, _open, _rename): + # Same file name twice: the staging paths have to differ, or + # concurrent uploads write into each other. + first = self._upload(self._admin('up2c'), name='clash.ts') + second = self._upload(self._bystander('up2d'), name='clash.ts', + content=CONTENT + b'different') + + self.assertEqual(first.status_code, 201) + self.assertEqual(second.status_code, 201) + + staged = [call.args[0] for call in _open.call_args_list] + self.assertEqual(len(staged), 2) + self.assertNotEqual(staged[0], staged[1]) + # And neither is named after what the caller sent. + for path in staged: + self.assertNotIn('clash', path) + + @mock.patch('mod_api.routes.uploads.os.remove') + @mock.patch('mod_api.routes.uploads.open', new_callable=mock.mock_open) + def test_upload_rejects_content_already_in_the_library( + self, _open, remove): + g.db.add(Sample(SHA, 'ts', 'already_here')) + g.db.commit() + + res = self._upload(self._admin('up3')) + + self.assertEqual(res.status_code, 409) + # The half-written temp file is cleaned up rather than left behind. + self.assertTrue(remove.called) + + @mock.patch('mod_api.routes.uploads.open', new_callable=mock.mock_open) + def test_upload_rejects_a_forbidden_extension(self, _open): + g.db.add(ForbiddenExtension('ts')) + g.db.commit() + + res = self._upload(self._admin('up4')) + + self.assertEqual(res.status_code, 403) + self.assertIsNone( + QueuedSample.query.filter(QueuedSample.sha == SHA).first()) + + def test_admin_sees_the_whole_queue(self): + self._queue_row(self.user_id) + + res = self.client.get( + '/api/v1/queued-samples', headers=self._admin('up5')) + + self.assertEqual(res.status_code, 200) + self.assertEqual(len(res.json['data']), 1) + + def test_others_see_only_their_own_queue(self): + self._queue_row(self.user_id) + + res = self.client.get( + '/api/v1/queued-samples', headers=self._bystander('up6')) + + self.assertEqual(res.status_code, 200) + self.assertEqual(res.json['data'], []) + + def test_someone_elses_queued_sample_reads_as_absent(self): + queued_id = self._queue_row(self.user_id) + + res = self.client.get( + f'/api/v1/queued-samples/{queued_id}', + headers=self._bystander('up7')) + + # Not 403: a different reply would let an id be used to find out + # what other people have queued. + self.assertEqual(res.status_code, 404) + + @mock.patch('mod_api.routes.uploads.os.path.isfile', return_value=True) + @mock.patch('mod_api.routes.uploads.os.rename') + def test_finalize_creates_a_sample(self, rename, _isfile): + # Version 1.2.3 is seeded by the test base. + queued_id = self._queue_row(self.admin_id) + + res = self.client.post( + f'/api/v1/queued-samples/{queued_id}/finalize', + data=json.dumps({'version': '1.2.3', 'platform': 'linux'}), + content_type='application/json', + headers=self._admin('up8')) + + self.assertEqual(res.status_code, 201) + self.assertIsNotNone(Sample.query.filter(Sample.sha == SHA).first()) + self.assertIsNone( + QueuedSample.query.filter(QueuedSample.id == queued_id).first()) + # The file only moves once the rows are committed. + self.assertTrue(rename.called) + + @mock.patch('mod_api.routes.uploads.os.path.isfile', return_value=True) + def test_finalize_rejects_an_unknown_version(self, _isfile): + queued_id = self._queue_row(self.admin_id) + + res = self.client.post( + f'/api/v1/queued-samples/{queued_id}/finalize', + data=json.dumps({'version': '9.9.9', 'platform': 'linux'}), + content_type='application/json', + headers=self._admin('up9')) + + self.assertEqual(res.status_code, 400) + + @mock.patch('mod_api.routes.uploads.os.path.isfile', return_value=False) + def test_finalize_refuses_when_the_file_is_gone(self, _isfile): + # Version 1.2.3 is seeded by the test base. + queued_id = self._queue_row(self.admin_id) + + res = self.client.post( + f'/api/v1/queued-samples/{queued_id}/finalize', + data=json.dumps({'version': '1.2.3', 'platform': 'linux'}), + content_type='application/json', + headers=self._admin('up10')) + + self.assertEqual(res.status_code, 409) + # The upload stays queued so it can be retried. + self.assertIsNotNone( + QueuedSample.query.filter(QueuedSample.id == queued_id).first()) + + @mock.patch('mod_api.routes.uploads.os.remove') + def test_delete_queued_sample(self, _remove): + queued_id = self._queue_row(self.admin_id) + + res = self.client.delete( + f'/api/v1/queued-samples/{queued_id}', + headers=self._admin('up11')) + + self.assertEqual(res.status_code, 200) + self.assertIsNone( + QueuedSample.query.filter(QueuedSample.id == queued_id).first()) + + @mock.patch('mod_api.routes.uploads.os.path.isfile', return_value=True) + @mock.patch('mod_api.routes.uploads.os.rename') + def test_link_attaches_the_upload_to_a_sample(self, rename, _isfile): + queued_id = self._queue_row(self.admin_id) + target = Sample('target_sha', 'ts', 'target') + g.db.add(target) + g.db.commit() + target_id = target.id + + res = self.client.post( + f'/api/v1/queued-samples/{queued_id}/link', + data=json.dumps({'sample_id': target_id}), + content_type='application/json', + headers=self._admin('up12')) + + self.assertEqual(res.status_code, 201) + self.assertEqual(res.json['sample_id'], target_id) + self.assertEqual( + ExtraFile.query.filter_by(sample_id=target_id).count(), 1) + self.assertIsNone( + QueuedSample.query.filter(QueuedSample.id == queued_id).first()) + self.assertTrue(rename.called) + + @mock.patch('mod_api.routes.uploads.os.path.isfile', return_value=True) + def test_link_rejects_an_unknown_sample(self, _isfile): + queued_id = self._queue_row(self.admin_id) + + res = self.client.post( + f'/api/v1/queued-samples/{queued_id}/link', + data=json.dumps({'sample_id': 999999}), + content_type='application/json', + headers=self._admin('up13')) + + self.assertEqual(res.status_code, 404) + # The upload stays queued when the link cannot be made. + self.assertIsNotNone( + QueuedSample.query.filter(QueuedSample.id == queued_id).first()) diff --git a/tests/test_auth/test_controllers.py b/tests/test_auth/test_controllers.py index 670170c22..3028599b7 100644 --- a/tests/test_auth/test_controllers.py +++ b/tests/test_auth/test_controllers.py @@ -273,8 +273,15 @@ def test_github_callback_valid_get(self, mock_post, mock_g, mock_user_model): mock_g.db.commit.assert_called_once() mock_g.log.error.assert_not_called() - def test_github_redirect(self): + @mock.patch('requests.Session.post') + def test_github_redirect(self, mock_post): """Test editing account where GitHub token is not null.""" + # The stored token sends manage() through github_token_validity, + # which asks GitHub whether it is still good. Answer that here so + # the test does not depend on reaching api.github.com: CI has no + # client id to ask with, and an intercepted TLS handshake on the + # runner fails the whole suite. + mock_post.return_value = MockResponse({}, 404) self.create_user_with_role( self.user.name, self.user.email, self.user.password, Role.admin, self.user.github_token) with self.app.test_client() as c: