diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml index c9918f2..1412e04 100644 --- a/.github/workflows/ci-checks.yml +++ b/.github/workflows/ci-checks.yml @@ -20,7 +20,7 @@ jobs: - name: "Set up Python 3" uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.12' - name: "Install dependencies" run: | pip install --upgrade pip @@ -38,7 +38,7 @@ jobs: - name: "Set up Python 3" uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.12' - name: "Install dependencies" run: "pip install flake8" - name: "Run flake8!" @@ -53,10 +53,10 @@ jobs: username: ${{ secrets.TEST_DOCKER_USERNAME }} password: ${{ secrets.TEST_DOCKER_PASSWORD }} - uses: actions/checkout@v2 - - name: "Set up Python 3.8" + - name: "Set up Python 3.12" uses: actions/setup-python@v2 - with: - python-version: "3.8" + with: + python-version: "3.12" - name: "Install dependencies" run: | pip install --upgrade pip @@ -79,7 +79,7 @@ jobs: needs: [black, pylint, flake8, pytest-manager] strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Login to Docker Hub uses: docker/login-action@v1 @@ -118,12 +118,13 @@ jobs: - name: "Set up Python 3" uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.12' - name: "Install dependencies" run: | pip install --upgrade pip - pip install -r requirements_manager.txt + pip install -r requirements_manager.txt pip install -r requirements_sdk.txt pip install -r requirements_dev.txt + python -m ipykernel install --user --name python3 - name: "Running E2E tests with pytest" run: "python -m pytest --verbose tests/e2e_test/" \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 7a6b09e..01fd8d1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,18 +1,25 @@ [pylint] -disable = +disable = R0801, - C0330, - C0326, no-self-argument, no-name-in-module, too-few-public-methods, too-many-arguments, + too-many-positional-arguments, logging-fstring-interpolation, fixme, missing-module-docstring, missing-function-docstring, missing-class-docstring, raise-missing-from, - unsubscriptable-object # TODO: Only required in python 3.9 + unsubscriptable-object, + consider-using-with, + use-dict-literal, + missing-timeout, + unspecified-encoding, + useless-option-value, + invalid-name, + import-error -max-line-length = 88 \ No newline at end of file +max-line-length = 88 +ignored-modules = IPython \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 24644d0..86ff11b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ## Stage 1: Build image -FROM python:3.8 AS build-image +FROM python:3.12 AS build-image # Install S2i RUN wget -c https://github.com/openshift/source-to-image/releases/download/v1.3.0/source-to-image-v1.3.0-eed2850f-linux-amd64.tar.gz \ @@ -20,7 +20,7 @@ COPY ./requirements_manager.txt . RUN pip install -r requirements_manager.txt ## Stage 2: Production image -FROM python:3.8-slim AS production-image +FROM python:3.12-slim AS production-image # Install Git RUN apt-get update && apt-get install -y git diff --git a/daeploy/_service/db.py b/daeploy/_service/db.py index 5fd852c..fb84e63 100644 --- a/daeploy/_service/db.py +++ b/daeploy/_service/db.py @@ -8,9 +8,9 @@ from contextlib import contextmanager import json -from sqlalchemy import create_engine, and_ +from sqlalchemy import create_engine, and_, MetaData from sqlalchemy.ext.automap import automap_base -from sqlalchemy.orm import sessionmaker, mapper, clear_mappers +from sqlalchemy.orm import sessionmaker, declarative_base from sqlalchemy import Column, DateTime, Float, Text from daeploy.utilities import get_db_table_limit @@ -20,7 +20,7 @@ SERVICE_DB_PATH = Path("service_db.db") ENGINE = create_engine(f"sqlite:///{str(SERVICE_DB_PATH)}") -Base = automap_base() +Base = declarative_base() Session = sessionmaker(bind=ENGINE) QUEUE = queue.Queue() @@ -69,9 +69,6 @@ def create_new_ts_table(name: str, dtype: Type) -> Type: # Create the actual table MapperClass.__table__.create(ENGINE, checkfirst=True) - # Map everything - mapper(MapperClass, MapperClass.__table__) - LOGGER.info(f"Created new table for variable {name}") return MapperClass @@ -221,16 +218,17 @@ def initialize_db(): global QUEUE QUEUE = queue.Queue() global TABLES - Base.prepare(ENGINE, reflect=True) # Automap any existing tables - TABLES = dict(Base.classes) # Make sure we keep track of the auto-mapped tables + # Reflect any existing tables using automap + AutoBase = automap_base(metadata=MetaData()) + AutoBase.prepare(autoload_with=ENGINE) + TABLES = dict(AutoBase.classes) WRITER_THREAD.start() LOGGER.info("DB started!") def remove_db(): """Remove db""" - global WRITER_THREAD - global QUEUE + global WRITER_THREAD, Base # Stop and join writer thread if alive if WRITER_THREAD.is_alive(): @@ -240,10 +238,16 @@ def remove_db(): # Reset it WRITER_THREAD = threading.Thread(target=_writer, daemon=True) + # Reset tables tracking + TABLES.clear() + # Remove db - SERVICE_DB_PATH.unlink() + ENGINE.dispose() + try: + SERVICE_DB_PATH.unlink() + except FileNotFoundError: + pass - # Reset mappers and metadata object - clear_mappers() - Base.metadata.clear() + # Reset base so new tables get fresh mappers + Base = declarative_base() LOGGER.info("DB has been shut down!") diff --git a/daeploy/_service/service.py b/daeploy/_service/service.py index 19d2a7f..aca252e 100644 --- a/daeploy/_service/service.py +++ b/daeploy/_service/service.py @@ -15,7 +15,7 @@ from fastapi.encoders import jsonable_encoder from fastapi.concurrency import run_in_threadpool from fastapi.middleware.cors import CORSMiddleware -from pydantic import create_model, validate_arguments +from pydantic import create_model, validate_call from daeploy._service.logger import setup_logging from daeploy._service.db import clean_database, initialize_db, remove_db, write_to_ts @@ -33,7 +33,6 @@ ) from daeploy.communication import notify, Severity - setup_logging() logger = logging.getLogger(__name__) @@ -219,7 +218,7 @@ async def wrapper(_request: Request, *args, **kwargs): _disable_http_logs(path) # Wrap the original func in a pydantic validation wrapper and return that - return validate_arguments(deco_func) + return validate_call(deco_func) # This ensures that we can use the decorator with or without arguments if not (callable(func) or func is None): @@ -370,7 +369,7 @@ def add_parameter( if isinstance(value, Number): value = float(value) - @validate_arguments() + @validate_call() def update_parameter(value: value.__class__) -> Any: logger.info(f"Parameter {parameter} changed to {value}") self.parameters[parameter]["value"] = value diff --git a/daeploy/cli/cli.py b/daeploy/cli/cli.py index ce6d3a4..55bc5cc 100644 --- a/daeploy/cli/cli.py +++ b/daeploy/cli/cli.py @@ -5,7 +5,7 @@ import os import json -import pkg_resources +from importlib.metadata import version as get_version, PackageNotFoundError import pytest import requests import typer @@ -73,9 +73,9 @@ def version_callback(value: bool): # Get SDK Version try: - sdk_version = pkg_resources.get_distribution("daeploy").version + sdk_version = get_version("daeploy") typer.echo(f"SDK version: {sdk_version}") - except pkg_resources.DistributionNotFound: + except PackageNotFoundError: pass # Get Manager Version @@ -654,13 +654,13 @@ def init( raise typer.Exit(1) # Find out which daeploy version that should be used by the service try: - dist = pkg_resources.get_distribution("daeploy") + daeploy_version = get_version("daeploy") daeploy_specifier = ( - str(dist.as_requirement()) - if dist.version != "0.0.0.dev0" - else dist.project_name + f"daeploy=={daeploy_version}" + if daeploy_version != "0.0.0.dev0" + else "daeploy" ) # Use full specificer unless in dev environment, then just go for the latest - except pkg_resources.DistributionNotFound: + except PackageNotFoundError: typer.echo( "`daeploy` package not found, assuming latest version " "should be used for the generated project." diff --git a/daeploy/cli/user.py b/daeploy/cli/user.py index f7083f8..75f1bb4 100644 --- a/daeploy/cli/user.py +++ b/daeploy/cli/user.py @@ -5,7 +5,6 @@ from daeploy.cli import cliutils - app = typer.Typer(help="Collection of user management commands") typer.Option(None, "-p", "--password", expose_value=False) diff --git a/daeploy/data_types.py b/daeploy/data_types.py index b2ec2a3..d55f266 100644 --- a/daeploy/data_types.py +++ b/daeploy/data_types.py @@ -1,20 +1,24 @@ -# pylint: disable=too-many-ancestors +# pylint: disable=too-many-ancestors, unused-argument from typing import Any, List, Dict import numpy as np import pandas as pd +from pydantic import GetCoreSchemaHandler +from pydantic_core import core_schema class ArrayInput(np.ndarray): """Pydantic compatible data type for numpy ndarray input.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="array", items={}) + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "array", "items": {}} @classmethod def validate(cls, value: List) -> np.ndarray: @@ -26,12 +30,14 @@ class ArrayOutput(np.ndarray): """Pydantic compatible data type for numpy ndarray output.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="array", items={}) + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "array", "items": {}} @classmethod def validate(cls, value: np.ndarray) -> List: @@ -43,16 +49,18 @@ class DataFrameInput(pd.DataFrame): """Pydantic compatible data type for pandas DataFrame input.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="object") + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "object"} @classmethod def validate(cls, value: Dict[str, Any]) -> pd.DataFrame: - # Transform input to ndarray + # Transform input to DataFrame return pd.DataFrame.from_dict(value) @@ -60,14 +68,16 @@ class DataFrameOutput(pd.DataFrame): """Pydantic compatible data type for pandas DataFrame output.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__( + cls, source_type: Any, handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + return core_schema.no_info_plain_validator_function(cls.validate) @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="object") + def __get_pydantic_json_schema__(cls, schema, handler): + return {"type": "object"} @classmethod def validate(cls, value: pd.DataFrame) -> Dict[str, Any]: - # Transform input to ndarray + # Transform DataFrame to dict return value.to_dict() diff --git a/manager/app.py b/manager/app.py index 6cb3f5b..3015f9b 100644 --- a/manager/app.py +++ b/manager/app.py @@ -19,7 +19,6 @@ from manager.database import service_db from manager.constants import get_manager_version, cors_enabled, cors_config - # Setup logger logging_api.setup_logging() LOGGER = logging.getLogger(__name__) diff --git a/manager/constants.py b/manager/constants.py index 081bf29..6428d14 100644 --- a/manager/constants.py +++ b/manager/constants.py @@ -1,6 +1,7 @@ """ Constants and config """ + import os from pathlib import Path diff --git a/manager/data_models/request_models.py b/manager/data_models/request_models.py index 10e3f13..6b38a91 100644 --- a/manager/data_models/request_models.py +++ b/manager/data_models/request_models.py @@ -3,8 +3,8 @@ import semver from pydantic.types import SecretStr -from pydantic import BaseModel, validator, HttpUrl -from fastapi import Path, UploadFile +from pydantic import BaseModel, field_validator, HttpUrl, ConfigDict +from fastapi import UploadFile, Query from manager.constants import ( DAEPLOY_DEFAULT_INTERNAL_PORT, @@ -16,8 +16,8 @@ class BaseService(BaseModel): name: str version: str - # pylint: disable=no-self-use - @validator("name") + @field_validator("name") + @classmethod def must_adhere_to_docker_requirements(cls, name): # Only allow a name to contain lower case letters, numbers and underscore # anywhere but in the beginning and end @@ -29,16 +29,16 @@ def must_adhere_to_docker_requirements(cls, name): ) return name - # pylint: disable=no-self-use - @validator("version") + @field_validator("version") + @classmethod def must_be_semver_string(cls, version): - if not semver.VersionInfo.isvalid(version): + if not semver.Version.is_valid(version): raise ValueError("Version must be a semantic version string.") return version class BaseNewServiceRequest(BaseService): - port: int = Path(default=DAEPLOY_DEFAULT_INTERNAL_PORT, gt=0) + port: int = Query(default=DAEPLOY_DEFAULT_INTERNAL_PORT, gt=0) run_args: Dict = {} @@ -49,8 +49,8 @@ class BaseNewS2IServiceRequest(BaseNewServiceRequest): class ServiceImageRequest(BaseNewServiceRequest): image: str - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -59,13 +59,14 @@ class Config: "run_args": {}, } } + ) class ServiceGitRequest(BaseNewS2IServiceRequest): git_url: HttpUrl - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -74,13 +75,14 @@ class Config: "run_args": {}, } } + ) class ServiceTarRequest(BaseNewS2IServiceRequest): file: UploadFile - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -89,13 +91,14 @@ class Config: "run_args": {}, } } + ) class ServicePickleRequest(ServiceTarRequest): requirements: List[str] - class Config: - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ "example": { "name": "myservice", "version": "0.0.1", @@ -104,6 +107,7 @@ class Config: "requirements": [], } } + ) class NotificationRequest(BaseModel): @@ -112,7 +116,7 @@ class NotificationRequest(BaseModel): msg: str severity: int dashboard: bool - emails: Union[List[str], None] + emails: Union[List[str], None] = None timer: int timestamp: str diff --git a/manager/data_models/response_models.py b/manager/data_models/response_models.py index 58c710a..d4f15ce 100644 --- a/manager/data_models/response_models.py +++ b/manager/data_models/response_models.py @@ -34,7 +34,7 @@ class StateResponse(BaseModel): Error: str StartedAt: str FinishedAt: str - Health: Optional[HealthResponse] + Health: Optional[HealthResponse] = None class NetworkSettingsResponse(BaseModel): @@ -45,8 +45,8 @@ class NetworkSettingsResponse(BaseModel): LinkLocalIPv6PrefixLen: int Ports: dict SandboxKey: str - SecondaryIPAddresses: Optional[str] - SecondaryIPv6Addresses: Optional[str] + SecondaryIPAddresses: Optional[str] = None + SecondaryIPv6Addresses: Optional[str] = None EndpointID: str Gateway: str GlobalIPv6Address: str @@ -76,7 +76,7 @@ class InspectResponse(BaseModel): MountLabel: str ProcessLabel: str AppArmorProfile: str - ExecIDs: Optional[List[str]] + ExecIDs: Optional[List[str]] = None HostConfig: dict GraphDriver: dict Mounts: list diff --git a/manager/database/auth_db.py b/manager/database/auth_db.py index 2865b94..0d73595 100644 --- a/manager/database/auth_db.py +++ b/manager/database/auth_db.py @@ -19,7 +19,8 @@ def add_user_record(username: str, password: str): """ with session_scope() as session: new_user = User( - name=username, password=bcrypt.hashpw(password.encode(), bcrypt.gensalt()) + name=username, + password=bcrypt.hashpw(password.encode(), bcrypt.gensalt()).decode(), ) session.add(new_user) diff --git a/manager/database/database.py b/manager/database/database.py index be8a6d4..ae2bc73 100644 --- a/manager/database/database.py +++ b/manager/database/database.py @@ -4,8 +4,7 @@ import logging from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, declarative_base from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey from manager.constants import DAEPLOY_DATA_DIR, get_admin_password @@ -128,9 +127,8 @@ def initialize_db(): def remove_db(): """Removes db""" + engine.dispose() try: MANAGER_DB_PATH.unlink() except FileNotFoundError: - # Path.unlink(missing_ok=True) gives same behavior but was - # not introduced until python 3.8 pass diff --git a/manager/routers/auth_api.py b/manager/routers/auth_api.py index a528335..7b0e268 100644 --- a/manager/routers/auth_api.py +++ b/manager/routers/auth_api.py @@ -115,8 +115,9 @@ def show_login_page(request: Request, destination: Optional[str] = "/"): # noqa: DAR101,DAR201,DAR401 """ return TEMPLATES.TemplateResponse( - "login.html", - {"request": request, "ACTION": f"/auth/login?destination={destination}"}, + request=request, + name="login.html", + context={"ACTION": f"/auth/login?destination={destination}"}, status_code=401, ) @@ -143,7 +144,12 @@ def login_user( LOGGER.exception(f"User {username} failed to login!") return RedirectResponse(url=destination, status_code=303) - if not bcrypt.checkpw(password.get_secret_value().encode(), record.password): + stored_pw = ( + record.password.encode() + if isinstance(record.password, str) + else record.password + ) + if not bcrypt.checkpw(password.get_secret_value().encode(), stored_pw): return RedirectResponse(url=destination, status_code=303) # Construct token diff --git a/manager/routers/dashboard_api.py b/manager/routers/dashboard_api.py index 8cc6982..6065735 100644 --- a/manager/routers/dashboard_api.py +++ b/manager/routers/dashboard_api.py @@ -2,9 +2,7 @@ from datetime import datetime import dash -from dash import dcc -from dash import html -from dash.dependencies import Input, Output +from dash import dcc, html, Input, Output from manager.routers.service_api import read_services, inspect_service from manager.routers.notification_api import get_notifications, delete_notifications @@ -67,12 +65,10 @@ def build_banner(): id="banner-text", children=[ html.Img(src=app.get_asset_url("daeploy_white_icon.png")), - dcc.Markdown( - """ + dcc.Markdown(""" ### Daeploy Dashboard by Viking Analytics AB - """ - ), + """), ], ), ], @@ -160,9 +156,11 @@ def generate_table_services(): html.Tr( # Main/Shadow [ - html.Td("*", className="green-text") - if service["main"] - else html.Td("") + ( + html.Td("*", className="green-text") + if service["main"] + else html.Td("") + ) ] + # Name diff --git a/manager/routers/service_api.py b/manager/routers/service_api.py index 35a059e..55d4a3c 100644 --- a/manager/routers/service_api.py +++ b/manager/routers/service_api.py @@ -87,7 +87,7 @@ def new_service_from_git_repo(service_request: ServiceGitRequest): """ check_service_exists(service_request.name, service_request.version) - image = build_service_image_s2i(service_request.git_url, service_request) + image = build_service_image_s2i(str(service_request.git_url), service_request) start_service_from_image(image, service_request) return "Accepted" diff --git a/manager/runtime_connectors.py b/manager/runtime_connectors.py index 75382b9..f88d59a 100644 --- a/manager/runtime_connectors.py +++ b/manager/runtime_connectors.py @@ -73,7 +73,13 @@ async def service_logs(self, service, tail, follow, since): class LocalDockerConnector(ConnectorBase): CLIENT = docker.from_env() - AIO_CLIENT = aiodocker.Docker() + _AIO_CLIENT = None + + @classmethod + def _get_aio_client(cls): + if cls._AIO_CLIENT is None: + cls._AIO_CLIENT = aiodocker.Docker() + return cls._AIO_CLIENT def __init__(self): # Create our own docker network @@ -395,7 +401,7 @@ async def service_logs( AsyncGenerator[str, None]: Async infinite generator if following, else async finite generator. """ - container = await self.AIO_CLIENT.containers.get( + container = await self._get_aio_client().containers.get( create_container_name(service.name, service.version) ) diff --git a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt index 7ee9e67..3bfd672 100644 --- a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt +++ b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/requirements.txt @@ -1,2 +1,2 @@ -daeploy==0.4.6 +daeploy==1.3.1 pandas diff --git a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py index aeb3b3a..0cbb3e6 100644 --- a/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py +++ b/manager/templates/daeploy_pickle_template/{{cookiecutter.project_name}}/service.py @@ -46,7 +46,7 @@ def predict(data: dict) -> List[Any]: logger.info(f"Recieved data: \n{data_df}") y_pred = model.predict(data_df) logger.info(f"Predicted: {y_pred}") - return list(y_pred) + return y_pred.tolist() if __name__ == "__main__": diff --git a/pytest.ini b/pytest.ini index ad16166..c1fd712 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,3 @@ [pytest] -testpaths = tests/ \ No newline at end of file +testpaths = tests/ +timeout = 180 \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index 22749eb..b040166 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -5,11 +5,14 @@ pytest pytest-sphinx pytest-asyncio pytest-pinned -pylint==2.7.4 +pytest-timeout +pylint black flake8 darglint streamlit +httpx async_asgi_testclient scikit-learn -nbconvert \ No newline at end of file +nbconvert +ipykernel \ No newline at end of file diff --git a/requirements_manager.txt b/requirements_manager.txt index d1b47a4..d8fad21 100644 --- a/requirements_manager.txt +++ b/requirements_manager.txt @@ -1,14 +1,14 @@ -fastapi==0.78.0 -uvicorn==0.16.0 -docker==5.0.3 -aiodocker==0.21.0 -semver==2.13.0 -python-multipart==0.0.5 +fastapi==0.135.3 +uvicorn==0.44.0 +docker==7.1.0 +aiodocker==0.26.0 +semver==3.0.4 +python-multipart==0.0.26 toml==0.10.2 -sqlalchemy==1.3.22 -dash==2.4.1 -pyjwt==2.4.0 -bcrypt==3.2.0 -jinja2==3.0.3 -cookiecutter==1.7.3 -cryptography==3.3.2 +sqlalchemy==2.0.49 +dash==4.1.0 +pyjwt==2.12.1 +bcrypt==5.0.0 +jinja2==3.1.6 +cookiecutter==2.7.1 +cryptography==46.0.7 diff --git a/setup.py b/setup.py index 310a4a8..1020618 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "Programming Language :: Python :: 3", "Operating System :: OS Independent", ], - python_requires=">=3.6", + python_requires=">=3.9", install_requires=required, entry_points={ "console_scripts": ["daeploy=daeploy.cli.cli:app"], diff --git a/tests/conftest.py b/tests/conftest.py index 3b0218a..09e965e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,10 +44,10 @@ def test_client_logged_in(test_client: TestClient, auth_enabled, database): response = test_client.post( "/auth/login", data={"username": "admin", "password": "admin"}, - allow_redirects=False, + follow_redirects=False, ) # Check that we have access! - response = test_client.get("/auth/verify", allow_redirects=False) + response = test_client.get("/auth/verify", follow_redirects=False) assert response.status_code == 200 yield test_client # Logs out when removing cookies in parent fixture diff --git a/tests/e2e_test/downstream/downstream.py b/tests/e2e_test/downstream/downstream.py index 222014a..f30b704 100644 --- a/tests/e2e_test/downstream/downstream.py +++ b/tests/e2e_test/downstream/downstream.py @@ -1,6 +1,7 @@ """ File used as a service in e2e tests. """ + import logging import time from pydantic import BaseModel diff --git a/tests/e2e_test/e2e_test.py b/tests/e2e_test/e2e_test.py index bebb0b6..2e4a4ac 100644 --- a/tests/e2e_test/e2e_test.py +++ b/tests/e2e_test/e2e_test.py @@ -1,5 +1,7 @@ import docker import pytest +import sys +import subprocess import time import requests import uuid @@ -9,7 +11,6 @@ import re import shutil -from setuptools import sandbox from pathlib import Path from typer.testing import CliRunner import nbformat @@ -72,22 +73,54 @@ def cli_auth_login(dummy_manager, cli_auth): @pytest.fixture(scope="module") -def pickle_service(cli_auth_login, headers): +def pickle_service(cli_auth_login, dummy_manager, headers): data = { "name": "pickle", "version": "0.1.0", "port": 8000, - "requirements": ["pandas", "sklearn"], + "requirements": ["pandas", "scikit-learn"], } - requests.request( + response = requests.request( "POST", url="http://localhost/services/~pickle", data=data, headers=headers, files={"file": ("filename", open(THIS_DIR / "pickle_e2e_testing.pkl", "rb"))}, ) - time.sleep(5) # Grace period + assert response.status_code == 202, response.text + + # Poll for the pickle service to be reachable; pandas + scikit-learn + # make the s2i build several minutes long, and the service still needs + # time to start up after the container appears. + deadline = time.time() + 800 + reachable = False + while time.time() < deadline: + try: + r = requests.get( + "http://localhost/services/pickle/openapi.json", + headers=headers, + timeout=5, + ) + if r.status_code == 200: + reachable = True + break + except requests.RequestException: + pass + time.sleep(5) + if not reachable: + client = docker.from_env() + print("Containers:", [c.name for c in client.containers.list(all=True)]) + print( + "dummy_manager logs:\n", + dummy_manager.logs(tail=200).decode(errors="replace"), + ) + for c in client.containers.list(all=True): + if "pickle" in c.name: + print( + f"{c.name} status={c.status} logs:\n", + c.logs(tail=200).decode(errors="replace"), + ) try: yield finally: @@ -168,9 +201,16 @@ def generate_requirements_file_for_service(service_folder): the path to the wheel file which contains the daeploy package. """ # TODO: No need to run the setup twice... - sandbox.run_setup( - str(THIS_DIR.parent.parent / "setup.py"), - ["bdist_wheel", "--dist-dir", str(service_folder)], + subprocess.run( + [ + sys.executable, + "setup.py", + "bdist_wheel", + "--dist-dir", + str(service_folder), + ], + cwd=str(THIS_DIR.parent.parent), + check=True, ) with (service_folder / "requirements.txt").open("w") as file_handle: file_handle.write(WHEEL_FILE_NAME) @@ -479,6 +519,7 @@ def test_docs_page_from_service_shows_correct_docs( assert "0.1.0" in service_docs.text +@pytest.mark.timeout(900) def test_service_from_pickle_endpoint(dummy_manager, pickle_service, headers): client = docker.from_env() containers = [con.name for con in client.containers.list()] @@ -492,7 +533,14 @@ def test_service_from_pickle_endpoint(dummy_manager, pickle_service, headers): json=data, headers=headers, ) - assert resp.status_code == 200 + if resp.status_code != 200: + for c in client.containers.list(all=True): + if "pickle" in c.name: + print( + f"{c.name} status={c.status} logs:\n", + c.logs(tail=200).decode(errors="replace"), + ) + assert resp.status_code == 200, resp.text # Test documentation started properly response = requests.get( diff --git a/tests/e2e_test/pickle_e2e_testing.pkl b/tests/e2e_test/pickle_e2e_testing.pkl index 27304fc..f712db5 100644 Binary files a/tests/e2e_test/pickle_e2e_testing.pkl and b/tests/e2e_test/pickle_e2e_testing.pkl differ diff --git a/tests/manager_test/admin_test.py b/tests/manager_test/admin_test.py index 07f2f30..a6b490f 100644 --- a/tests/manager_test/admin_test.py +++ b/tests/manager_test/admin_test.py @@ -38,7 +38,7 @@ def change_user(client, username, password): client.post( "/auth/login", data={"username": username, "password": password}, - allow_redirects=False, + follow_redirects=False, ) diff --git a/tests/manager_test/auth_test.py b/tests/manager_test/auth_test.py index 13c3e7a..159d201 100644 --- a/tests/manager_test/auth_test.py +++ b/tests/manager_test/auth_test.py @@ -33,7 +33,7 @@ def test_login_page(exclude_middleware): def test_verification_without_auth(database): - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 200 @@ -43,39 +43,39 @@ def test_failed_login(database, auth_enabled): response = client.post( "/auth/login", data={"username": "admin", "password": "wrongpassword"}, - allow_redirects=False, + follow_redirects=False, ) assert response.status_code == 303 # No access after - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 303 def test_cookie_token(database, auth_enabled): # No access from beginning - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 303 # Login response = client.post( "/auth/login", data={"username": "admin", "password": "admin"}, - allow_redirects=False, + follow_redirects=False, ) assert response.status_code == 303 # Check that we have access! - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 200 # Logout - response = client.get("/auth/logout", allow_redirects=False) + response = client.get("/auth/logout", follow_redirects=False) assert response.status_code == 303 # No access at the end - response = client.get("/auth/verify", allow_redirects=False) + response = client.get("/auth/verify", follow_redirects=False) assert response.status_code == 303 @@ -85,7 +85,7 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer mumbojumbo"}, - allow_redirects=True, + follow_redirects=True, ) assert response.status_code == 401 @@ -111,7 +111,7 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer {token}"}, - allow_redirects=False, + follow_redirects=False, ) assert response.status_code == 200 @@ -119,7 +119,7 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer {token}"}, - allow_redirects=True, + follow_redirects=True, ) assert response.status_code == 200 @@ -131,6 +131,6 @@ def test_API_token(clear_cookies, database, auth_enabled): response = client.get( "/auth/verify", headers={"Authorization": f"Bearer {token}"}, - allow_redirects=True, + follow_redirects=True, ) assert response.status_code == 401 diff --git a/tests/manager_test/endpoint_test.py b/tests/manager_test/endpoint_test.py index 4098bcb..fa66180 100644 --- a/tests/manager_test/endpoint_test.py +++ b/tests/manager_test/endpoint_test.py @@ -10,6 +10,7 @@ import pytest from async_asgi_testclient import TestClient as AsyncTestClient from docker.errors import ImageNotFound +from fastapi.exceptions import ResponseValidationError from fastapi.testclient import TestClient from manager import proxy from manager.routers import logging_api, notification_api, service_api @@ -17,7 +18,6 @@ from manager.constants import DAEPLOY_DEFAULT_S2I_BUILD_IMAGE, get_manager_version from manager.data_models.request_models import BaseService - client = TestClient(app) async_client = AsyncTestClient(app) @@ -195,12 +195,12 @@ def test_post_services_git_request( response = client.post("/services/~git", json=req) assert response.status_code == 202 - service_api.run_s2i.assert_called_with( - url="https://github.com/sclorg/django-ex", - build_image=DAEPLOY_DEFAULT_S2I_BUILD_IMAGE, - name=SERVICE_NAME, - version=SERVICE_VERSION, - ) + service_api.run_s2i.assert_called_once() + call_kwargs = service_api.run_s2i.call_args[1] + assert str(call_kwargs["url"]) == "https://github.com/sclorg/django-ex" + assert call_kwargs["build_image"] == DAEPLOY_DEFAULT_S2I_BUILD_IMAGE + assert call_kwargs["name"] == SERVICE_NAME + assert call_kwargs["version"] == SERVICE_VERSION assert response.json() == "Accepted" @@ -228,12 +228,12 @@ def test_post_services_git_request_changed_builder_image( response = client.post("/services/~git", json=req) assert response.status_code == 202 - service_api.run_s2i.assert_called_with( - url="https://github.com/sclorg/django-ex", - build_image="centos/python-38-centos7", - name=SERVICE_NAME, - version=SERVICE_VERSION, - ) + service_api.run_s2i.assert_called_once() + call_kwargs = service_api.run_s2i.call_args[1] + assert str(call_kwargs["url"]) == "https://github.com/sclorg/django-ex" + assert call_kwargs["build_image"] == "centos/python-38-centos7" + assert call_kwargs["name"] == SERVICE_NAME + assert call_kwargs["version"] == SERVICE_VERSION assert response.json() == "Accepted" @@ -438,8 +438,10 @@ def test_service_delete(mocked_docker_connection, database): ) service_name = SERVICE_NAME service_version = SERVICE_VERSION - response = client.delete( - "/services/", json={"name": service_name, "version": service_version} + response = client.request( + "DELETE", + "/services/", + json={"name": service_name, "version": service_version}, ) assert response.status_code == 200 @@ -462,7 +464,8 @@ def test_service_delete_keep_image(mocked_docker_connection, database): ) service_name = SERVICE_NAME service_version = SERVICE_VERSION - response = client.delete( + response = client.request( + "DELETE", "/services/", json={"name": service_name, "version": service_version}, params={"remove_image": False}, @@ -576,7 +579,7 @@ def test_service_inspection(mocked_docker_connection): service_name = SERVICE_NAME service_version = SERVICE_VERSION - with pytest.raises(pydantic.ValidationError): + with pytest.raises((pydantic.ValidationError, ResponseValidationError)): client.get( f"/services/~inspection?name={service_name}&version={service_version}" ) diff --git a/tests/manager_test/local_docker_connection_test.py b/tests/manager_test/local_docker_connection_test.py index 98791e3..608b99a 100644 --- a/tests/manager_test/local_docker_connection_test.py +++ b/tests/manager_test/local_docker_connection_test.py @@ -211,13 +211,13 @@ async def test_service_logs(local_docker_connection): def check_required_inspection_keys(container_info): - assert set(InspectResponse.schema()["required"]).issubset( + assert set(InspectResponse.model_json_schema()["required"]).issubset( set(container_info.keys()) ) - assert set(NetworkSettingsResponse.schema()["required"]).issubset( + assert set(NetworkSettingsResponse.model_json_schema()["required"]).issubset( set(container_info["NetworkSettings"].keys()) ) - assert set(StateResponse.schema()["required"]).issubset( + assert set(StateResponse.model_json_schema()["required"]).issubset( set(container_info["State"].keys()) ) diff --git a/tests/manager_test/notifications_test.py b/tests/manager_test/notifications_test.py index 612c608..2877c8c 100644 --- a/tests/manager_test/notifications_test.py +++ b/tests/manager_test/notifications_test.py @@ -97,7 +97,7 @@ def test_email_notification_not_send_when_frozen(email_func, notifications_dict) notification_api.new_notification(notification_3) notification_api.new_notification(notification_3) # The email func is only called once! - email_func.called_once() + email_func.assert_called_once() @patch("manager.routers.notification_api._send_notification_as_email") diff --git a/tests/sdk_test/cli_test.py b/tests/sdk_test/cli_test.py index ba79025..7b3332f 100644 --- a/tests/sdk_test/cli_test.py +++ b/tests/sdk_test/cli_test.py @@ -143,7 +143,8 @@ def test_version_flag_without_manager(): ["--version"], ) assert result.exit_code == 0 - assert "Manager" not in result.stdout + assert "SDK version" in result.stdout + assert "Manager version:" not in result.stdout def test_deploy_from_git_source(dummy_manager, cli_auth_login, clean_services): @@ -814,7 +815,7 @@ def test_logs_date_format(cli_auth_login, clean_services): ["logs", "test_service", "1.0.0", "--date", "2020/01/24"], ) assert logs.exit_code == 2 - assert "does not match the formats" in logs.stdout + assert "does not match the formats" in logs.output logs = runner.invoke( app, ["logs", "test_service", "1.0.0", "--date", "1970-01-24"], diff --git a/tests/sdk_test/daeploy_test.py b/tests/sdk_test/daeploy_test.py index d16c186..1cac8ca 100644 --- a/tests/sdk_test/daeploy_test.py +++ b/tests/sdk_test/daeploy_test.py @@ -508,7 +508,7 @@ def test_local_invocation_pydantic_validation(): assert valid_entrypoint_method_args(32, "Urban") == "hello" # Args of wrong type! - with pytest.raises(pydantic.error_wrappers.ValidationError): + with pytest.raises(pydantic.ValidationError): wrapped(32, "Urban") assert wrapped("Urban", 32) == "hello" @@ -589,7 +589,8 @@ def test_entrypoint_get(): client = TestClient(service.app) req = {"name": "Rune", "age": 100} - response = client.get( + response = client.request( + "GET", "/valid_entrypoint_method_args", json=req, headers={"accept": "application/json"},