Updating requirements and fixing models.

.github/workflows/ci-checks.yml

@@ -20,7 +20,7 @@ jobs:
         -   name: "Set up Python 3"
             uses: actions/setup-python@v2
             with: 
-                python-version: '3.8'
+                python-version: '3.12'
         -   name: "Install dependencies"
             run: |
                 pip install --upgrade pip
@@ -38,7 +38,7 @@ jobs:
         -   name: "Set up Python 3"
             uses: actions/setup-python@v2
             with: 
-                python-version: '3.8'
+                python-version: '3.12'
         -   name: "Install dependencies"
             run: "pip install flake8"
         -   name: "Run flake8!"
@@ -53,10 +53,10 @@ jobs:
             username: ${{ secrets.TEST_DOCKER_USERNAME }}
             password: ${{ secrets.TEST_DOCKER_PASSWORD }}  
         -   uses: actions/checkout@v2
-        -   name: "Set up Python 3.8"
+        -   name: "Set up Python 3.12"
             uses: actions/setup-python@v2
-            with: 
-                python-version: "3.8"
+            with:
+                python-version: "3.12"
         -   name: "Install dependencies"
             run: |
                 pip install --upgrade pip
@@ -79,7 +79,7 @@ jobs:
         needs: [black, pylint, flake8, pytest-manager]
         strategy:
             matrix:
-              python-version: [3.6, 3.7, 3.8, 3.9]
+              python-version: ["3.10", "3.11", "3.12"]
         steps: 
         - name: Login to Docker Hub
           uses: docker/login-action@v1
@@ -118,12 +118,13 @@ jobs:
         -   name: "Set up Python 3"
             uses: actions/setup-python@v2
             with: 
-                python-version: '3.8'
+                python-version: '3.12'
         -   name: "Install dependencies"
             run: |
                 pip install --upgrade pip
-                pip install -r requirements_manager.txt 
+                pip install -r requirements_manager.txt
                 pip install -r requirements_sdk.txt
                 pip install -r requirements_dev.txt
+                python -m ipykernel install --user --name python3
         -   name: "Running E2E tests with pytest"
             run: "python -m pytest --verbose tests/e2e_test/"

.pylintrc

@@ -1,18 +1,25 @@
 [pylint]
-disable = 
+disable =
     R0801,
-    C0330, 
-    C0326,
     no-self-argument,
     no-name-in-module,
     too-few-public-methods,
     too-many-arguments,
+    too-many-positional-arguments,
     logging-fstring-interpolation,
     fixme,
     missing-module-docstring,
     missing-function-docstring,
     missing-class-docstring,
     raise-missing-from,
-    unsubscriptable-object # TODO: Only required in python 3.9
+    unsubscriptable-object,
+    consider-using-with,
+    use-dict-literal,
+    missing-timeout,
+    unspecified-encoding,
+    useless-option-value,
+    invalid-name,
+    import-error
 
-max-line-length = 88
+max-line-length = 88
+ignored-modules = IPython

Dockerfile

@@ -1,5 +1,5 @@
 ## Stage 1: Build image
-FROM python:3.8 AS build-image
+FROM python:3.12 AS build-image
 
 # Install S2i
 RUN wget -c https://github.com/openshift/source-to-image/releases/download/v1.3.0/source-to-image-v1.3.0-eed2850f-linux-amd64.tar.gz \
@@ -20,7 +20,7 @@ COPY ./requirements_manager.txt .
 RUN pip install -r requirements_manager.txt
 
 ## Stage 2: Production image
-FROM python:3.8-slim AS production-image
+FROM python:3.12-slim AS production-image
 
 # Install Git
 RUN apt-get update && apt-get install -y git

daeploy/_service/db.py

@@ -8,9 +8,9 @@
 from contextlib import contextmanager
 import json
 
-from sqlalchemy import create_engine, and_
+from sqlalchemy import create_engine, and_, MetaData
 from sqlalchemy.ext.automap import automap_base
-from sqlalchemy.orm import sessionmaker, mapper, clear_mappers
+from sqlalchemy.orm import sessionmaker, declarative_base
 from sqlalchemy import Column, DateTime, Float, Text
 
 from daeploy.utilities import get_db_table_limit
@@ -20,7 +20,7 @@
 SERVICE_DB_PATH = Path("service_db.db")
 
 ENGINE = create_engine(f"sqlite:///{str(SERVICE_DB_PATH)}")
-Base = automap_base()
+Base = declarative_base()
 Session = sessionmaker(bind=ENGINE)
 
 QUEUE = queue.Queue()
@@ -69,9 +69,6 @@ def create_new_ts_table(name: str, dtype: Type) -> Type:
     # Create the actual table
     MapperClass.__table__.create(ENGINE, checkfirst=True)
 
-    # Map everything
-    mapper(MapperClass, MapperClass.__table__)
-
     LOGGER.info(f"Created new table for variable {name}")
 
     return MapperClass
@@ -221,16 +218,17 @@ def initialize_db():
     global QUEUE
     QUEUE = queue.Queue()
     global TABLES
-    Base.prepare(ENGINE, reflect=True)  # Automap any existing tables
-    TABLES = dict(Base.classes)  # Make sure we keep track of the auto-mapped tables
+    # Reflect any existing tables using automap
+    AutoBase = automap_base(metadata=MetaData())
+    AutoBase.prepare(autoload_with=ENGINE)
+    TABLES = dict(AutoBase.classes)
     WRITER_THREAD.start()
     LOGGER.info("DB started!")
 
 
 def remove_db():
     """Remove db"""
-    global WRITER_THREAD
-    global QUEUE
+    global WRITER_THREAD, Base
 
     # Stop and join writer thread if alive
     if WRITER_THREAD.is_alive():
@@ -240,10 +238,16 @@ def remove_db():
     # Reset it
     WRITER_THREAD = threading.Thread(target=_writer, daemon=True)
 
+    # Reset tables tracking
+    TABLES.clear()
+
     # Remove db
-    SERVICE_DB_PATH.unlink()
+    ENGINE.dispose()
+    try:
+        SERVICE_DB_PATH.unlink()
+    except FileNotFoundError:
+        pass
 
-    # Reset mappers and metadata object
-    clear_mappers()
-    Base.metadata.clear()
+    # Reset base so new tables get fresh mappers
+    Base = declarative_base()
     LOGGER.info("DB has been shut down!")

daeploy/_service/service.py

@@ -15,7 +15,7 @@
 from fastapi.encoders import jsonable_encoder
 from fastapi.concurrency import run_in_threadpool
 from fastapi.middleware.cors import CORSMiddleware
-from pydantic import create_model, validate_arguments
+from pydantic import create_model, validate_call
 
 from daeploy._service.logger import setup_logging
 from daeploy._service.db import clean_database, initialize_db, remove_db, write_to_ts
@@ -33,7 +33,6 @@
 )
 from daeploy.communication import notify, Severity
 
-
 setup_logging()
 logger = logging.getLogger(__name__)
 
@@ -219,7 +218,7 @@ async def wrapper(_request: Request, *args, **kwargs):
                 _disable_http_logs(path)
 
             # Wrap the original func in a pydantic validation wrapper and return that
-            return validate_arguments(deco_func)
+            return validate_call(deco_func)
 
         # This ensures that we can use the decorator with or without arguments
         if not (callable(func) or func is None):
@@ -370,7 +369,7 @@ def add_parameter(
         if isinstance(value, Number):
             value = float(value)
 
-        @validate_arguments()
+        @validate_call()
         def update_parameter(value: value.__class__) -> Any:
             logger.info(f"Parameter {parameter} changed to {value}")
             self.parameters[parameter]["value"] = value

daeploy/cli/cli.py

@@ -5,7 +5,7 @@
 import os
 import json
 
-import pkg_resources
+from importlib.metadata import version as get_version, PackageNotFoundError
 import pytest
 import requests
 import typer
@@ -73,9 +73,9 @@ def version_callback(value: bool):
 
     # Get SDK Version
     try:
-        sdk_version = pkg_resources.get_distribution("daeploy").version
+        sdk_version = get_version("daeploy")
         typer.echo(f"SDK version: {sdk_version}")
-    except pkg_resources.DistributionNotFound:
+    except PackageNotFoundError:
         pass
 
     # Get Manager Version
@@ -654,13 +654,13 @@ def init(
         raise typer.Exit(1)
     # Find out which daeploy version that should be used by the service
     try:
-        dist = pkg_resources.get_distribution("daeploy")
+        daeploy_version = get_version("daeploy")
         daeploy_specifier = (
-            str(dist.as_requirement())
-            if dist.version != "0.0.0.dev0"
-            else dist.project_name
+            f"daeploy=={daeploy_version}"
+            if daeploy_version != "0.0.0.dev0"
+            else "daeploy"
         )  # Use full specificer unless in dev environment, then just go for the latest
-    except pkg_resources.DistributionNotFound:
+    except PackageNotFoundError:
         typer.echo(
             "`daeploy` package not found, assuming latest version "
             "should be used for the generated project."

daeploy/cli/user.py

@@ -5,7 +5,6 @@
 
 from daeploy.cli import cliutils
 
-
 app = typer.Typer(help="Collection of user management commands")
 typer.Option(None, "-p", "--password", expose_value=False)
 

daeploy/data_types.py

@@ -1,20 +1,24 @@
-# pylint: disable=too-many-ancestors
+# pylint: disable=too-many-ancestors, unused-argument
 from typing import Any, List, Dict
 
 import numpy as np
 import pandas as pd
+from pydantic import GetCoreSchemaHandler
+from pydantic_core import core_schema
 
 
 class ArrayInput(np.ndarray):
     """Pydantic compatible data type for numpy ndarray input."""
 
     @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        return core_schema.no_info_plain_validator_function(cls.validate)
 
     @classmethod
-    def __modify_schema__(cls, field_schema):
-        field_schema.update(type="array", items={})
+    def __get_pydantic_json_schema__(cls, schema, handler):
+        return {"type": "array", "items": {}}
 
     @classmethod
     def validate(cls, value: List) -> np.ndarray:
@@ -26,12 +30,14 @@ class ArrayOutput(np.ndarray):
     """Pydantic compatible data type for numpy ndarray output."""
 
     @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        return core_schema.no_info_plain_validator_function(cls.validate)
 
     @classmethod
-    def __modify_schema__(cls, field_schema):
-        field_schema.update(type="array", items={})
+    def __get_pydantic_json_schema__(cls, schema, handler):
+        return {"type": "array", "items": {}}
 
     @classmethod
     def validate(cls, value: np.ndarray) -> List:
@@ -43,31 +49,35 @@ class DataFrameInput(pd.DataFrame):
     """Pydantic compatible data type for pandas DataFrame input."""
 
     @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        return core_schema.no_info_plain_validator_function(cls.validate)
 
     @classmethod
-    def __modify_schema__(cls, field_schema):
-        field_schema.update(type="object")
+    def __get_pydantic_json_schema__(cls, schema, handler):
+        return {"type": "object"}
 
     @classmethod
     def validate(cls, value: Dict[str, Any]) -> pd.DataFrame:
-        # Transform input to ndarray
+        # Transform input to DataFrame
         return pd.DataFrame.from_dict(value)
 
 
 class DataFrameOutput(pd.DataFrame):
     """Pydantic compatible data type for pandas DataFrame output."""
 
     @classmethod
-    def __get_validators__(cls):
-        yield cls.validate
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        return core_schema.no_info_plain_validator_function(cls.validate)
 
     @classmethod
-    def __modify_schema__(cls, field_schema):
-        field_schema.update(type="object")
+    def __get_pydantic_json_schema__(cls, schema, handler):
+        return {"type": "object"}
 
     @classmethod
     def validate(cls, value: pd.DataFrame) -> Dict[str, Any]:
-        # Transform input to ndarray
+        # Transform DataFrame to dict
         return value.to_dict()

manager/app.py

@@ -19,7 +19,6 @@
 from manager.database import service_db
 from manager.constants import get_manager_version, cors_enabled, cors_config
 
-
 # Setup logger
 logging_api.setup_logging()
 LOGGER = logging.getLogger(__name__)

manager/constants.py

@@ -1,6 +1,7 @@
 """
 Constants and config
 """
+
 import os
 from pathlib import Path