From 8f05f79424fcabd61a3b5eee3cb483eba2684c61 Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:20:23 +0200 Subject: [PATCH 01/12] chore: Rename package and `jobby` command to `jobq` --- client/pyproject.toml | 6 +++--- client/src/cli/__init__.py | 2 +- client/src/cli/commands/list.py | 2 +- client/src/cli/commands/logs.py | 2 +- client/src/cli/commands/status.py | 2 +- client/src/cli/commands/stop.py | 2 +- client/src/cli/commands/submit.py | 2 +- client/src/cli/parser.py | 8 ++++---- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/client/pyproject.toml b/client/pyproject.toml index 2dd4a510..b9c42990 100644 --- a/client/pyproject.toml +++ b/client/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=62", "setuptools-scm[toml]", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "job-queue" +name = "jobq" description = "" version = "0.1.0" readme = "README.md" @@ -28,7 +28,7 @@ maintainers = [ [project.scripts] jobs_execute = "jobs.execute:execute" -jobby = "cli:main" +jobq = "cli:main" [project.optional-dependencies] dev = [ @@ -55,7 +55,7 @@ docs = [ ] [tool.setuptools.package-data] -jobs = ["py.typed"] +jobq = ["py.typed"] # Automatically determine version number from Git tags [tool.setuptools_scm] diff --git a/client/src/cli/__init__.py b/client/src/cli/__init__.py index 73bf3080..17849cb0 100644 --- a/client/src/cli/__init__.py +++ b/client/src/cli/__init__.py @@ -27,4 +27,4 @@ def main(**kwargs: str) -> None: # CLI name, so it can be used with Click's CliRunner for testing -name = "jobby" +name = "jobq" diff --git a/client/src/cli/commands/list.py b/client/src/cli/commands/list.py index ee413c1d..89c05292 100644 --- a/client/src/cli/commands/list.py +++ b/client/src/cli/commands/list.py @@ -69,7 +69,7 @@ def status_flags(wl: openapi_client.WorkloadMetadata) -> str: def add_parser(subparsers: Any, parent: argparse.ArgumentParser) -> None: - # jobby status, the status querying command + # jobq status, the status querying command parser: argparse.ArgumentParser = subparsers.add_parser( "list", parents=[parent], diff --git a/client/src/cli/commands/logs.py b/client/src/cli/commands/logs.py index 7a44faaa..b421232d 100644 --- a/client/src/cli/commands/logs.py +++ b/client/src/cli/commands/logs.py @@ -71,7 +71,7 @@ def handle_logs_cmd(args: argparse.Namespace) -> None: def add_parser(subparsers: Any, parent: argparse.ArgumentParser) -> None: - # jobby logs, command to fetch logs for workload + # jobq logs, command to fetch logs for workload parser = subparsers.add_parser( "logs", parents=[parent], diff --git a/client/src/cli/commands/status.py b/client/src/cli/commands/status.py index 0dd64a2c..9fe5024a 100644 --- a/client/src/cli/commands/status.py +++ b/client/src/cli/commands/status.py @@ -15,7 +15,7 @@ def status(client: openapi_client.JobManagementApi, args: argparse.Namespace) -> def add_parser(subparsers: Any, parent: ArgumentParser) -> None: - # jobby status, the status querying command + # jobq status, the status querying command parser: argparse.ArgumentParser = subparsers.add_parser( "status", parents=[parent], diff --git a/client/src/cli/commands/stop.py b/client/src/cli/commands/stop.py index f177fd1e..2d210b9c 100644 --- a/client/src/cli/commands/stop.py +++ b/client/src/cli/commands/stop.py @@ -14,7 +14,7 @@ def stop(client: openapi_client.JobManagementApi, args: argparse.Namespace) -> N def add_parser(subparsers: Any, parent: argparse.ArgumentParser) -> None: - # jobby stop, the execution termination command + # jobq stop, the execution termination command parser = subparsers.add_parser( "stop", parents=[parent], diff --git a/client/src/cli/commands/submit.py b/client/src/cli/commands/submit.py index 8483431b..7446a75e 100644 --- a/client/src/cli/commands/submit.py +++ b/client/src/cli/commands/submit.py @@ -97,7 +97,7 @@ def discover_job(args: argparse.Namespace) -> Job: def add_parser(subparsers: Any, parent: argparse.ArgumentParser) -> None: - # jobby submit, the job submission command + # jobq submit, the job submission command parser = subparsers.add_parser( "submit", parents=[parent], diff --git a/client/src/cli/parser.py b/client/src/cli/parser.py index d4cdd468..ffc07c60 100644 --- a/client/src/cli/parser.py +++ b/client/src/cli/parser.py @@ -5,7 +5,7 @@ from .commands import logs, status, stop, submit try: - __version__ = version("job-queue") + __version__ = version("jobq") except PackageNotFoundError: pass @@ -65,13 +65,13 @@ def main_parser() -> argparse.ArgumentParser: help="Enable quiet mode.", ) - # Main parser, invoked when running `jobby ` , i.e. without a subcommand. + # Main parser, invoked when running `jobq ` , i.e. without a subcommand. # Since we pass required=True to the commands subparser below, # any action that does not immediately exit (like e.g. help and version do) # will prompt an error about requiring a subcommand. parser = argparse.ArgumentParser( add_help=False, - prog="jobby", + prog="jobq", parents=[base_parser], description=description, formatter_class=CustomFormatter, @@ -86,7 +86,7 @@ def main_parser() -> argparse.ArgumentParser: parser.add_argument( "--version", action="version", - help="Show jobby version and exit.", + help="Show jobq version and exit.", version=f"%(prog)s version {__version__}", ) From 7d36a70c06bbb2f17c1eed23f6d554cf0f938fb4 Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:27:06 +0200 Subject: [PATCH 02/12] chore: Rename `jobs` package to `jobq` --- client/examples/example_hello.py | 2 +- client/examples/example_hello_prod.py | 2 +- client/examples/example_mnist_tf.py | 2 +- client/src/cli/commands/submit.py | 4 ++-- client/src/{jobs => jobq}/__init__.py | 6 +++--- client/src/{jobs => jobq}/assembler/__init__.py | 0 client/src/{jobs => jobq}/assembler/config.py | 2 +- client/src/{jobs => jobq}/assembler/renderers.py | 4 ++-- client/src/{jobs => jobq}/execute.py | 0 client/src/{jobs => jobq}/image.py | 0 client/src/{jobs => jobq}/job.py | 14 +++++++------- client/src/{jobs => jobq}/py.typed | 0 client/src/{jobs => jobq}/submission_context.py | 0 client/src/{jobs => jobq}/types.py | 0 client/src/{jobs => jobq}/utils/__init__.py | 0 client/src/{jobs => jobq}/utils/helpers.py | 0 client/src/{jobs => jobq}/utils/math.py | 0 client/src/{jobs => jobq}/utils/processes.py | 2 +- client/tests/smoke/test_build_from_yaml.py | 4 ++-- client/tests/unit/test_job.py | 2 +- client/tests/unit/test_utils.py | 4 ++-- 21 files changed, 24 insertions(+), 24 deletions(-) rename client/src/{jobs => jobq}/__init__.py (76%) rename client/src/{jobs => jobq}/assembler/__init__.py (100%) rename client/src/{jobs => jobq}/assembler/config.py (98%) rename client/src/{jobs => jobq}/assembler/renderers.py (98%) rename client/src/{jobs => jobq}/execute.py (100%) rename client/src/{jobs => jobq}/image.py (100%) rename client/src/{jobs => jobq}/job.py (97%) rename client/src/{jobs => jobq}/py.typed (100%) rename client/src/{jobs => jobq}/submission_context.py (100%) rename client/src/{jobs => jobq}/types.py (100%) rename client/src/{jobs => jobq}/utils/__init__.py (100%) rename client/src/{jobs => jobq}/utils/helpers.py (100%) rename client/src/{jobs => jobq}/utils/math.py (100%) rename client/src/{jobs => jobq}/utils/processes.py (98%) diff --git a/client/examples/example_hello.py b/client/examples/example_hello.py index aca5fe4c..63d444ad 100644 --- a/client/examples/example_hello.py +++ b/client/examples/example_hello.py @@ -1,7 +1,7 @@ import time from pathlib import Path -from jobs import ImageOptions, JobOptions, ResourceOptions, SchedulingOptions, job +from jobq import ImageOptions, JobOptions, ResourceOptions, SchedulingOptions, job @job( diff --git a/client/examples/example_hello_prod.py b/client/examples/example_hello_prod.py index f6b41863..351d77f7 100644 --- a/client/examples/example_hello_prod.py +++ b/client/examples/example_hello_prod.py @@ -1,7 +1,7 @@ import time from pathlib import Path -from jobs import ImageOptions, JobOptions, ResourceOptions, SchedulingOptions, job +from jobq import ImageOptions, JobOptions, ResourceOptions, SchedulingOptions, job @job( diff --git a/client/examples/example_mnist_tf.py b/client/examples/example_mnist_tf.py index a9e715e5..3168d02b 100644 --- a/client/examples/example_mnist_tf.py +++ b/client/examples/example_mnist_tf.py @@ -1,6 +1,6 @@ from pathlib import Path -from jobs import ImageOptions, JobOptions, ResourceOptions, job +from jobq import ImageOptions, JobOptions, ResourceOptions, job USE_GPU = False diff --git a/client/src/cli/commands/submit.py b/client/src/cli/commands/submit.py index 7446a75e..55d6c737 100644 --- a/client/src/cli/commands/submit.py +++ b/client/src/cli/commands/submit.py @@ -6,8 +6,8 @@ from typing import Any import openapi_client -from jobs import Image, Job -from jobs.submission_context import SubmissionContext +from jobq import Image, Job +from jobq.submission_context import SubmissionContext from openapi_client import ExecutionMode from .util import with_job_mgmt_api diff --git a/client/src/jobs/__init__.py b/client/src/jobq/__init__.py similarity index 76% rename from client/src/jobs/__init__.py rename to client/src/jobq/__init__.py index 7aae1d73..884442c9 100644 --- a/client/src/jobs/__init__.py +++ b/client/src/jobq/__init__.py @@ -1,6 +1,6 @@ -from jobs import assembler -from jobs.image import Image -from jobs.job import ( +from jobq import assembler +from jobq.image import Image +from jobq.job import ( ImageOptions, Job, JobOptions, diff --git a/client/src/jobs/assembler/__init__.py b/client/src/jobq/assembler/__init__.py similarity index 100% rename from client/src/jobs/assembler/__init__.py rename to client/src/jobq/assembler/__init__.py diff --git a/client/src/jobs/assembler/config.py b/client/src/jobq/assembler/config.py similarity index 98% rename from client/src/jobs/assembler/config.py rename to client/src/jobq/assembler/config.py index 88495a5d..28ce8ffd 100644 --- a/client/src/jobs/assembler/config.py +++ b/client/src/jobq/assembler/config.py @@ -5,7 +5,7 @@ import yaml -from jobs.types import AnyPath +from jobq.types import AnyPath @dataclass(frozen=True, slots=True) diff --git a/client/src/jobs/assembler/renderers.py b/client/src/jobq/assembler/renderers.py similarity index 98% rename from client/src/jobs/assembler/renderers.py rename to client/src/jobq/assembler/renderers.py index 2c87c369..67a9620e 100644 --- a/client/src/jobs/assembler/renderers.py +++ b/client/src/jobq/assembler/renderers.py @@ -8,8 +8,8 @@ from typing_extensions import override -from jobs.assembler import Config -from jobs.assembler.config import DependencySpec, MetaSpec, UserSpec +from jobq.assembler import Config +from jobq.assembler.config import DependencySpec, MetaSpec, UserSpec class Renderer(ABC): diff --git a/client/src/jobs/execute.py b/client/src/jobq/execute.py similarity index 100% rename from client/src/jobs/execute.py rename to client/src/jobq/execute.py diff --git a/client/src/jobs/image.py b/client/src/jobq/image.py similarity index 100% rename from client/src/jobs/image.py rename to client/src/jobq/image.py diff --git a/client/src/jobs/job.py b/client/src/jobq/job.py similarity index 97% rename from client/src/jobs/job.py rename to client/src/jobq/job.py index 4332d6cd..277d8881 100644 --- a/client/src/jobs/job.py +++ b/client/src/jobq/job.py @@ -18,13 +18,13 @@ from pydantic import BaseModel, ConfigDict, Field, StrictInt, StrictStr from typing_extensions import Self -from jobs.assembler import config -from jobs.assembler.renderers import RENDERERS -from jobs.image import Image -from jobs.types import AnyPath, DictSerializable, JsonSerializable, K8sResourceKind -from jobs.utils.helpers import remove_none_values -from jobs.utils.math import to_rational -from jobs.utils.processes import run_command +from jobq.assembler import config +from jobq.assembler.renderers import RENDERERS +from jobq.image import Image +from jobq.types import AnyPath, DictSerializable, JsonSerializable, K8sResourceKind +from jobq.utils.helpers import remove_none_values +from jobq.utils.math import to_rational +from jobq.utils.processes import run_command class BuildMode(enum.Enum): diff --git a/client/src/jobs/py.typed b/client/src/jobq/py.typed similarity index 100% rename from client/src/jobs/py.typed rename to client/src/jobq/py.typed diff --git a/client/src/jobs/submission_context.py b/client/src/jobq/submission_context.py similarity index 100% rename from client/src/jobs/submission_context.py rename to client/src/jobq/submission_context.py diff --git a/client/src/jobs/types.py b/client/src/jobq/types.py similarity index 100% rename from client/src/jobs/types.py rename to client/src/jobq/types.py diff --git a/client/src/jobs/utils/__init__.py b/client/src/jobq/utils/__init__.py similarity index 100% rename from client/src/jobs/utils/__init__.py rename to client/src/jobq/utils/__init__.py diff --git a/client/src/jobs/utils/helpers.py b/client/src/jobq/utils/helpers.py similarity index 100% rename from client/src/jobs/utils/helpers.py rename to client/src/jobq/utils/helpers.py diff --git a/client/src/jobs/utils/math.py b/client/src/jobq/utils/math.py similarity index 100% rename from client/src/jobs/utils/math.py rename to client/src/jobq/utils/math.py diff --git a/client/src/jobs/utils/processes.py b/client/src/jobq/utils/processes.py similarity index 98% rename from client/src/jobs/utils/processes.py rename to client/src/jobq/utils/processes.py index 0ada7526..c1e44a46 100644 --- a/client/src/jobs/utils/processes.py +++ b/client/src/jobq/utils/processes.py @@ -10,7 +10,7 @@ from io import TextIOBase from typing import TextIO -from jobs.types import AnyPath +from jobq.types import AnyPath def run_command( diff --git a/client/tests/smoke/test_build_from_yaml.py b/client/tests/smoke/test_build_from_yaml.py index df61aeae..b3b0bc93 100644 --- a/client/tests/smoke/test_build_from_yaml.py +++ b/client/tests/smoke/test_build_from_yaml.py @@ -1,7 +1,7 @@ from pathlib import Path -from jobs import assembler -from jobs.job import ImageOptions, Job, JobOptions, ResourceOptions +from jobq import assembler +from jobq.job import ImageOptions, Job, JobOptions, ResourceOptions def test_build_image_from_yaml(): diff --git a/client/tests/unit/test_job.py b/client/tests/unit/test_job.py index 486df739..c9cfee0c 100644 --- a/client/tests/unit/test_job.py +++ b/client/tests/unit/test_job.py @@ -2,7 +2,7 @@ import pytest -from jobs.job import Job, JobOptions, ResourceOptions, validate_labels +from jobq.job import Job, JobOptions, ResourceOptions, validate_labels @pytest.mark.parametrize( diff --git a/client/tests/unit/test_utils.py b/client/tests/unit/test_utils.py index c0f593f0..18bef436 100644 --- a/client/tests/unit/test_utils.py +++ b/client/tests/unit/test_utils.py @@ -1,7 +1,7 @@ import pytest -from jobs.utils.helpers import remove_none_values -from jobs.utils.math import to_rational +from jobq.utils.helpers import remove_none_values +from jobq.utils.math import to_rational @pytest.mark.parametrize( From 6976651a0a1662f90827ea76a35aebafbb76aa0a Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:27:56 +0200 Subject: [PATCH 03/12] chore: Follow client package rename in backend --- backend/src/jobs_server/models.py | 2 +- backend/src/jobs_server/routers/jobs.py | 2 +- backend/src/jobs_server/runner/base.py | 2 +- backend/src/jobs_server/runner/docker.py | 4 ++-- backend/src/jobs_server/runner/kueue.py | 4 ++-- backend/src/jobs_server/runner/ray.py | 4 ++-- backend/src/jobs_server/utils/k8s.py | 2 +- backend/src/jobs_server/utils/kueue.py | 4 ++-- backend/tests/e2e/test_jobs.py | 2 +- backend/tests/integration/test_jobs.py | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/backend/src/jobs_server/models.py b/backend/src/jobs_server/models.py index 0f77c408..be6737c7 100644 --- a/backend/src/jobs_server/models.py +++ b/backend/src/jobs_server/models.py @@ -5,7 +5,7 @@ from typing import Annotated, Any, Self, TypeAlias from annotated_types import Ge -from jobs import JobOptions +from jobq import JobOptions from pydantic import AfterValidator, BaseModel, Field, StrictStr from jobs_server.utils.kueue import JobId, KueueWorkload, WorkloadSpec, WorkloadStatus diff --git a/backend/src/jobs_server/routers/jobs.py b/backend/src/jobs_server/routers/jobs.py index 90ab5ef4..4245aa24 100644 --- a/backend/src/jobs_server/routers/jobs.py +++ b/backend/src/jobs_server/routers/jobs.py @@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Response from fastapi import status as http_status from fastapi.responses import StreamingResponse -from jobs import Image, Job +from jobq import Image, Job from jobs_server.dependencies import Kubernetes, ManagedWorkload from jobs_server.exceptions import PodNotReadyError diff --git a/backend/src/jobs_server/runner/base.py b/backend/src/jobs_server/runner/base.py index 80700369..9d738242 100644 --- a/backend/src/jobs_server/runner/base.py +++ b/backend/src/jobs_server/runner/base.py @@ -1,7 +1,7 @@ import abc from typing import ClassVar, Self -from jobs import Image, Job +from jobq import Image, Job from jobs_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier diff --git a/backend/src/jobs_server/runner/docker.py b/backend/src/jobs_server/runner/docker.py index 6a5b76e6..f19af79c 100644 --- a/backend/src/jobs_server/runner/docker.py +++ b/backend/src/jobs_server/runner/docker.py @@ -2,8 +2,8 @@ import textwrap import docker -from jobs import Image, Job -from jobs.job import DockerResourceOptions +from jobq import Image, Job +from jobq.job import DockerResourceOptions from jobs_server.models import ExecutionMode, SubmissionContext from jobs_server.runner.base import Runner, _make_executor_command diff --git a/backend/src/jobs_server/runner/kueue.py b/backend/src/jobs_server/runner/kueue.py index 9a995649..a0331ac2 100644 --- a/backend/src/jobs_server/runner/kueue.py +++ b/backend/src/jobs_server/runner/kueue.py @@ -1,8 +1,8 @@ import logging from dataclasses import asdict -from jobs import Image, Job -from jobs.types import K8sResourceKind +from jobq import Image, Job +from jobq.types import K8sResourceKind from kubernetes import client from jobs_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier diff --git a/backend/src/jobs_server/runner/ray.py b/backend/src/jobs_server/runner/ray.py index d5ce26b7..688f5495 100644 --- a/backend/src/jobs_server/runner/ray.py +++ b/backend/src/jobs_server/runner/ray.py @@ -5,8 +5,8 @@ from dataclasses import asdict import yaml -from jobs import Image, Job -from jobs.types import K8sResourceKind +from jobq import Image, Job +from jobq.types import K8sResourceKind from kubernetes import client from jobs_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier diff --git a/backend/src/jobs_server/utils/k8s.py b/backend/src/jobs_server/utils/k8s.py index 5b34b9bf..cf2a6dfc 100644 --- a/backend/src/jobs_server/utils/k8s.py +++ b/backend/src/jobs_server/utils/k8s.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Protocol, TypeVar -from jobs.job import Job +from jobq.job import Job from kubernetes import client from jobs_server.utils.helpers import traverse diff --git a/backend/src/jobs_server/utils/kueue.py b/backend/src/jobs_server/utils/kueue.py index 0e2bb512..828c00cc 100644 --- a/backend/src/jobs_server/utils/kueue.py +++ b/backend/src/jobs_server/utils/kueue.py @@ -2,8 +2,8 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, cast -from jobs.job import Job -from jobs.utils.helpers import remove_none_values +from jobq.job import Job +from jobq.utils.helpers import remove_none_values from kubernetes import client, dynamic from pydantic import UUID4, BaseModel, ConfigDict, field_validator diff --git a/backend/tests/e2e/test_jobs.py b/backend/tests/e2e/test_jobs.py index 79e44808..34de4027 100644 --- a/backend/tests/e2e/test_jobs.py +++ b/backend/tests/e2e/test_jobs.py @@ -3,7 +3,7 @@ import pytest from fastapi.encoders import jsonable_encoder from fastapi.testclient import TestClient -from jobs import JobOptions, ResourceOptions, SchedulingOptions +from jobq import JobOptions, ResourceOptions, SchedulingOptions from testcontainers.core.image import DockerImage from jobs_server.models import ( diff --git a/backend/tests/integration/test_jobs.py b/backend/tests/integration/test_jobs.py index efc3985d..54e405e6 100644 --- a/backend/tests/integration/test_jobs.py +++ b/backend/tests/integration/test_jobs.py @@ -5,7 +5,7 @@ import pytest from fastapi.encoders import jsonable_encoder from fastapi.testclient import TestClient -from jobs import JobOptions +from jobq import JobOptions from kubernetes import client as k8s_client from pytest_mock import MockFixture From 1a5cb2aeae0b4866a3f676ac26a2f05d9e39ba28 Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:29:42 +0200 Subject: [PATCH 04/12] chore: Update backend deps to refer to rebranded client lib --- backend/pyproject.toml | 2 +- backend/requirements-dev.txt | 14 +++++++------- backend/requirements.txt | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 2e509274..8282cf4c 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -19,7 +19,7 @@ maintainers = [ dependencies = [ "fastapi[standard]", "kubernetes", - "job-queue @ git+https://github.com/aai-institute/jobq.git@main#subdirectory=client", + "jobq @ git+https://github.com/aai-institute/jobq.git@7d36a70#subdirectory=client", # FIXME: revert to main once rebranding is merged ] [project.optional-dependencies] diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index dc2c4fbc..c688cb30 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -33,7 +33,7 @@ dnspython==2.6.1 # via email-validator docker==7.1.0 # via - # job-queue + # jobq # testcontainers email-validator==2.2.0 # via fastapi @@ -56,7 +56,7 @@ httptools==0.6.1 httpx==0.27.0 # via fastapi humanize==4.10.0 - # via job-queue + # via jobq identify==2.6.0 # via pre-commit idna==3.7 @@ -69,7 +69,7 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.4 # via fastapi -job-queue @ git+https://github.com/aai-institute/jobq.git@9217a54cd4cc6dcc820aa26d3d69f11871ef53ca#subdirectory=client +jobq @ git+https://github.com/aai-institute/jobq.git@7d36a70c06bbb2f17c1eed23f6d554cf0f938fb4#subdirectory=client # via jobs-server (pyproject.toml) kubernetes==30.1.0 # via jobs-server (pyproject.toml) @@ -104,7 +104,7 @@ pyasn1-modules==0.4.0 pydantic==2.8.2 # via # fastapi - # job-queue + # jobq pydantic-core==2.20.1 # via pydantic pygments==2.18.0 @@ -122,7 +122,7 @@ pytest-mock==3.14.0 # via jobs-server (pyproject.toml) python-dateutil==2.9.0.post0 # via - # job-queue + # jobq # kubernetes python-dotenv==1.0.1 # via uvicorn @@ -130,7 +130,7 @@ python-multipart==0.0.9 # via fastapi pyyaml==6.0.2 # via - # job-queue + # jobq # kubernetes # pre-commit # uvicorn @@ -143,7 +143,7 @@ requests-oauthlib==2.0.0 # via kubernetes rich==13.7.1 # via - # job-queue + # jobq # typer rsa==4.9 # via google-auth diff --git a/backend/requirements.txt b/backend/requirements.txt index 213270da..18987a56 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,7 +24,7 @@ click==8.1.7 dnspython==2.6.1 # via email-validator docker==7.1.0 - # via job-queue + # via jobq email-validator==2.2.0 # via fastapi fastapi==0.112.0 @@ -44,7 +44,7 @@ httptools==0.6.1 httpx==0.27.0 # via fastapi humanize==4.10.0 - # via job-queue + # via jobq idna==3.7 # via # anyio @@ -53,7 +53,7 @@ idna==3.7 # requests jinja2==3.1.4 # via fastapi -job-queue @ git+https://github.com/aai-institute/jobq.git@9217a54cd4cc6dcc820aa26d3d69f11871ef53ca#subdirectory=client +jobq @ git+https://github.com/aai-institute/jobq.git@7d36a70c06bbb2f17c1eed23f6d554cf0f938fb4#subdirectory=client # via jobs-server (pyproject.toml) kubernetes==30.1.0 # via jobs-server (pyproject.toml) @@ -76,14 +76,14 @@ pyasn1-modules==0.4.0 pydantic==2.8.2 # via # fastapi - # job-queue + # jobq pydantic-core==2.20.1 # via pydantic pygments==2.18.0 # via rich python-dateutil==2.9.0.post0 # via - # job-queue + # jobq # kubernetes python-dotenv==1.0.1 # via uvicorn @@ -91,7 +91,7 @@ python-multipart==0.0.9 # via fastapi pyyaml==6.0.2 # via - # job-queue + # jobq # kubernetes # uvicorn requests==2.32.3 @@ -103,7 +103,7 @@ requests-oauthlib==2.0.0 # via kubernetes rich==13.7.1 # via - # job-queue + # jobq # typer rsa==4.9 # via google-auth From 60bc4bb9a45298f6fca4ff043c44321f246a58ce Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:32:08 +0200 Subject: [PATCH 05/12] chore: Rename backend package to `jobq_server` --- .../src/{jobs_server => jobq_server}/__init__.py | 0 .../src/{jobs_server => jobq_server}/__main__.py | 2 +- .../dependencies/__init__.py | 6 +++--- .../src/{jobs_server => jobq_server}/exceptions.py | 0 backend/src/{jobs_server => jobq_server}/models.py | 2 +- .../routers/__init__.py | 0 .../{jobs_server => jobq_server}/routers/jobs.py | 12 ++++++------ backend/src/jobq_server/runner/__init__.py | 12 ++++++++++++ .../{jobs_server => jobq_server}/runner/base.py | 2 +- .../{jobs_server => jobq_server}/runner/docker.py | 6 +++--- .../{jobs_server => jobq_server}/runner/kueue.py | 10 +++++----- .../src/{jobs_server => jobq_server}/runner/ray.py | 10 +++++----- .../services/__init__.py | 0 .../{jobs_server => jobq_server}/services/k8s.py | 10 +++++----- .../{jobs_server => jobq_server}/utils/__init__.py | 0 .../{jobs_server => jobq_server}/utils/fastapi.py | 0 .../{jobs_server => jobq_server}/utils/helpers.py | 0 .../src/{jobs_server => jobq_server}/utils/k8s.py | 4 ++-- .../{jobs_server => jobq_server}/utils/kueue.py | 12 ++++++------ backend/src/jobs_server/runner/__init__.py | 12 ------------ backend/tests/e2e/conftest.py | 2 +- backend/tests/e2e/test_jobs.py | 2 +- backend/tests/integration/conftest.py | 10 +++++----- backend/tests/integration/test_jobs.py | 14 +++++++------- backend/tests/unit/test_helpers.py | 2 +- backend/tests/unit/test_k8s.py | 2 +- backend/tests/unit/test_models.py | 2 +- 27 files changed, 67 insertions(+), 67 deletions(-) rename backend/src/{jobs_server => jobq_server}/__init__.py (100%) rename backend/src/{jobs_server => jobq_server}/__main__.py (96%) rename backend/src/{jobs_server => jobq_server}/dependencies/__init__.py (80%) rename backend/src/{jobs_server => jobq_server}/exceptions.py (100%) rename backend/src/{jobs_server => jobq_server}/models.py (98%) rename backend/src/{jobs_server => jobq_server}/routers/__init__.py (100%) rename backend/src/{jobs_server => jobq_server}/routers/jobs.py (95%) create mode 100644 backend/src/jobq_server/runner/__init__.py rename backend/src/{jobs_server => jobq_server}/runner/base.py (93%) rename backend/src/{jobs_server => jobq_server}/runner/docker.py (86%) rename backend/src/{jobs_server => jobq_server}/runner/kueue.py (91%) rename backend/src/{jobs_server => jobq_server}/runner/ray.py (94%) rename backend/src/{jobs_server => jobq_server}/services/__init__.py (100%) rename backend/src/{jobs_server => jobq_server}/services/k8s.py (94%) rename backend/src/{jobs_server => jobq_server}/utils/__init__.py (100%) rename backend/src/{jobs_server => jobq_server}/utils/fastapi.py (100%) rename backend/src/{jobs_server => jobq_server}/utils/helpers.py (100%) rename backend/src/{jobs_server => jobq_server}/utils/k8s.py (98%) rename backend/src/{jobs_server => jobq_server}/utils/kueue.py (96%) delete mode 100644 backend/src/jobs_server/runner/__init__.py diff --git a/backend/src/jobs_server/__init__.py b/backend/src/jobq_server/__init__.py similarity index 100% rename from backend/src/jobs_server/__init__.py rename to backend/src/jobq_server/__init__.py diff --git a/backend/src/jobs_server/__main__.py b/backend/src/jobq_server/__main__.py similarity index 96% rename from backend/src/jobs_server/__main__.py rename to backend/src/jobq_server/__main__.py index eb6e5833..f8122068 100644 --- a/backend/src/jobs_server/__main__.py +++ b/backend/src/jobq_server/__main__.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from kubernetes import config -from jobs_server.routers import jobs +from jobq_server.routers import jobs @asynccontextmanager diff --git a/backend/src/jobs_server/dependencies/__init__.py b/backend/src/jobq_server/dependencies/__init__.py similarity index 80% rename from backend/src/jobs_server/dependencies/__init__.py rename to backend/src/jobq_server/dependencies/__init__.py index 02a6a827..c7818009 100644 --- a/backend/src/jobs_server/dependencies/__init__.py +++ b/backend/src/jobq_server/dependencies/__init__.py @@ -2,9 +2,9 @@ from fastapi import Depends, HTTPException -from jobs_server.models import JobId -from jobs_server.services.k8s import KubernetesService -from jobs_server.utils.kueue import KueueWorkload +from jobq_server.models import JobId +from jobq_server.services.k8s import KubernetesService +from jobq_server.utils.kueue import KueueWorkload def k8s_service() -> KubernetesService: diff --git a/backend/src/jobs_server/exceptions.py b/backend/src/jobq_server/exceptions.py similarity index 100% rename from backend/src/jobs_server/exceptions.py rename to backend/src/jobq_server/exceptions.py diff --git a/backend/src/jobs_server/models.py b/backend/src/jobq_server/models.py similarity index 98% rename from backend/src/jobs_server/models.py rename to backend/src/jobq_server/models.py index be6737c7..ea3b647b 100644 --- a/backend/src/jobs_server/models.py +++ b/backend/src/jobq_server/models.py @@ -8,7 +8,7 @@ from jobq import JobOptions from pydantic import AfterValidator, BaseModel, Field, StrictStr -from jobs_server.utils.kueue import JobId, KueueWorkload, WorkloadSpec, WorkloadStatus +from jobq_server.utils.kueue import JobId, KueueWorkload, WorkloadSpec, WorkloadStatus def validate_image_ref(ref: str) -> str: diff --git a/backend/src/jobs_server/routers/__init__.py b/backend/src/jobq_server/routers/__init__.py similarity index 100% rename from backend/src/jobs_server/routers/__init__.py rename to backend/src/jobq_server/routers/__init__.py diff --git a/backend/src/jobs_server/routers/jobs.py b/backend/src/jobq_server/routers/jobs.py similarity index 95% rename from backend/src/jobs_server/routers/jobs.py rename to backend/src/jobq_server/routers/jobs.py index 4245aa24..07e3527b 100644 --- a/backend/src/jobs_server/routers/jobs.py +++ b/backend/src/jobq_server/routers/jobs.py @@ -8,9 +8,9 @@ from fastapi.responses import StreamingResponse from jobq import Image, Job -from jobs_server.dependencies import Kubernetes, ManagedWorkload -from jobs_server.exceptions import PodNotReadyError -from jobs_server.models import ( +from jobq_server.dependencies import Kubernetes, ManagedWorkload +from jobq_server.exceptions import PodNotReadyError +from jobq_server.models import ( CreateJobModel, ExecutionMode, ListWorkloadModel, @@ -18,9 +18,9 @@ WorkloadIdentifier, WorkloadMetadata, ) -from jobs_server.runner import Runner -from jobs_server.utils.fastapi import make_dependable -from jobs_server.utils.kueue import JobId +from jobq_server.runner import Runner +from jobq_server.utils.fastapi import make_dependable +from jobq_server.utils.kueue import JobId router = APIRouter(tags=["Job management"]) diff --git a/backend/src/jobq_server/runner/__init__.py b/backend/src/jobq_server/runner/__init__.py new file mode 100644 index 00000000..e137063c --- /dev/null +++ b/backend/src/jobq_server/runner/__init__.py @@ -0,0 +1,12 @@ +from jobq_server.runner.base import ExecutionMode, Runner +from jobq_server.runner.docker import DockerRunner +from jobq_server.runner.kueue import KueueRunner +from jobq_server.runner.ray import RayJobRunner + +__all__ = [ + "DockerRunner", + "ExecutionMode", + "KueueRunner", + "RayJobRunner", + "Runner", +] diff --git a/backend/src/jobs_server/runner/base.py b/backend/src/jobq_server/runner/base.py similarity index 93% rename from backend/src/jobs_server/runner/base.py rename to backend/src/jobq_server/runner/base.py index 9d738242..1b5e6ec7 100644 --- a/backend/src/jobs_server/runner/base.py +++ b/backend/src/jobq_server/runner/base.py @@ -3,7 +3,7 @@ from jobq import Image, Job -from jobs_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier +from jobq_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier class Runner(abc.ABC): diff --git a/backend/src/jobs_server/runner/docker.py b/backend/src/jobq_server/runner/docker.py similarity index 86% rename from backend/src/jobs_server/runner/docker.py rename to backend/src/jobq_server/runner/docker.py index f19af79c..5c16d55a 100644 --- a/backend/src/jobs_server/runner/docker.py +++ b/backend/src/jobq_server/runner/docker.py @@ -5,9 +5,9 @@ from jobq import Image, Job from jobq.job import DockerResourceOptions -from jobs_server.models import ExecutionMode, SubmissionContext -from jobs_server.runner.base import Runner, _make_executor_command -from jobs_server.utils.helpers import remove_none_values +from jobq_server.models import ExecutionMode, SubmissionContext +from jobq_server.runner.base import Runner, _make_executor_command +from jobq_server.utils.helpers import remove_none_values class DockerRunner(Runner): diff --git a/backend/src/jobs_server/runner/kueue.py b/backend/src/jobq_server/runner/kueue.py similarity index 91% rename from backend/src/jobs_server/runner/kueue.py rename to backend/src/jobq_server/runner/kueue.py index a0331ac2..a542bdab 100644 --- a/backend/src/jobs_server/runner/kueue.py +++ b/backend/src/jobq_server/runner/kueue.py @@ -5,15 +5,15 @@ from jobq.types import K8sResourceKind from kubernetes import client -from jobs_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier -from jobs_server.runner.base import Runner, _make_executor_command -from jobs_server.services.k8s import KubernetesService -from jobs_server.utils.k8s import ( +from jobq_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier +from jobq_server.runner.base import Runner, _make_executor_command +from jobq_server.services.k8s import KubernetesService +from jobq_server.utils.k8s import ( gvk, k8s_annotations, sanitize_rfc1123_domain_name, ) -from jobs_server.utils.kueue import kueue_scheduling_labels +from jobq_server.utils.kueue import kueue_scheduling_labels class KueueRunner(Runner): diff --git a/backend/src/jobs_server/runner/ray.py b/backend/src/jobq_server/runner/ray.py similarity index 94% rename from backend/src/jobs_server/runner/ray.py rename to backend/src/jobq_server/runner/ray.py index 688f5495..540c158a 100644 --- a/backend/src/jobs_server/runner/ray.py +++ b/backend/src/jobq_server/runner/ray.py @@ -9,15 +9,15 @@ from jobq.types import K8sResourceKind from kubernetes import client -from jobs_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier -from jobs_server.runner.base import Runner, _make_executor_command -from jobs_server.services.k8s import KubernetesService -from jobs_server.utils.k8s import ( +from jobq_server.models import ExecutionMode, SubmissionContext, WorkloadIdentifier +from jobq_server.runner.base import Runner, _make_executor_command +from jobq_server.services.k8s import KubernetesService +from jobq_server.utils.k8s import ( gvk, k8s_annotations, sanitize_rfc1123_domain_name, ) -from jobs_server.utils.kueue import kueue_scheduling_labels +from jobq_server.utils.kueue import kueue_scheduling_labels class RayJobRunner(Runner): diff --git a/backend/src/jobs_server/services/__init__.py b/backend/src/jobq_server/services/__init__.py similarity index 100% rename from backend/src/jobs_server/services/__init__.py rename to backend/src/jobq_server/services/__init__.py diff --git a/backend/src/jobs_server/services/k8s.py b/backend/src/jobq_server/services/k8s.py similarity index 94% rename from backend/src/jobs_server/services/k8s.py rename to backend/src/jobq_server/services/k8s.py index bb5fca54..24fb2e26 100644 --- a/backend/src/jobs_server/services/k8s.py +++ b/backend/src/jobq_server/services/k8s.py @@ -5,11 +5,11 @@ from kubernetes import client, config, dynamic -from jobs_server.exceptions import PodNotReadyError, WorkloadNotFound -from jobs_server.models import JobId -from jobs_server.utils.helpers import traverse -from jobs_server.utils.k8s import GroupVersionKind -from jobs_server.utils.kueue import KueueWorkload +from jobq_server.exceptions import PodNotReadyError, WorkloadNotFound +from jobq_server.models import JobId +from jobq_server.utils.helpers import traverse +from jobq_server.utils.k8s import GroupVersionKind +from jobq_server.utils.kueue import KueueWorkload class KubernetesService: diff --git a/backend/src/jobs_server/utils/__init__.py b/backend/src/jobq_server/utils/__init__.py similarity index 100% rename from backend/src/jobs_server/utils/__init__.py rename to backend/src/jobq_server/utils/__init__.py diff --git a/backend/src/jobs_server/utils/fastapi.py b/backend/src/jobq_server/utils/fastapi.py similarity index 100% rename from backend/src/jobs_server/utils/fastapi.py rename to backend/src/jobq_server/utils/fastapi.py diff --git a/backend/src/jobs_server/utils/helpers.py b/backend/src/jobq_server/utils/helpers.py similarity index 100% rename from backend/src/jobs_server/utils/helpers.py rename to backend/src/jobq_server/utils/helpers.py diff --git a/backend/src/jobs_server/utils/k8s.py b/backend/src/jobq_server/utils/k8s.py similarity index 98% rename from backend/src/jobs_server/utils/k8s.py rename to backend/src/jobq_server/utils/k8s.py index cf2a6dfc..ab2bb93b 100644 --- a/backend/src/jobs_server/utils/k8s.py +++ b/backend/src/jobq_server/utils/k8s.py @@ -7,10 +7,10 @@ from jobq.job import Job from kubernetes import client -from jobs_server.utils.helpers import traverse +from jobq_server.utils.helpers import traverse if TYPE_CHECKING: - from jobs_server.models import SubmissionContext + from jobq_server.models import SubmissionContext def sanitize_rfc1123_domain_name(s: str) -> str: diff --git a/backend/src/jobs_server/utils/kueue.py b/backend/src/jobq_server/utils/kueue.py similarity index 96% rename from backend/src/jobs_server/utils/kueue.py rename to backend/src/jobq_server/utils/kueue.py index 828c00cc..60d35b78 100644 --- a/backend/src/jobs_server/utils/kueue.py +++ b/backend/src/jobq_server/utils/kueue.py @@ -7,13 +7,13 @@ from kubernetes import client, dynamic from pydantic import UUID4, BaseModel, ConfigDict, field_validator -from jobs_server.exceptions import WorkloadNotFound -from jobs_server.utils.helpers import traverse -from jobs_server.utils.k8s import build_metadata, filter_conditions, gvk +from jobq_server.exceptions import WorkloadNotFound +from jobq_server.utils.helpers import traverse +from jobq_server.utils.k8s import build_metadata, filter_conditions, gvk if TYPE_CHECKING: - from jobs_server.models import JobStatus - from jobs_server.services.k8s import KubernetesService + from jobq_server.models import JobStatus + from jobq_server.services.k8s import KubernetesService JobId = UUID4 @@ -148,7 +148,7 @@ def for_managed_resource(cls, uid: str, namespace: str): @property def execution_status(self) -> "JobStatus": - from jobs_server.models import JobStatus + from jobq_server.models import JobStatus if filter_conditions(self, reason="Succeeded"): return JobStatus.SUCCEEDED diff --git a/backend/src/jobs_server/runner/__init__.py b/backend/src/jobs_server/runner/__init__.py deleted file mode 100644 index 3709e3d4..00000000 --- a/backend/src/jobs_server/runner/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from jobs_server.runner.base import ExecutionMode, Runner -from jobs_server.runner.docker import DockerRunner -from jobs_server.runner.kueue import KueueRunner -from jobs_server.runner.ray import RayJobRunner - -__all__ = [ - "DockerRunner", - "ExecutionMode", - "KueueRunner", - "RayJobRunner", - "Runner", -] diff --git a/backend/tests/e2e/conftest.py b/backend/tests/e2e/conftest.py index 2f55bae2..b63e7e72 100644 --- a/backend/tests/e2e/conftest.py +++ b/backend/tests/e2e/conftest.py @@ -9,7 +9,7 @@ from kubernetes import config from testcontainers.core.image import DockerImage -from jobs_server import app +from jobq_server import app from .clusters import KindCluster, KubernetesCluster, MinikubeCluster diff --git a/backend/tests/e2e/test_jobs.py b/backend/tests/e2e/test_jobs.py index 34de4027..240ffd76 100644 --- a/backend/tests/e2e/test_jobs.py +++ b/backend/tests/e2e/test_jobs.py @@ -6,7 +6,7 @@ from jobq import JobOptions, ResourceOptions, SchedulingOptions from testcontainers.core.image import DockerImage -from jobs_server.models import ( +from jobq_server.models import ( CreateJobModel, ExecutionMode, JobStatus, diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py index b221d986..d9903eba 100644 --- a/backend/tests/integration/conftest.py +++ b/backend/tests/integration/conftest.py @@ -3,16 +3,16 @@ from fastapi.testclient import TestClient from pytest_mock import MockFixture -import jobs_server.services.k8s -from jobs_server import app +import jobq_server.services.k8s +from jobq_server import app @pytest.fixture def client(mocker: MockFixture) -> TestClient: # Mock entire Kubernetes and Docker client functionality - mocker.patch.object(jobs_server.services.k8s.config, "load_config") - mocker.patch.object(jobs_server.services.k8s.config, "load_incluster_config") - mocker.patch.object(jobs_server.services.k8s.config, "load_kube_config") + mocker.patch.object(jobq_server.services.k8s.config, "load_config") + mocker.patch.object(jobq_server.services.k8s.config, "load_incluster_config") + mocker.patch.object(jobq_server.services.k8s.config, "load_kube_config") mocker.patch.object(docker, "from_env") return TestClient(app) diff --git a/backend/tests/integration/test_jobs.py b/backend/tests/integration/test_jobs.py index 54e405e6..de32dbdf 100644 --- a/backend/tests/integration/test_jobs.py +++ b/backend/tests/integration/test_jobs.py @@ -9,18 +9,18 @@ from kubernetes import client as k8s_client from pytest_mock import MockFixture -from jobs_server.exceptions import PodNotReadyError -from jobs_server.models import ( +from jobq_server.exceptions import PodNotReadyError +from jobq_server.models import ( CreateJobModel, JobStatus, WorkloadIdentifier, WorkloadMetadata, ) -from jobs_server.runner import KueueRunner, RayJobRunner -from jobs_server.runner.base import ExecutionMode, Runner -from jobs_server.runner.docker import DockerRunner -from jobs_server.services.k8s import KubernetesService -from jobs_server.utils.kueue import ( +from jobq_server.runner import KueueRunner, RayJobRunner +from jobq_server.runner.base import ExecutionMode, Runner +from jobq_server.runner.docker import DockerRunner +from jobq_server.services.k8s import KubernetesService +from jobq_server.utils.kueue import ( KueueWorkload, WorkloadAdmission, WorkloadSpec, diff --git a/backend/tests/unit/test_helpers.py b/backend/tests/unit/test_helpers.py index bdb574e8..d050a394 100644 --- a/backend/tests/unit/test_helpers.py +++ b/backend/tests/unit/test_helpers.py @@ -2,7 +2,7 @@ import pytest -from jobs_server.utils.helpers import traverse +from jobq_server.utils.helpers import traverse @pytest.mark.parametrize( diff --git a/backend/tests/unit/test_k8s.py b/backend/tests/unit/test_k8s.py index 9162362c..63dc84e4 100644 --- a/backend/tests/unit/test_k8s.py +++ b/backend/tests/unit/test_k8s.py @@ -1,7 +1,7 @@ import pytest from kubernetes import client -from jobs_server.utils.k8s import build_metadata +from jobq_server.utils.k8s import build_metadata @pytest.mark.parametrize( diff --git a/backend/tests/unit/test_models.py b/backend/tests/unit/test_models.py index a8b23e2f..a8533fbe 100644 --- a/backend/tests/unit/test_models.py +++ b/backend/tests/unit/test_models.py @@ -2,7 +2,7 @@ import pytest -from jobs_server.models import validate_image_ref +from jobq_server.models import validate_image_ref @pytest.mark.parametrize( From 5f1a3546b478f8afa67408593caaec3616a8d840 Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:35:58 +0200 Subject: [PATCH 06/12] chore: Update project name in CI workflows --- .github/workflows/backend-python.yaml | 112 +++++++++++------------ .github/workflows/client-python.yaml | 112 +++++++++++------------ .github/workflows/docs.yaml | 78 ++++++++-------- .github/workflows/release.yaml | 126 +++++++++++++------------- 4 files changed, 214 insertions(+), 214 deletions(-) diff --git a/.github/workflows/backend-python.yaml b/.github/workflows/backend-python.yaml index f0042ac8..82c07b22 100644 --- a/.github/workflows/backend-python.yaml +++ b/.github/workflows/backend-python.yaml @@ -1,63 +1,63 @@ name: "Backend: Python" on: - push: - branches: - - main - pull_request: - paths: - - backend/** - - .github/** - branches: - - main + push: + branches: + - main + pull_request: + paths: + - backend/** + - .github/** + branches: + - main defaults: - run: - working-directory: backend + run: + working-directory: backend jobs: - lint: - runs-on: ubuntu-latest - env: - MYPY_CACHE_DIR: "${{ github.workspace }}/.cache/mypy" - RUFF_CACHE_DIR: "${{ github.workspace }}/.cache/ruff" - PRE_COMMIT_HOME: "${{ github.workspace }}/.cache/pre-commit" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/python-lint - with: - pythonVersion: "3.11" - workingDirectory: backend - test: - name: Test jobs-infra on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - steps: - - uses: actions/checkout@v4 - - name: Set up oldest supported Python on ${{ matrix.os }} - uses: ./.github/actions/python-deps - with: - pythonVersion: "3.11" - workingDirectory: "backend" - - name: Execute python tests - run: | - uv pip install . # need to install without --no-deps to work around non-portable dependency resolution in `uv pip compile` - uv run pytest -s --cov=src --cov-report=xml --junitxml=junit.xml - - name: Upload results to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./backend/coverage.xml - slug: aai-institute/infrastructure-product - flags: backend - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./backend/junit.xml - slug: aai-institute/infrastructure-product - flags: backend + lint: + runs-on: ubuntu-latest + env: + MYPY_CACHE_DIR: "${{ github.workspace }}/.cache/mypy" + RUFF_CACHE_DIR: "${{ github.workspace }}/.cache/ruff" + PRE_COMMIT_HOME: "${{ github.workspace }}/.cache/pre-commit" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/python-lint + with: + pythonVersion: "3.11" + workingDirectory: backend + test: + name: Test jobs-infra on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up oldest supported Python on ${{ matrix.os }} + uses: ./.github/actions/python-deps + with: + pythonVersion: "3.11" + workingDirectory: "backend" + - name: Execute python tests + run: | + uv pip install . # need to install without --no-deps to work around non-portable dependency resolution in `uv pip compile` + uv run pytest -s --cov=src --cov-report=xml --junitxml=junit.xml + - name: Upload results to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./backend/coverage.xml + slug: aai-institute/jobq + flags: backend + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./backend/junit.xml + slug: aai-institute/jobq + flags: backend diff --git a/.github/workflows/client-python.yaml b/.github/workflows/client-python.yaml index 24b18f9c..815e3998 100644 --- a/.github/workflows/client-python.yaml +++ b/.github/workflows/client-python.yaml @@ -1,63 +1,63 @@ name: "Client library: Python" on: - push: - branches: - - main - pull_request: - paths: - - client/** - - .github/** - branches: - - main + push: + branches: + - main + pull_request: + paths: + - client/** + - .github/** + branches: + - main defaults: - run: - working-directory: client + run: + working-directory: client jobs: - lint: - runs-on: ubuntu-latest - env: - MYPY_CACHE_DIR: "${{ github.workspace }}/.cache/mypy" - RUFF_CACHE_DIR: "${{ github.workspace }}/.cache/ruff" - PRE_COMMIT_HOME: "${{ github.workspace }}/.cache/pre-commit" - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/python-lint - with: - pythonVersion: "3.11" - workingDirectory: client - test: - name: Test jobs-infra on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - steps: - - uses: actions/checkout@v4 - - name: Set up oldest supported Python on ${{ matrix.os }} - uses: ./.github/actions/python-deps - with: - pythonVersion: "3.10" - workingDirectory: "client" - - name: Execute python tests - run: | - uv pip install . # need to install without --no-deps to work around non-portable dependency resolution in `uv pip compile` - uv run pytest --cov=src --cov-report=xml --junitxml=junit.xml - - name: Upload results to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./client/coverage.xml - slug: aai-institute/infrastructure-product - flags: client - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./client/junit.xml - slug: aai-institute/infrastructure-product - flags: client + lint: + runs-on: ubuntu-latest + env: + MYPY_CACHE_DIR: "${{ github.workspace }}/.cache/mypy" + RUFF_CACHE_DIR: "${{ github.workspace }}/.cache/ruff" + PRE_COMMIT_HOME: "${{ github.workspace }}/.cache/pre-commit" + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/python-lint + with: + pythonVersion: "3.11" + workingDirectory: client + test: + name: Test jobs-infra on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - name: Set up oldest supported Python on ${{ matrix.os }} + uses: ./.github/actions/python-deps + with: + pythonVersion: "3.10" + workingDirectory: "client" + - name: Execute python tests + run: | + uv pip install . # need to install without --no-deps to work around non-portable dependency resolution in `uv pip compile` + uv run pytest --cov=src --cov-report=xml --junitxml=junit.xml + - name: Upload results to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./client/coverage.xml + slug: aai-institute/jobq + flags: client + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./client/junit.xml + slug: aai-institute/jobq + flags: client diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e834c444..7cfb5cff 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,48 +1,48 @@ name: "Client library: Documentation" on: - push: - branches: - - main - pull_request: - paths: - - client/** - branches: - - main + push: + branches: + - main + pull_request: + paths: + - client/** + branches: + - main defaults: - run: - working-directory: client + run: + working-directory: client permissions: - contents: write + contents: write jobs: - deploy-docs: - runs-on: ubuntu-latest - name: Deploy documentation to GitHub Pages - # publish on 'main' only to prevent version clutter - if: ${{ github.ref == 'refs/heads/main' }} - concurrency: docs - steps: - - uses: actions/checkout@v4 - - name: Set up Python and dependencies - uses: ./.github/actions/python-deps - with: - pythonVersion: "3.11" - workingDirectory: "client" - - name: Build pre-release documentation - run: uv run mkdocs build - - name: Archive built documentation - uses: actions/upload-artifact@v4 - with: - name: docs - path: client/public/docs - - name: Deploy pre-release documentation - if: ${{ github.ref_name == 'main'}} - uses: ./.github/actions/mike-docs - with: - version: development - pre_release: true # include pre-release notification banner - push: true - workingDirectory: "client" + deploy-docs: + runs-on: ubuntu-latest + name: Deploy documentation to GitHub Pages + # publish on 'main' only to prevent version clutter + if: ${{ github.ref == 'refs/heads/main' }} + concurrency: docs + steps: + - uses: actions/checkout@v4 + - name: Set up Python and dependencies + uses: ./.github/actions/python-deps + with: + pythonVersion: "3.11" + workingDirectory: "client" + - name: Build pre-release documentation + run: uv run mkdocs build + - name: Archive built documentation + uses: actions/upload-artifact@v4 + with: + name: docs + path: client/public/docs + - name: Deploy pre-release documentation + if: ${{ github.ref_name == 'main'}} + uses: ./.github/actions/mike-docs + with: + version: development + pre_release: true # include pre-release notification banner + push: true + workingDirectory: "client" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b5dd8932..1e50c987 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,72 +1,72 @@ name: "Client library: Release" on: - workflow_dispatch: - release: - types: - - published + workflow_dispatch: + release: + types: + - published permissions: - contents: write + contents: write defaults: - run: - working-directory: client + run: + working-directory: client jobs: - build-release: - runs-on: ubuntu-latest - name: Deploy documentation to GitHub Pages - concurrency: docs - steps: - - uses: actions/checkout@v4 - - name: Set up Python and dependencies - uses: ./.github/actions/python-deps - with: - pythonVersion: "3.11" - workingDirectory: "client" - # Package build - - name: Build and check - id: build - run: | - uv run python -m build - echo "package_wheel=$(basename dist/*.whl)" >> $GITHUB_OUTPUT - echo "package_sdist=$(basename dist/*.gz)" >> $GITHUB_OUTPUT - - name: Upload build artifacts - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist - - name: Add wheel as release asset - uses: actions/upload-release-asset@latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: dist/${{ steps.build.outputs.package_wheel }} - asset_name: ${{ steps.build.outputs.package_wheel }} - asset_content_type: application/zip - - name: Add sdist as release asset - uses: actions/upload-release-asset@latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ github.event.release.upload_url }} - asset_path: dist/${{ steps.build.outputs.package_sdist }} - asset_name: ${{ steps.build.outputs.package_sdist }} - asset_content_type: application/zip - # Docs - - name: Build release documentation - run: uv run mkdocs build - - name: Archive built documentation - uses: actions/upload-artifact@v4 - with: - name: docs - path: client/public/docs - - name: Deploy release documentation - uses: ./.github/actions/mike-docs - with: - version: ${{ github.event.release.tag_name }} - alias: latest - push: true - workingDirectory: client + build-release: + runs-on: ubuntu-latest + name: Deploy documentation to GitHub Pages + concurrency: docs + steps: + - uses: actions/checkout@v4 + - name: Set up Python and dependencies + uses: ./.github/actions/python-deps + with: + pythonVersion: "3.11" + workingDirectory: "client" + # Package build + - name: Build and check + id: build + run: | + uv run python -m build + echo "package_wheel=$(basename dist/*.whl)" >> $GITHUB_OUTPUT + echo "package_sdist=$(basename dist/*.gz)" >> $GITHUB_OUTPUT + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist + - name: Add wheel as release asset + uses: actions/upload-release-asset@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: dist/${{ steps.build.outputs.package_wheel }} + asset_name: ${{ steps.build.outputs.package_wheel }} + asset_content_type: application/zip + - name: Add sdist as release asset + uses: actions/upload-release-asset@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: dist/${{ steps.build.outputs.package_sdist }} + asset_name: ${{ steps.build.outputs.package_sdist }} + asset_content_type: application/zip + # Docs + - name: Build release documentation + run: uv run mkdocs build + - name: Archive built documentation + uses: actions/upload-artifact@v4 + with: + name: docs + path: client/public/docs + - name: Deploy release documentation + uses: ./.github/actions/mike-docs + with: + version: ${{ github.event.release.tag_name }} + alias: latest + push: true + workingDirectory: client From 64bff364cb2931195ed22602527b7cb8e675ceff Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:36:14 +0200 Subject: [PATCH 07/12] chore: Update published server container image name --- .github/workflows/backend-docker.yaml | 120 +++++++++++++------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/backend-docker.yaml b/.github/workflows/backend-docker.yaml index fd32256e..4ddc5c66 100644 --- a/.github/workflows/backend-docker.yaml +++ b/.github/workflows/backend-docker.yaml @@ -1,72 +1,72 @@ name: "Backend: Docker Image" on: - push: - branches: - - main - pull_request: - paths: - - backend/** - - .github/** - branches: - - main + push: + branches: + - main + pull_request: + paths: + - backend/** + - .github/** + branches: + - main jobs: - build: - runs-on: ubuntu-latest + build: + runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - packages: write - attestations: write + permissions: + id-token: write + contents: read + packages: write + attestations: write - env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository_owner }}/jobs-server + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/jobq-server - steps: - - name: Checkout repository - uses: actions/checkout@v4 + steps: + - name: Checkout repository + uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - # branch event - type=ref,event=branch - # tag event - type=ref,event=tag - # pull request event - type=ref,event=pr + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # branch event + type=ref,event=branch + # tag event + type=ref,event=tag + # pull request event + type=ref,event=pr - - name: Build and push Docker image - uses: docker/build-push-action@v6 - id: push - with: - context: backend - file: backend/Dockerfile - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true - github-token: ${{ github.token }} + - name: Build and push Docker image + uses: docker/build-push-action@v6 + id: push + with: + context: backend + file: backend/Dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + github-token: ${{ github.token }} From 097c23fce6cb89ca4ecc10c10a5b8c2d11c18745 Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:41:51 +0200 Subject: [PATCH 08/12] chore: Rename backend package --- backend/Dockerfile | 2 +- backend/pyproject.toml | 2 +- backend/requirements-dev.txt | 20 ++++++++++---------- backend/requirements.txt | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index fa1e3000..45495612 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -24,4 +24,4 @@ FROM python:${PYTHON_VERSION} WORKDIR /code COPY --from=build /code /code -CMD ["/code/.venv/bin/fastapi", "run", "src/jobs_server/__main__.py", "--host", "0.0.0.0", "--port", "8000"] +CMD ["/code/.venv/bin/fastapi", "run", "src/jobq_server/__main__.py", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8282cf4c..aae648b6 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=62", "setuptools-scm[toml]", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "jobs-server" +name = "jobq-server" description = "" version = "0.1.0" readme = "README.md" diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index c688cb30..20ec580d 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -8,7 +8,7 @@ anyio==4.4.0 # starlette # watchfiles build==1.2.1 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) cachetools==5.4.0 # via google-auth certifi==2024.7.4 @@ -38,7 +38,7 @@ docker==7.1.0 email-validator==2.2.0 # via fastapi fastapi==0.112.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) fastapi-cli==0.0.5 # via fastapi filelock==3.15.4 @@ -70,9 +70,9 @@ iniconfig==2.0.0 jinja2==3.1.4 # via fastapi jobq @ git+https://github.com/aai-institute/jobq.git@7d36a70c06bbb2f17c1eed23f6d554cf0f938fb4#subdirectory=client - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) kubernetes==30.1.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 @@ -94,7 +94,7 @@ platformdirs==4.2.2 pluggy==1.5.0 # via pytest pre-commit==3.8.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) pyasn1==0.6.0 # via # pyasn1-modules @@ -113,13 +113,13 @@ pyproject-hooks==1.1.0 # via build pytest==8.3.2 # via - # jobs-server (pyproject.toml) + # jobq-server (pyproject.toml) # pytest-cov # pytest-mock pytest-cov==5.0.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) pytest-mock==3.14.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) python-dateutil==2.9.0.post0 # via # jobq @@ -148,7 +148,7 @@ rich==13.7.1 rsa==4.9 # via google-auth ruff==0.5.6 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) shellingham==1.5.4 # via typer six==1.16.0 @@ -162,7 +162,7 @@ sniffio==1.3.1 starlette==0.37.2 # via fastapi testcontainers==4.8.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) typer==0.12.3 # via fastapi-cli typing-extensions==4.12.2 diff --git a/backend/requirements.txt b/backend/requirements.txt index 18987a56..87ec7e3e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -28,7 +28,7 @@ docker==7.1.0 email-validator==2.2.0 # via fastapi fastapi==0.112.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) fastapi-cli==0.0.5 # via fastapi google-auth==2.33.0 @@ -54,9 +54,9 @@ idna==3.7 jinja2==3.1.4 # via fastapi jobq @ git+https://github.com/aai-institute/jobq.git@7d36a70c06bbb2f17c1eed23f6d554cf0f938fb4#subdirectory=client - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) kubernetes==30.1.0 - # via jobs-server (pyproject.toml) + # via jobq-server (pyproject.toml) markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 From 46b58f355076afdf63ca0de81c0dec599294481e Mon Sep 17 00:00:00 2001 From: Adrian Rumpold Date: Wed, 18 Sep 2024 11:46:09 +0200 Subject: [PATCH 09/12] chore: Apply name change in backend deployment --- .../{jobs-server => jobq-server}/.helmignore | 0 backend/deploy/jobq-server/Chart.yaml | 7 ++++++ .../{jobs-server => jobq-server}/README.md | 8 +++---- .../templates/_helpers.tpl | 20 ++++++++-------- .../templates/deployment.yaml | 10 ++++---- .../templates/rbac.yaml | 24 +++++++++---------- .../templates/service.yaml | 6 ++--- .../templates/serviceaccount.yaml | 4 ++-- .../templates/tests/test-connection.yaml | 6 ++--- .../{jobs-server => jobq-server}/values.yaml | 6 +---- backend/deploy/jobs-server/Chart.yaml | 7 ------ backend/skaffold.yaml | 8 +++---- 12 files changed, 51 insertions(+), 55 deletions(-) rename backend/deploy/{jobs-server => jobq-server}/.helmignore (100%) create mode 100644 backend/deploy/jobq-server/Chart.yaml rename backend/deploy/{jobs-server => jobq-server}/README.md (86%) rename backend/deploy/{jobs-server => jobq-server}/templates/_helpers.tpl (75%) rename backend/deploy/{jobs-server => jobq-server}/templates/deployment.yaml (86%) rename backend/deploy/{jobs-server => jobq-server}/templates/rbac.yaml (67%) rename backend/deploy/{jobs-server => jobq-server}/templates/service.yaml (56%) rename backend/deploy/{jobs-server => jobq-server}/templates/serviceaccount.yaml (72%) rename backend/deploy/{jobs-server => jobq-server}/templates/tests/test-connection.yaml (59%) rename backend/deploy/{jobs-server => jobq-server}/values.yaml (90%) delete mode 100644 backend/deploy/jobs-server/Chart.yaml diff --git a/backend/deploy/jobs-server/.helmignore b/backend/deploy/jobq-server/.helmignore similarity index 100% rename from backend/deploy/jobs-server/.helmignore rename to backend/deploy/jobq-server/.helmignore diff --git a/backend/deploy/jobq-server/Chart.yaml b/backend/deploy/jobq-server/Chart.yaml new file mode 100644 index 00000000..847a8a3f --- /dev/null +++ b/backend/deploy/jobq-server/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: jobq-server +description: Helm chart for the jobq backend server + +type: application +version: 0.1.0 +appVersion: "main" diff --git a/backend/deploy/jobs-server/README.md b/backend/deploy/jobq-server/README.md similarity index 86% rename from backend/deploy/jobs-server/README.md rename to backend/deploy/jobq-server/README.md index 34555bd0..22a6c2cb 100644 --- a/backend/deploy/jobs-server/README.md +++ b/backend/deploy/jobq-server/README.md @@ -1,8 +1,8 @@ -# jobs-server +# jobq-server -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.0](https://img.shields.io/badge/AppVersion-0.1.0-informational?style=flat-square) +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: main](https://img.shields.io/badge/AppVersion-main-informational?style=flat-square) -Helm chart for Jobs Server +Helm chart for the jobq backend server ## Values @@ -11,7 +11,7 @@ Helm chart for Jobs Server | affinity | object | `{}` | | | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"jobs-server"` | | +| image.repository | string | `"ghcr.io/aai-institute/jobq-server"` | | | image.tag | string | `""` | | | imagePullSecrets | list | `[]` | | | livenessProbe.httpGet.path | string | `"/health"` | | diff --git a/backend/deploy/jobs-server/templates/_helpers.tpl b/backend/deploy/jobq-server/templates/_helpers.tpl similarity index 75% rename from backend/deploy/jobs-server/templates/_helpers.tpl rename to backend/deploy/jobq-server/templates/_helpers.tpl index 204a916f..cc5c5d8a 100644 --- a/backend/deploy/jobs-server/templates/_helpers.tpl +++ b/backend/deploy/jobq-server/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "jobs-server.name" -}} +{{- define "jobq-server.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "jobs-server.fullname" -}} +{{- define "jobq-server.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "jobs-server.chart" -}} +{{- define "jobq-server.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "jobs-server.labels" -}} -helm.sh/chart: {{ include "jobs-server.chart" . }} -{{ include "jobs-server.selectorLabels" . }} +{{- define "jobq-server.labels" -}} +helm.sh/chart: {{ include "jobq-server.chart" . }} +{{ include "jobq-server.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,17 +45,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "jobs-server.selectorLabels" -}} -app.kubernetes.io/name: {{ include "jobs-server.name" . }} +{{- define "jobq-server.selectorLabels" -}} +app.kubernetes.io/name: {{ include "jobq-server.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} -{{- define "jobs-server.serviceAccountName" -}} +{{- define "jobq-server.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} -{{- default (include "jobs-server.fullname" .) .Values.serviceAccount.name }} +{{- default (include "jobq-server.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} diff --git a/backend/deploy/jobs-server/templates/deployment.yaml b/backend/deploy/jobq-server/templates/deployment.yaml similarity index 86% rename from backend/deploy/jobs-server/templates/deployment.yaml rename to backend/deploy/jobq-server/templates/deployment.yaml index ad01a053..19eb4306 100644 --- a/backend/deploy/jobs-server/templates/deployment.yaml +++ b/backend/deploy/jobq-server/templates/deployment.yaml @@ -1,14 +1,14 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "jobs-server.fullname" . }} + name: {{ include "jobq-server.fullname" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - {{- include "jobs-server.selectorLabels" . | nindent 6 }} + {{- include "jobq-server.selectorLabels" . | nindent 6 }} template: metadata: {{- with .Values.podAnnotations }} @@ -16,7 +16,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "jobs-server.labels" . | nindent 8 }} + {{- include "jobq-server.labels" . | nindent 8 }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -25,7 +25,7 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "jobs-server.serviceAccountName" . }} + serviceAccountName: {{ include "jobq-server.serviceAccountName" . }} securityContext: runAsUser: 1000 runAsGroup: 1000 diff --git a/backend/deploy/jobs-server/templates/rbac.yaml b/backend/deploy/jobq-server/templates/rbac.yaml similarity index 67% rename from backend/deploy/jobs-server/templates/rbac.yaml rename to backend/deploy/jobq-server/templates/rbac.yaml index d3f089d2..6f8e955e 100644 --- a/backend/deploy/jobs-server/templates/rbac.yaml +++ b/backend/deploy/jobq-server/templates/rbac.yaml @@ -2,9 +2,9 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} rules: - apiGroups: ["kueue.x-k8s.io"] resources: ["workloads", "localqueues"] @@ -25,9 +25,9 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} rules: - apiGroups: ["kueue.x-k8s.io"] resources: ["workloadpriorityclasses", "clusterqueues"] @@ -39,30 +39,30 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} subjects: - kind: ServiceAccount - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} subjects: - kind: ServiceAccount - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} namespace: {{ .Release.Namespace }} {{- end -}} diff --git a/backend/deploy/jobs-server/templates/service.yaml b/backend/deploy/jobq-server/templates/service.yaml similarity index 56% rename from backend/deploy/jobs-server/templates/service.yaml rename to backend/deploy/jobq-server/templates/service.yaml index 94786ba7..171c6995 100644 --- a/backend/deploy/jobs-server/templates/service.yaml +++ b/backend/deploy/jobq-server/templates/service.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "jobs-server.fullname" . }} + name: {{ include "jobq-server.fullname" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} ports: @@ -12,4 +12,4 @@ spec: protocol: TCP name: http selector: - {{- include "jobs-server.selectorLabels" . | nindent 4 }} + {{- include "jobq-server.selectorLabels" . | nindent 4 }} diff --git a/backend/deploy/jobs-server/templates/serviceaccount.yaml b/backend/deploy/jobq-server/templates/serviceaccount.yaml similarity index 72% rename from backend/deploy/jobs-server/templates/serviceaccount.yaml rename to backend/deploy/jobq-server/templates/serviceaccount.yaml index d8adc067..3c150288 100644 --- a/backend/deploy/jobs-server/templates/serviceaccount.yaml +++ b/backend/deploy/jobq-server/templates/serviceaccount.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "jobs-server.serviceAccountName" . }} + name: {{ include "jobq-server.serviceAccountName" . }} labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/backend/deploy/jobs-server/templates/tests/test-connection.yaml b/backend/deploy/jobq-server/templates/tests/test-connection.yaml similarity index 59% rename from backend/deploy/jobs-server/templates/tests/test-connection.yaml rename to backend/deploy/jobq-server/templates/tests/test-connection.yaml index bd27b45d..9d5e481a 100644 --- a/backend/deploy/jobs-server/templates/tests/test-connection.yaml +++ b/backend/deploy/jobq-server/templates/tests/test-connection.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Pod metadata: - name: "{{ include "jobs-server.fullname" . }}-test-connection" + name: "{{ include "jobq-server.fullname" . }}-test-connection" labels: - {{- include "jobs-server.labels" . | nindent 4 }} + {{- include "jobq-server.labels" . | nindent 4 }} annotations: "helm.sh/hook": test "helm.sh/hook-delete-policy": hook-succeeded @@ -12,5 +12,5 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "jobs-server.fullname" . }}:{{ .Values.service.port }}/health'] + args: ['{{ include "jobq-server.fullname" . }}:{{ .Values.service.port }}/health'] restartPolicy: Never diff --git a/backend/deploy/jobs-server/values.yaml b/backend/deploy/jobq-server/values.yaml similarity index 90% rename from backend/deploy/jobs-server/values.yaml rename to backend/deploy/jobq-server/values.yaml index d9e3e91e..d22eadec 100644 --- a/backend/deploy/jobs-server/values.yaml +++ b/backend/deploy/jobq-server/values.yaml @@ -1,11 +1,7 @@ -# Default values for jobs-server. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - replicaCount: 1 image: - repository: ghcr.io/aai-institute/jobs-server + repository: ghcr.io/aai-institute/jobq-server pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" diff --git a/backend/deploy/jobs-server/Chart.yaml b/backend/deploy/jobs-server/Chart.yaml deleted file mode 100644 index ee32b043..00000000 --- a/backend/deploy/jobs-server/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -name: jobs-server -description: Helm chart for Jobs Server - -type: application -version: 0.1.0 -appVersion: "0.1.0" diff --git a/backend/skaffold.yaml b/backend/skaffold.yaml index 1cfc442c..00ff7149 100644 --- a/backend/skaffold.yaml +++ b/backend/skaffold.yaml @@ -4,14 +4,14 @@ metadata: name: backend build: artifacts: - - image: ghcr.io/aai-institute/jobs-server + - image: ghcr.io/aai-institute/jobq-server docker: dockerfile: Dockerfile deploy: helm: releases: - - name: jobs-server - chartPath: deploy/jobs-server + - name: jobq-server + chartPath: deploy/jobq-server valuesFiles: - - deploy/jobs-server/values.yaml + - deploy/jobq-server/values.yaml version: 0.1.0 From 30881ff1d5fad7fd6b661d41dc7247aba850f7a9 Mon Sep 17 00:00:00 2001 From: Nicholas Junge Date: Wed, 18 Sep 2024 11:57:42 +0200 Subject: [PATCH 10/12] Fix occurrences of jobby in k8s labels --- backend/src/jobq_server/utils/k8s.py | 2 +- client/examples/example_hello.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/jobq_server/utils/k8s.py b/backend/src/jobq_server/utils/k8s.py index ab2bb93b..9d935c82 100644 --- a/backend/src/jobq_server/utils/k8s.py +++ b/backend/src/jobq_server/utils/k8s.py @@ -28,7 +28,7 @@ def k8s_annotations( """Determine the Kubernetes annotations for a Job""" # Store as annotations since labels have restrictive value formats options = job.options.labels if job.options else {} - context = {"x-jobby.io/submission-context": json.dumps(context)} if context else {} + context = {"x-jobq.io/submission-context": json.dumps(context)} if context else {} return options | context diff --git a/client/examples/example_hello.py b/client/examples/example_hello.py index 63d444ad..5505c251 100644 --- a/client/examples/example_hello.py +++ b/client/examples/example_hello.py @@ -6,7 +6,7 @@ @job( options=JobOptions( - labels={"type": "hello-world@dev", "x-jobby.io/key": "value"}, + labels={"type": "hello-world@dev", "x-jobq.io/key": "value"}, resources=ResourceOptions(memory="1Gi", cpu="1"), scheduling=SchedulingOptions( priority_class="background", queue_name="user-queue" From ad8ba9c215a0747ba0a1c01430f7b5ad336edecc Mon Sep 17 00:00:00 2001 From: Nicholas Junge Date: Wed, 18 Sep 2024 11:57:58 +0200 Subject: [PATCH 11/12] Regen client lockfiles with jobq dependency, rename in backend README --- backend/README.md | 2 +- client/requirements-dev.txt | 48 ++++++++++++++++++------------------- client/requirements.txt | 12 +++++----- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/backend/README.md b/backend/README.md index e63e9880..3ba65b1a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,5 +1,5 @@ ## Installation -For development in the backend you need the client, `job-queue` as an editable dependency. +For development in the backend you need the client, `jobq`, as an editable dependency. The installation via the `requirements-dev.txt` depends on setting the `PROJECT_ROOT` environment variable you can set with executing ```bash export PROJECT_ROOT=$(pwd) diff --git a/client/requirements-dev.txt b/client/requirements-dev.txt index 161650e1..3e005a7a 100644 --- a/client/requirements-dev.txt +++ b/client/requirements-dev.txt @@ -3,13 +3,13 @@ annotated-types==0.7.0 # via pydantic appnope==0.1.4 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) babel==2.16.0 # via mkdocs-material black==24.8.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) build==1.2.1 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) certifi==2024.8.30 # via requests cfgv==3.4.0 @@ -18,7 +18,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via - # job-queue (pyproject.toml) + # jobq (pyproject.toml) # black # mkdocs # mkdocstrings @@ -31,9 +31,9 @@ coverage==7.6.1 distlib==0.3.8 # via virtualenv docker==7.1.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) docstring-parser==0.16 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) filelock==3.15.4 # via virtualenv ghp-import==2.1.0 @@ -41,7 +41,7 @@ ghp-import==2.1.0 griffe==1.2.0 # via mkdocstrings-python humanize==4.10.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) identify==2.6.0 # via pre-commit idna==3.8 @@ -80,10 +80,10 @@ mergedeep==1.3.4 # mkdocs # mkdocs-get-deps mike==2.1.3 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocs==1.6.1 # via - # job-queue (pyproject.toml) + # jobq (pyproject.toml) # mike # mkdocs-autorefs # mkdocs-callouts @@ -98,24 +98,24 @@ mkdocs-autorefs==1.2.0 # mkdocstrings # mkdocstrings-python mkdocs-callouts==1.14.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocs-gen-files==0.5.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocs-get-deps==0.2.0 # via mkdocs mkdocs-include-dir-to-nav==1.2.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocs-literate-nav==0.6.1 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocs-material==9.5.34 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocs-material-extensions==1.3.1 # via mkdocs-material mkdocs-section-index==0.3.9 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) mkdocstrings==0.26.0 # via - # job-queue (pyproject.toml) + # jobq (pyproject.toml) # mkdocstrings-python mkdocstrings-python==1.11.1 # via mkdocstrings @@ -144,9 +144,9 @@ platformdirs==4.2.2 pluggy==1.5.0 # via pytest pre-commit==3.8.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) pydantic==2.9.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) pydantic-core==2.23.2 # via pydantic pygments==2.18.0 @@ -163,17 +163,17 @@ pyproject-hooks==1.1.0 # via build pytest==8.3.2 # via - # job-queue (pyproject.toml) + # jobq (pyproject.toml) # pytest-cov pytest-cov==5.0.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) python-dateutil==2.9.0.post0 # via - # job-queue (pyproject.toml) + # jobq (pyproject.toml) # ghp-import pyyaml==6.0.2 # via - # job-queue (pyproject.toml) + # jobq (pyproject.toml) # mike # mkdocs # mkdocs-get-deps @@ -191,9 +191,9 @@ requests==2.32.3 # docker # mkdocs-material rich==13.8.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) ruff==0.6.4 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) six==1.16.0 # via python-dateutil typing-extensions==4.12.2 diff --git a/client/requirements.txt b/client/requirements.txt index b29f3af1..4be9faac 100644 --- a/client/requirements.txt +++ b/client/requirements.txt @@ -7,9 +7,9 @@ certifi==2024.8.30 charset-normalizer==3.3.2 # via requests docker==7.1.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) humanize==4.10.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) idna==3.8 # via requests markdown-it-py==3.0.0 @@ -17,19 +17,19 @@ markdown-it-py==3.0.0 mdurl==0.1.2 # via markdown-it-py pydantic==2.9.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) pydantic-core==2.23.2 # via pydantic pygments==2.18.0 # via rich python-dateutil==2.9.0.post0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) pyyaml==6.0.2 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) requests==2.32.3 # via docker rich==13.8.0 - # via job-queue (pyproject.toml) + # via jobq (pyproject.toml) six==1.16.0 # via python-dateutil typing-extensions==4.12.2 From aafb9575fa3e3f082c20f9b9772cf26f71cfa30f Mon Sep 17 00:00:00 2001 From: Max Mynter Date: Wed, 18 Sep 2024 12:07:45 +0200 Subject: [PATCH 12/12] Fix occurrences of `Infrastructure Product` where applicable --- backend/src/jobq_server/__main__.py | 2 +- client/docs/index.md | 2 +- client/mkdocs.yml | 2 +- infra/Pulumi.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/jobq_server/__main__.py b/backend/src/jobq_server/__main__.py index f8122068..b73e3ce2 100644 --- a/backend/src/jobq_server/__main__.py +++ b/backend/src/jobq_server/__main__.py @@ -15,7 +15,7 @@ async def lifespan(app: FastAPI): app = FastAPI( - title="infrastructure-product API", + title="the jobq cluster workflow management tool backend", description="Backend service for the appliedAI infrastructure product", lifespan=lifespan, ) diff --git a/client/docs/index.md b/client/docs/index.md index 346ef01e..f20640a1 100644 --- a/client/docs/index.md +++ b/client/docs/index.md @@ -1,3 +1,3 @@ -# Infrastructure Project +# Jobq, a cluster workflow scheduling tool This documentation is work in progress. diff --git a/client/mkdocs.yml b/client/mkdocs.yml index 42433933..85721428 100644 --- a/client/mkdocs.yml +++ b/client/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: infrastructure-product +site_name: Jobq site_dir: public/docs repo_url: https://github.com/aai-institute/infrastructure-product edit_uri: edit/main/docs/ diff --git a/infra/Pulumi.yaml b/infra/Pulumi.yaml index 1f4b0d06..7a4b9214 100644 --- a/infra/Pulumi.yaml +++ b/infra/Pulumi.yaml @@ -1,6 +1,6 @@ name: gcp-infra-product-dev runtime: nodejs -description: Google Cloud playground environment for the MLOps Infrastructure Product +description: Google Cloud playground environment for jobq, the MLOps Infrastructure Product config: pulumi:tags: value: