beamds.beam.orchestration package

Submodules

beamds.beam.orchestration.cluster module

beamds.beam.orchestration.config module

class beamds.beam.orchestration.config.BeamManagerConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig

parameters = [BeamParam(name='clusters', type=<class 'list'>, default=[], help='list of clusters', tags=None), BeamParam(name='labels', type=<class 'dict'>, default={'runai/node-pool': 'cpu-only'}, help='Labels for the deployment', tags=None)]

class beamds.beam.orchestration.config.CronJobConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig

parameters = [BeamParam(name='cron_job_name', type=<class 'str'>, default='beam-cron-job', help='Cron job name', tags=None), BeamParam(name='job_schedule', type=<class 'str'>, default='*/2 * * * *', help='Cron job schedule', tags=None), BeamParam(name='restart_policy_configs', type=<class 'dict'>, default={'condition': 'OnFailure', 'delay': '5s', 'active_deadline_seconds': 300, 'max_attempts': 3, 'window': '120s'}, help='Restart policy configuration for the deployment', tags=None)]

class beamds.beam.orchestration.config.JobConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig

parameters = [BeamParam(name='job_name', type=<class 'str'>, default='beam-job', help='Job Name', tags=None)]

class beamds.beam.orchestration.config.K8SConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: BeamConfig

parameters = [BeamParam(name='api_url', type=<class 'str'>, default='https://api.kh-dev.dt.local:6443', help='URL of the Kubernetes API server', tags=None), BeamParam(name='api_token', type=<class 'str'>, default='eyJhbGciOiJSUzI1NiIsImtpZCI6Imhtdk5nbTRoenVRenhkd0lWdnBWMUI0MmV2ZGpxMk8wQ0NaMlhmejZBc1UifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZXYiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoieW9zLXRva2VuLWQycDUyIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6InlvcyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImNlOWUzNzkyLThmZTAtNDgxNC05YTVlLWNlMTdmODJjOGU5MiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpkZXY6eW9zIn0.mKgXusdiiEVN3MzjQ6mZOTfjoY8LFz-1RxVCrDcq38V5AcxaEiTvOGm-6-Vd4ZTV15DR7ds2OqBqZcpcdeuD_eSqZofsfF_dFM8483mXsA8obzBjXiOw0sLeUAq7ZCzb0sTVOySfz4v84MGHgCbMOfD92sfVsfbXhvAXYY2HLX2Vh5og6spjz0P__BBpL--8rfaR1bpua8bMhR5gOreuednJ8hTFPsxTtgZkNppBdHC6WO0j6rm5APDLhu0CMj1_Dwdee4KL0xtt5vKK1YDqy2fdq4ApFP5kYIZu0YnIsliI-msGgX1ioT_eqj_7oz6Hdi5gdSiNDVGnXbhwkdYchslB4evLCEGXAEI2uFQ0d2wVkCcFjGqiVjHdpQa6JCxWClXBveap8o78eM_c59WV343YQri2pfiGthAZUYxIz5mXddV9237OHUh6YwUFyosaKv853c_W-py8rCsxUVFA_o7PFkfHnVogPETjJw-ZzVTxk_PYzxGl9Dh8kEVhJCiPrFBlNtoJVnaEcdNKD_z8I2hr3ca6DB6k6Ws-ABIYWOKO3yu07wp6RdTYeoS3wjWB9GkcjW52UHBi1hQ2qrR1m-X0DsdTrg_PTuw-9KgXz5LnekPJwMrzRn2DFaswOmXOynTEM_PbvlsQ55DBntix_r2df2rWnCWgxbw9MuFog44', help='API token for the Kubernetes API server', tags=None), BeamParam(name='beam_version', type=<class 'str'>, default=None, help='Beam version', tags=None), BeamParam(name='project_name', type=<class 'str'>, default='dev', help='Name of the project', tags=None), BeamParam(name='deployment_name', type=<class 'str'>, default='demo', help='Name of the deployment', tags=None), BeamParam(name='debug_sleep', type=<class 'bool'>, default=False, help='Sleep Infinity deployment', tags=None), BeamParam(name='labels', type=<class 'dict'>, default={'runai/node-pool': 'cpu-only'}, help='Labels for the deployment', tags=None), BeamParam(name='image_name', type=<class 'str'>, default='harbor.dt.local/public/beam:20240801', help='Name of the image to deploy', tags=None), BeamParam(name='command', type=<class 'dict'>, default={}, help='Command configuration for the deployment', tags=None), BeamParam(name='os_namespace', type=<class 'str'>, default='dev', help='Namespace for the deployment', tags=None), BeamParam(name='replicas', type=<class 'int'>, default=1, help='Number of replicas for the deployment', tags=None), BeamParam(name='entrypoint_args', type=<class 'list'>, default=[], help='Arguments for the container entrypoint', tags=None), BeamParam(name='entrypoint_envs', type=<class 'dict'>, default={}, help='Environment variables for the container entrypoint', tags=None), BeamParam(name='use_scc', type=<class 'bool'>, default=True, help='Use SCC control parameter', tags=None), BeamParam(name='scc_name', type=<class 'str'>, default='anyuid', help='SCC name', tags=None), BeamParam(name='create_service_account', type=<class 'bool'>, default=True, help='Create service account', tags=None), BeamParam(name='security_context_config', type=<class 'dict'>, default={'add_capabilities': ['SYS_CHROOT', 'CAP_AUDIT_CONTROL', 'CAP_AUDIT_WRITE'], 'enable_security_context': False, 'privileged': False}, help='Security context configuration', tags=None), BeamParam(name='use_node_selector', type=<class 'bool'>, default=False, help='Use node selector', tags=None), BeamParam(name='node_selector', type=<class 'dict'>, default={'gpu-type': 'tesla-a100'}, help='Node selector', tags=None), BeamParam(name='cpu_requests', type=<class 'str'>, default='4', help='CPU requests', tags=None), BeamParam(name='cpu_limits', type=<class 'str'>, default='4', help='CPU limits', tags=None), BeamParam(name='memory_requests', type=<class 'str'>, default='0.4', help='Memory requests [e.g.: 0.5Gi, 1.5, 400m]', tags=None), BeamParam(name='memory_limits', type=<class 'str'>, default='0.4', help='Memory limits [e.g.: 0.5Gi, 1.5, 400m]', tags=None), BeamParam(name='gpu_requests', type=<class 'str'>, default='1', help='GPU requests', tags=None), BeamParam(name='gpu_limits', type=<class 'str'>, default='1', help='GPU limits', tags=None), BeamParam(name='use_gpu', type=<class 'bool'>, default=False, help='Use GPU', tags=None), BeamParam(name='n_pods', type=<class 'int'>, default=1, help='Number of pods', tags=None), BeamParam(name='storage_configs', type=<class 'list'>, default=[{'create_pvc': False, 'pvc_access_mode': 'ReadWriteMany', 'pvc_mount_path': '/data-pvc', 'pvc_name': 'data-pvc', 'pvc_size': '500'}], help='Storage configuration for the deployment', tags=None), BeamParam(name='service_configs', type=<class 'list'>, default=[], help='Service configurations', tags=None), BeamParam(name='service_configs', type=<class 'list'>, default=[{'create_ingress': False, 'create_route': True, 'ingress_host': 'home-page.example.com', 'port': 35000, 'port_name': 'flask-port', 'service_name': 'flask', 'service_type': 'ClusterIP'}], help='Service configuration for the deployment', tags=None), BeamParam(name='user_idm_configs', type=<class 'list'>, default=[{'create_role_binding': False, 'project_name': 'ben-guryon', 'role_binding_name': 'yos', 'role_name': 'admin', 'user_name': 'yos'}], help='User IDM configurations', tags=None), BeamParam(name='route_timeout', type=<class 'int'>, default=599, help='Route timeout', tags=None), BeamParam(name='memory_storage_configs', type=<class 'list'>, default=[{'enabled': True, 'mount_path': '/dev/shm', 'name': 'dshm', 'size_gb': 8}], help='Memory storage configuration for the deployment', tags=None), BeamParam(name='restart_policy_configs', type=<class 'dict'>, default={'condition': 'Always', 'delay': '5s', 'active_deadline_seconds': 300, 'max_attempts': 3, 'window': '120s'}, help='Restart policy configuration for the deployment', tags=None), BeamParam(name='check_project_exists', type=<class 'bool'>, default=True, help='Check if project exists', tags=None), BeamParam(name='entrypoint', type=<class 'str'>, default=None, help='Entrypoint for the container', tags=None), BeamParam(name='dockerfile', type=<class 'str'>, default=None, help='Dockerfile for the container', tags=None), BeamParam(name='docker_kwargs', type=<class 'dict'>, default={'version': '1.0.0', 'author': 'user@example.com'}, help='Auxiliary Docker arguments (for the build process)', tags=None)]

class beamds.beam.orchestration.config.RayClusterConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig

parameters = [BeamParam(name='n-pods', type=<class 'int'>, default=1, help='Number of Ray worker pods', tags=None)]

class beamds.beam.orchestration.config.RnDClusterConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig

parameters = [BeamParam(name='replicas', type=<class 'int'>, default=1, help='Number of replica pods', tags=None), BeamParam(name='send_email', type=<class 'bool'>, default=False, help='Send email', tags=None), BeamParam(name='body', type=<class 'str'>, default='Here is the cluster information:', help='Email body', tags=None), BeamParam(name='from_email', type=<class 'str'>, default='dayotech2018@gmail.com', help='From email address', tags=None), BeamParam(name='from_email_password', type=<class 'str'>, default='mkhdokjqwwmazyrf', help='From email password', tags=None), BeamParam(name='to_email', type=<class 'str'>, default=None, help='To email address', tags=None), BeamParam(name='send_email', type=<class 'bool'>, default=False, help='Send email or not', tags=None), BeamParam(name='smtp_server', type=<class 'str'>, default='smtp.gmail.com', help='SMTP server', tags=None), BeamParam(name='smtp_port', type=<class 'int'>, default=587, help='SMTP port', tags=None), BeamParam(name='subject', type=<class 'str'>, default='Cluster Deployment Information', help='Email subject', tags=None)]

class beamds.beam.orchestration.config.ServeClusterConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig, BeamServeConfig

defaults = {'alg': None, 'n_threads': 16}

parameters = [BeamParam(name='alg', type=<class 'str'>, default='/tmp/bundle', help='Algorithm object can be - bundle, object, image', tags=None), BeamParam(name='alg_image_name', type=<class 'str'>, default='alg-demo:latest', help='Algorithm image name', tags=None), BeamParam(name='base_image', type=<class 'str'>, default='harbor.dt.local/public/beam:20240801', help='Base image', tags=None), BeamParam(name='base_url', type=<class 'str'>, default='tcp://10.0.7.55:2375', help='Base URL', tags=None), BeamParam(name='requirements_blacklist', type=<class 'list'>, default=['sklearn'], help='Requirements blacklist', tags=None), BeamParam(name='send_email', type=<class 'bool'>, default=False, help='Send email', tags=None), BeamParam(name='body', type=<class 'str'>, default='Here is the cluster information:', help='Email body', tags=None), BeamParam(name='from_email', type=<class 'str'>, default='dayotech2018@gmail.com', help='From email address', tags=None), BeamParam(name='from_email_password', type=<class 'str'>, default='mkhdokjqwwmazyrf', help='From email password', tags=None), BeamParam(name='to_email', type=<class 'str'>, default='cluster@demo.com', help='To email address', tags=None), BeamParam(name='send_email', type=<class 'bool'>, default=False, help='Send email or not', tags=None), BeamParam(name='smtp_server', type=<class 'str'>, default='smtp.gmail.com', help='SMTP server', tags=None), BeamParam(name='smtp_port', type=<class 'int'>, default=587, help='SMTP port', tags=None), BeamParam(name='subject', type=<class 'str'>, default='Cluster Deployment Information', help='Email subject', tags=None), BeamParam(name='registry_url', type=<class 'str'>, default='harbor.dt.local', help='Registry URL', tags=None), BeamParam(name='registry_username', type=<class 'str'>, default='admin', help='Registry username', tags=None), BeamParam(name='registry_password', type=<class 'str'>, default='Har@123', help='Registry password', tags=None), BeamParam(name='registry_project_name', type=<class 'str'>, default='public', help='Registry project name', tags=None), BeamParam(name='push_image', type=<class 'bool'>, default=True, help='Push image to registry', tags=None), BeamParam(name='pods', type=<class 'list'>, default=[], help='List of pods', tags=None), BeamParam(name='copy-bundle', type=<class 'bool'>, default=False, help='Copy bundle to tmp directory', tags=None), BeamParam(name='path_to_state', type=<class 'str'>, default='/tmp', help='Path to bundle', tags=None)]

class beamds.beam.orchestration.config.StatefulSetConfig(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: K8SConfig

Configuration for StatefulSets

parameters = [BeamParam(name='statefulset_name', type=<class 'str'>, default='beam-statefulset', help='StatefulSet name', tags=None), BeamParam(name='replicas', type=<class 'int'>, default=1, help='Number of StatefulSet replicas', tags=None), BeamParam(name='service_name', type=<class 'str'>, default=None, help='Service name associated with the StatefulSet', tags=None), BeamParam(name='volume_claims', type=<class 'list'>, default=[], help='Volume claims for the StatefulSet', tags=None), BeamParam(name='update_strategy', type=<class 'str'>, default='RollingUpdate', help='Update strategy for the StatefulSet', tags=None), BeamParam(name='pod_management_policy', type=<class 'str'>, default='OrderedReady', help='Pod management policy', tags=None)]

beamds.beam.orchestration.dataclasses module

class beamds.beam.orchestration.dataclasses.CommandConfig(executable: str, arguments: List[str] = <factory>)

Bases: object

arguments: List[str]

as_list() → List[str]

Converts the command configuration to a list format suitable for Kubernetes containers.

executable: str

class beamds.beam.orchestration.dataclasses.ContainerConfig

Bases: object

class beamds.beam.orchestration.dataclasses.DeploymentConfig(container: beamds.beam.orchestration.dataclasses.ContainerConfig, pod: beamds.beam.orchestration.dataclasses.PodConfig)

Bases: object

container: ContainerConfig

get_labels() → Dict[str, str]

Returns the labels as a dictionary.

pod: PodConfig

class beamds.beam.orchestration.dataclasses.MemoryStorageConfig(name: str, mount_path: str, size_gb: beamds.beam.orchestration.units.K8SUnits | str | int = None, enabled: bool = True)

Bases: object

enabled: bool = True

mount_path: str

name: str

size_gb: K8SUnits | str | int = None

class beamds.beam.orchestration.dataclasses.PodConfig

Bases: object

class beamds.beam.orchestration.dataclasses.PodInfos(raw_pod_data: Dict[str, Any], metadata: beamds.beam.orchestration.dataclasses.PodMetadata)

Bases: object

metadata: PodMetadata

property name

raw_pod_data: Dict[str, Any]

class beamds.beam.orchestration.dataclasses.PodMetadata(name: str)

Bases: object

name: str

class beamds.beam.orchestration.dataclasses.RayPortsConfig(ray_ports: List[int] = <factory>)

Bases: object

ray_ports: List[int]

class beamds.beam.orchestration.dataclasses.RestartPolicyConfig(condition: str, max_attempts: int, delay: str, active_deadline_seconds: int, window: str)

Bases: object

active_deadline_seconds: int

condition: str

delay: str

max_attempts: int

to_dict()

window: str

class beamds.beam.orchestration.dataclasses.SecurityContextConfig(privileged: bool = False, runAsUser: str = None, add_capabilities: List[str] = <factory>, enable_security_context: bool = False)

Bases: object

add_capabilities: List[str]

enable_security_context: bool = False

privileged: bool = False

runAsUser: str = None

class beamds.beam.orchestration.dataclasses.ServiceConfig(port: int, service_name: str, service_type: str, port_name: str, create_route: bool = False, route_protocol: str = 'http', create_ingress: bool = False, ingress_host: str = None, ingress_path: str = '/', ingress_tls_secret: str = None, route_timeout: str = '599')

Bases: object

create_ingress: bool = False

create_route: bool = False

ingress_host: str = None

ingress_path: str = '/'

ingress_tls_secret: str = None

port: int

port_name: str

route_protocol: str = 'http'

route_timeout: str = '599'

service_name: str

service_type: str

class beamds.beam.orchestration.dataclasses.StorageConfig(pvc_name: str, pvc_mount_path: str, create_pvc: bool = False, pvc_size: beamds.beam.orchestration.units.K8SUnits | str | int = '1Gi', pvc_access_mode: str = 'ReadWriteMany')

Bases: object

create_pvc: bool = False

pvc_access_mode: str = 'ReadWriteMany'

pvc_mount_path: str

pvc_name: str

pvc_size: K8SUnits | str | int = '1Gi'

class beamds.beam.orchestration.dataclasses.UserIdmConfig(user_name: str, role_name: str, role_binding_name: str, project_name: str, role_namespace: str = 'default', create_role_binding: bool = False)

Bases: object

create_role_binding: bool = False

project_name: str

role_binding_name: str

role_name: str

role_namespace: str = 'default'

user_name: str

beamds.beam.orchestration.deploy module

beamds.beam.orchestration.k8s module

beamds.beam.orchestration.pod module

class beamds.beam.orchestration.pod.BeamPod(*args, _store_init_path=None, _save_init_args=True, **kwargs)

Bases: Processor

execute(command, pod_name=None, **kwargs)

Execute a command on a specific pod or on each pod if no pod name is provided.

static extract_pod_info(pod)

get_logs(**kwargs)

Get logs from each pod.

get_pod_resources()

Get resource usage for each pod.

get_pod_status()

Get the status of each pod.

property port_mapping

start()

Start each pod.

stop()

Stop each pod.

to_dict()

beamds.beam.orchestration.units module

class beamds.beam.orchestration.units.K8SUnits(value: int | float | str | K8SUnits, resource_type='memory')

Bases: object

MAX_CPU = 24000

MAX_MEMORY = 549755813888

property as_str

static parse_mixed_memory(value: float)

parse_numeric(value: int | float)

parse_str(value: str)

parse_value(value: int | float | str | K8SUnits)

validate_limits()

beamds.beam.orchestration.utils module

beamds.beam.orchestration.utils.convert_datetimes(data)

beamds.beam.orchestration.utils.ensure_rfc1123_compliance(name)

Ensures that a name complies with RFC 1123 by: - Lowercase all characters. - Replacing any invalid characters with dashes. - Replacing consecutive dashes with a single dash. - Trimming to the maximum allowed length for Kubernetes names (63 characters). - Ensuring the name doesn’t start or end with a dash. - Handling empty strings by assigning a default name.

Module contents

class beamds.beam.orchestration.K8SUnits(value: int | float | str | K8SUnits, resource_type='memory')

Bases: object

MAX_CPU = 24000

MAX_MEMORY = 549755813888

property as_str

static parse_mixed_memory(value: float)

parse_numeric(value: int | float)

parse_str(value: str)

parse_value(value: int | float | str | K8SUnits)

validate_limits()