import sys
from types import MappingProxyType
from typing import TYPE_CHECKING
from weaver.base import Constants
[docs]WPS_BOUNDINGBOX = "bbox"
[docs]WPS_REFERENCE = "reference"
[docs]WPS_COMPLEX_DATA = "ComplexData"
[docs]WPS_LITERAL_DATA_BOOLEAN = frozenset(["bool", "boolean"])
[docs]WPS_LITERAL_DATA_DATETIME = frozenset(["date", "time", "dateTime"])
[docs]WPS_LITERAL_DATA_FLOAT = frozenset(["scale", "angle", "float", "double"])
[docs]WPS_LITERAL_DATA_INTEGER = frozenset(["int", "integer", "long", "positiveInteger", "nonNegativeInteger"])
[docs]WPS_LITERAL_DATA_STRING = frozenset({"anyURI", "string"} | WPS_LITERAL_DATA_DATETIME)
[docs]WPS_LITERAL_DATA_TYPES = frozenset(
WPS_LITERAL_DATA_BOOLEAN |
WPS_LITERAL_DATA_DATETIME |
WPS_LITERAL_DATA_FLOAT |
WPS_LITERAL_DATA_INTEGER |
WPS_LITERAL_DATA_STRING
)
# WPS 'type' string variations employed to indicate a Complex (file) I/O by different libraries
# for literal types, see 'any2cwl_literal_datatype' and 'any2wps_literal_datatype' functions
[docs]WPS_COMPLEX_TYPES = frozenset([WPS_COMPLEX, WPS_COMPLEX_DATA, WPS_REFERENCE])
# WPS 'type' string of all combinations (type of data / library implementation)
[docs]WPS_DATA_TYPES = frozenset({WPS_LITERAL, WPS_BOUNDINGBOX} | WPS_COMPLEX_TYPES)
[docs]class OpenSearchField(Constants):
[docs] START_DATE = "StartDate"
[docs] COLLECTION = "collection"
# data source cache
[docs] LOCAL_FILE_SCHEME = "opensearchfile" # must be a valid url scheme parsable by urlparse
[docs]CWL_NAMESPACE_ID = "cwl"
[docs]CWL_NAMESPACE_URL = "https://w3id.org/cwl/cwl#"
[docs]CWL_NAMESPACE = MappingProxyType({CWL_NAMESPACE_ID: CWL_NAMESPACE_URL})
# FIXME: convert to 'Constants' class
# CWL package (requirements/hints) corresponding to `ProcessType.APPLICATION`
[docs]CWL_REQUIREMENT_APP_BUILTIN = "BuiltinRequirement"
[docs]CWL_REQUIREMENT_APP_DOCKER = "DockerRequirement"
[docs]CWL_REQUIREMENT_APP_DOCKER_GPU = "DockerGpuRequirement" # backward compatibility
[docs]CWL_REQUIREMENT_APP_ESGF_CWT = "ESGF-CWTRequirement"
[docs]CWL_REQUIREMENT_APP_OGC_API = "OGCAPIRequirement"
[docs]CWL_REQUIREMENT_APP_WPS1 = "WPS1Requirement"
[docs]CWL_REQUIREMENT_APP_TYPES = frozenset([
CWL_REQUIREMENT_APP_BUILTIN,
CWL_REQUIREMENT_APP_DOCKER,
CWL_REQUIREMENT_APP_DOCKER_GPU, # backward compatibility, use 'DockerRequirement+cwltool:CUDARequirement' instead
CWL_REQUIREMENT_APP_ESGF_CWT,
CWL_REQUIREMENT_APP_OGC_API,
CWL_REQUIREMENT_APP_WPS1,
])
"""
Set of :term:`CWL` requirements consisting of known :term:`Application Package` by this `Weaver` instance.
"""
[docs]CWL_REQUIREMENT_APP_LOCAL = frozenset([
CWL_REQUIREMENT_APP_BUILTIN,
CWL_REQUIREMENT_APP_DOCKER,
CWL_REQUIREMENT_APP_DOCKER_GPU,
])
"""
Set of :term:`CWL` requirements that correspond to local execution of an :term:`Application Package`.
"""
# FIXME: convert to 'Constants' class
[docs]CWL_REQUIREMENT_APP_REMOTE = frozenset([
CWL_REQUIREMENT_APP_ESGF_CWT,
CWL_REQUIREMENT_APP_OGC_API,
CWL_REQUIREMENT_APP_WPS1,
])
"""
Set of :term:`CWL` requirements that correspond to remote execution of an :term:`Application Package`.
"""
[docs]CWL_REQUIREMENT_CUDA_DEFAULT_PARAMETERS = MappingProxyType({
# use older minimal version/capability to allow more chances to match any available GPU
# if this causes an issue for an actual application, it must provide it explicitly anyway
"cudaVersionMin": "10.0",
"cudaComputeCapability": "3.0",
# use minimum defaults, single GPU
"cudaDeviceCountMin": 1,
"cudaDeviceCountMax": 1,
})
"""
Parameters employed by default for updating :data:`CWL_REQUIREMENT_APP_DOCKER_GPU` into :data:`CWL_REQUIREMENT_CUDA`.
"""
# FIXME: convert to 'Constants' class
# NOTE: depending on the 'cwlVersion' of the document, some items are extensions or native to the standard specification
[docs]CWL_REQUIREMENT_CUDA = f"{CWL_TOOL_NAMESPACE_ID}:CUDARequirement"
[docs]CWL_REQUIREMENT_CUDA_NAMESPACE = CWL_TOOL_NAMESPACE
[docs]CWL_REQUIREMENT_ENV_VAR = "EnvVarRequirement"
[docs]CWL_REQUIREMENT_INIT_WORKDIR = "InitialWorkDirRequirement"
[docs]CWL_REQUIREMENT_INLINE_JAVASCRIPT = "InlineJavascriptRequirement"
[docs]CWL_REQUIREMENT_INPLACE_UPDATE = "InplaceUpdateRequirement"
[docs]CWL_REQUIREMENT_LOAD_LISTING = "LoadListingRequirement"
[docs]CWL_REQUIREMENT_MPI = "MPIRequirement" # no implication yet
[docs]CWL_REQUIREMENT_NETWORK_ACCESS = "NetworkAccess"
[docs]CWL_REQUIREMENT_PROCESS_GENERATOR = "ProcessGenerator"
[docs]CWL_REQUIREMENT_RESOURCE = "ResourceRequirement"
[docs]CWL_REQUIREMENT_SCATTER = "ScatterFeatureRequirement"
[docs]CWL_REQUIREMENT_SECRETS = "Secrets"
[docs]CWL_REQUIREMENT_TIME_LIMIT = "ToolTimeLimit"
[docs]CWL_REQUIREMENT_WORK_REUSE = "WorkReuse" # default is to reuse, employed to explicitly disable
[docs]CWL_REQUIREMENT_FEATURES = frozenset([
CWL_REQUIREMENT_CUDA,
CWL_REQUIREMENT_ENV_VAR,
CWL_REQUIREMENT_INIT_WORKDIR,
CWL_REQUIREMENT_INPLACE_UPDATE,
CWL_REQUIREMENT_INLINE_JAVASCRIPT,
CWL_REQUIREMENT_LOAD_LISTING,
# CWL_REQUIREMENT_MPI, # no implication yet
CWL_REQUIREMENT_NETWORK_ACCESS,
# CWL_REQUIREMENT_PROCESS_GENERATOR, # explicitly unsupported, works against Weaver's behavior
CWL_REQUIREMENT_RESOURCE, # FIXME: perform pre-check on job submit? (https://github.com/crim-ca/weaver/issues/138)
CWL_REQUIREMENT_SCATTER,
# CWL_REQUIREMENT_SECRETS, # FIXME: support CWL Secrets (https://github.com/crim-ca/weaver/issues/511)
CWL_REQUIREMENT_TIME_LIMIT,
CWL_REQUIREMENT_WORK_REUSE, # allow it, but makes sense only for Workflow steps if cwltool handles it by itself
])
"""
Set of :term:`CWL` requirements that corresponds to extra functionalities not completely defining
an :term:`Application Package` by themselves.
"""
[docs]CWL_REQUIREMENTS_SUPPORTED = frozenset(
CWL_REQUIREMENT_APP_TYPES |
CWL_REQUIREMENT_FEATURES
)
"""
Set of all :term:`CWL` requirements or hints that are supported for deployment of valid :term:`Application Package`.
"""
# CWL package types and extensions
[docs]PACKAGE_EXTENSIONS = frozenset(["yaml", "yml", "json", "cwl", "job"])
[docs]PACKAGE_SIMPLE_TYPES = frozenset(["string", "boolean", "float", "int", "integer", "long", "double"])
[docs]PACKAGE_LITERAL_TYPES = frozenset(PACKAGE_SIMPLE_TYPES | {"null", "Any"})
[docs]PACKAGE_FILE_TYPE = "File"
[docs]PACKAGE_DIRECTORY_TYPE = "Directory"
[docs]PACKAGE_COMPLEX_TYPES = frozenset([PACKAGE_FILE_TYPE, PACKAGE_DIRECTORY_TYPE])
[docs]PACKAGE_ENUM_BASE = "enum"
[docs]PACKAGE_CUSTOM_TYPES = frozenset([PACKAGE_ENUM_BASE]) # can be anything, but support "enum" which is more common
[docs]PACKAGE_ARRAY_BASE = "array"
[docs]PACKAGE_ARRAY_MAX_SIZE = sys.maxsize # pywps doesn't allow None, so use max size # FIXME: unbounded (weaver #165)
[docs]PACKAGE_ARRAY_ITEMS = frozenset(PACKAGE_SIMPLE_TYPES | PACKAGE_CUSTOM_TYPES | PACKAGE_COMPLEX_TYPES)
[docs]PACKAGE_ARRAY_TYPES = frozenset([f"{item}[]" for item in PACKAGE_ARRAY_ITEMS])
# string values the lowest 'type' field can have by itself (as simple mapping {type: <type-string>})
[docs]PACKAGE_TYPE_NULLABLE = frozenset(PACKAGE_SIMPLE_TYPES | PACKAGE_CUSTOM_TYPES | PACKAGE_COMPLEX_TYPES)
# shortcut notations that can be employed to convert basic types into corresponding array or nullable variants
[docs]PACKAGE_SHORTCUTS = frozenset(
{f"{typ}?" for typ in PACKAGE_TYPE_NULLABLE} |
PACKAGE_ARRAY_TYPES |
{f"{typ}?" for typ in PACKAGE_ARRAY_TYPES}
)
[docs]PACKAGE_TYPE_POSSIBLE_VALUES = frozenset(
PACKAGE_LITERAL_TYPES |
PACKAGE_COMPLEX_TYPES |
PACKAGE_SHORTCUTS
)
# OpenAPI definitions
[docs]OAS_COMPLEX_TYPES = frozenset(["object"])
[docs]OAS_ARRAY_TYPES = frozenset(["array"])
[docs]OAS_LITERAL_TYPES = frozenset(["boolean", "integer", "number", "string"])
[docs]OAS_LITERAL_NUMERIC = frozenset(["integer", "number"])
)
[docs]OAS_KEYWORD_TYPES = frozenset(["allOf", "anyOf", "oneOf", "not"])
[docs]OAS_DATA_TYPES = frozenset(
OAS_COMPLEX_TYPES |
OAS_ARRAY_TYPES |
OAS_LITERAL_TYPES
)
[docs]class ProcessSchema(Constants):
"""
Schema selector to represent a :term:`Process` description.
"""
if TYPE_CHECKING:
from typing import Union
from weaver.typedefs import Literal
# pylint: disable=invalid-name
[docs] CWL_RequirementNames = Literal[
CWL_REQUIREMENT_APP_BUILTIN,
CWL_REQUIREMENT_APP_DOCKER,
CWL_REQUIREMENT_APP_DOCKER_GPU,
CWL_REQUIREMENT_APP_ESGF_CWT,
CWL_REQUIREMENT_APP_OGC_API,
CWL_REQUIREMENT_APP_WPS1,
CWL_REQUIREMENT_CUDA,
CWL_REQUIREMENT_ENV_VAR,
CWL_REQUIREMENT_INIT_WORKDIR,
CWL_REQUIREMENT_INLINE_JAVASCRIPT,
CWL_REQUIREMENT_INPLACE_UPDATE,
CWL_REQUIREMENT_LOAD_LISTING,
CWL_REQUIREMENT_MPI,
CWL_REQUIREMENT_NETWORK_ACCESS,
CWL_REQUIREMENT_RESOURCE,
CWL_REQUIREMENT_SCATTER,
CWL_REQUIREMENT_SECRETS,
CWL_REQUIREMENT_TIME_LIMIT,
CWL_REQUIREMENT_WORK_REUSE,
]
ProcessSchemaType = Literal[ProcessSchema.OGC, ProcessSchema.OLD, ProcessSchema.WPS]
WPS_ComplexType = Literal[WPS_COMPLEX, WPS_COMPLEX_DATA, WPS_REFERENCE]
WPS_DataType = Union[Literal[WPS_LITERAL, WPS_BOUNDINGBOX], WPS_ComplexType]
JobInputsOutputsSchemaType = Literal[
JobInputsOutputsSchema.OGC_STRICT,
JobInputsOutputsSchema.OLD_STRICT,
JobInputsOutputsSchema.OGC,
JobInputsOutputsSchema.OLD
]