Skip to content

checkers

Classes#

CheckFailDelta #

Bases: _CheckDelta

Checks for deltas between consecutive values larger than 'fail_delta'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'fail_delta'

CheckFailMax #

Bases: _CheckMax

Checks for values greater than 'fail_max'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'fail_max'

CheckFailMin #

Bases: _CheckMin

Checks for values less than 'fail_min'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'fail_min'

CheckFailRangeMax #

Bases: _CheckMax

Checks for values greater than 'fail_range'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'fail_range'

CheckFailRangeMin #

Bases: _CheckMin

Checks for values less than 'fail_range'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'fail_range'

CheckMissing #

Bases: QualityChecker

Checks if any data are missing. A variable's data are considered missing if they are set to the variable's _FillValue (if it has a _FillValue) or NaN (NaT for datetime- like variables).

Functions#

run #
run(
    dataset: xr.Dataset, variable_name: str
) -> NDArray[np.bool_]
Source code in tsdat/qc/checkers.py 
46
47
48
49
50
51
52
53
54
55
56
57
def run(self, dataset: xr.Dataset, variable_name: str) -> NDArray[np.bool_]:
    results: NDArray[np.bool_] = dataset[variable_name].isnull().data

    if "_FillValue" in dataset[variable_name].attrs:
        fill_value = dataset[variable_name].attrs["_FillValue"]
        results |= dataset[variable_name].data == fill_value

    elif np.issubdtype(dataset[variable_name].data.dtype, str):  # type: ignore
        fill_value = ""
        results |= dataset[variable_name].data == fill_value

    return results

CheckMonotonic #

Bases: QualityChecker

Checks if any values are not ordered strictly monotonically (i.e. values must all be increasing or all decreasing). The check marks values as failed if they break from a monotonic order.

Attributes#

parameters class-attribute instance-attribute #
parameters: Parameters = Parameters()

Classes#

Parameters #

Bases: BaseModel

Attributes#
dim class-attribute instance-attribute # 
dim: Optional[str] = None
require_decreasing class-attribute instance-attribute # 
require_decreasing: bool = False
require_increasing class-attribute instance-attribute # 
require_increasing: bool = False
Functions#
check_monotonic_not_increasing_and_decreasing classmethod # 
check_monotonic_not_increasing_and_decreasing(
    inc: bool, values: Dict[str, Any]
) -> bool
Source code in tsdat/qc/checkers.py 
73
74
75
76
77
78
79
80
81
82
83
@validator("require_increasing")
@classmethod
def check_monotonic_not_increasing_and_decreasing(
    cls, inc: bool, values: Dict[str, Any]
) -> bool:
    if inc and values["require_decreasing"]:
        raise ValueError(
            "CheckMonotonic -> Parameters: cannot set both 'require_increasing'"
            " and 'require_decreasing'. Please set one or both to False."
        )
    return inc

Functions#

get_axis #
get_axis(variable: xr.DataArray) -> int
Source code in tsdat/qc/checkers.py 
146
147
148
149
def get_axis(self, variable: xr.DataArray) -> int:
    if not (dim := self.parameters.dim):
        dim = variable.dims[0]  # type: ignore
    return variable.get_axis_num(dim)  # type: ignore
run #
run(
    dataset: xr.Dataset, variable_name: str
) -> Union[NDArray[np.bool_], None]
Source code in tsdat/qc/checkers.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def run(
    self, dataset: xr.Dataset, variable_name: str
) -> Union[NDArray[np.bool_], None]:
    variable = dataset[variable_name]
    failures = np.full(variable.shape, False)

    if self.parameters.dim is None and len(variable.shape) == 2:
        logger.warning(
            "Variable '%s' has shape '%s'. 2D variables must provide a 'dim'"
            " parameter for the name of the dimension to check the variable for"
            " monotonicity.",
            variable_name,
            variable.shape,
        )
        return None

    if variable.values.dtype.kind in {"U", "S"}:  # type: ignore
        logger.warning(
            "Variable '%s' has dtype '%s', which is currently not supported for"
            " monotonicity checks.",
            variable_name,
            variable.values.dtype,  # type: ignore
        )
        return None

    axis = self.get_axis(variable)
    zero = np.timedelta64(0) if is_datetime_like(variable.data) else 0
    direction: Literal["increasing", "decreasing", ""] = ""

    if self.parameters.require_decreasing:
        direction = "decreasing"
    elif self.parameters.require_increasing:
        direction = "increasing"
    else:
        diff = np.diff(variable.data, axis=axis)  # type: ignore
        direction = (
            "increasing"
            if np.sum(diff > zero) >= np.sum(diff < zero)
            else "decreasing"
        )

    # Find all the values where things break, not just those flagged by diff
    # if any(failures) and not all(failures):
    if len(variable.shape) == 1:
        prev = variable.values[0]
        for i, value in enumerate(variable.values[1:]):
            success = value < prev if direction == "decreasing" else value > prev
            if success:
                prev = value  # only update prev on success
            else:
                failures[i + 1] = True
    else:
        # 2D diff isn't as clever with failing indexes; just report all individual
        # points that fail
        diff = np.gradient(variable.data)[axis]
        failures = diff <= zero if direction == "increasing" else diff >= zero

    return failures

CheckValidDelta #

Bases: _CheckDelta

Checks for deltas between consecutive values larger than 'valid_delta'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'valid_delta'

CheckValidMax #

Bases: _CheckMax

Checks for values greater than 'valid_max'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'valid_max'

CheckValidMin #

Bases: _CheckMin

Checks for values less than 'valid_min'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'valid_min'

CheckValidRangeMax #

Bases: _CheckMax

Checks for values greater than 'valid_range'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'valid_range'

CheckValidRangeMin #

Bases: _CheckMin

Checks for values less than 'valid_range'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'valid_range'

CheckWarnDelta #

Bases: _CheckDelta

Checks for deltas between consecutive values larger than 'warn_delta'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'warn_delta'

CheckWarnMax #

Bases: _CheckMax

Checks for values greater than 'warn_max'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'warn_max'

CheckWarnMin #

Bases: _CheckMin

Checks for values less than 'warn_min'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'warn_min'

CheckWarnRangeMax #

Bases: _CheckMax

Checks for values greater than 'warn_range'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'warn_range'

CheckWarnRangeMin #

Bases: _CheckMin

Checks for values less than 'warn_range'.

Attributes#

attribute_name class-attribute instance-attribute #
attribute_name: str = 'warn_range'

Functions#

is_datetime_like #

is_datetime_like(data: NDArray[Any]) -> bool

Checks if the array has a datetime-like dtype (datetime, timedelta, date)

Source code in tsdat/qc/checkers.py 
33
34
35
def is_datetime_like(data: NDArray[Any]) -> bool:
    """Checks if the array has a datetime-like dtype (datetime, timedelta, date)"""
    return np.issubdtype(data.dtype, (np.datetime64, np.timedelta64))