Skip to content
Snippets Groups Projects
Commit c456858e authored by Oriol Tintó's avatar Oriol Tintó
Browse files

Fix dataset chunking.

parent ecf47de3
No related branches found
No related tags found
2 merge requests!10Code cleaning, better documentation and updated CI.,!9Fix CI publishing.
import chunk
import os import os
from copy import deepcopy from copy import deepcopy
from pathlib import Path from pathlib import Path
......
from abc import ABC
from .definitions import lossy_compressors_and_modes from .definitions import lossy_compressors_and_modes
import logging import logging
from typing import Mapping, Union from typing import Mapping, Union, Protocol
import hdf5plugin import hdf5plugin
...@@ -21,7 +22,6 @@ class _Mapping(Mapping): ...@@ -21,7 +22,6 @@ class _Mapping(Mapping):
""" """
Subclass to implement dunder methods that are mandatory for Mapping to avoid repeating the code everywhere. Subclass to implement dunder methods that are mandatory for Mapping to avoid repeating the code everywhere.
""" """
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
self._kwargs = {} self._kwargs = {}
...@@ -55,19 +55,6 @@ class Encoding(_Mapping): ...@@ -55,19 +55,6 @@ class Encoding(_Mapping):
def __repr__(self): def __repr__(self):
return f"{self.__class__.__name__}({self.to_string()})" return f"{self.__class__.__name__}({self.to_string()})"
def set_chunk_sizes(self, chunk_sizes: tuple) -> None:
"""
Method to add chunksizes into the encoding dictionary.
Parameters
----------
chunk_sizes
Returns
-------
"""
self._kwargs["chunksizes"] = chunk_sizes
class VariableEncoding(_Mapping): class VariableEncoding(_Mapping):
""" """
...@@ -97,7 +84,6 @@ class VariableEncoding(_Mapping): ...@@ -97,7 +84,6 @@ class VariableEncoding(_Mapping):
>>> VariableEncoding(backend="snappy", compression_level=9) >>> VariableEncoding(backend="snappy", compression_level=9)
""" """
def __new__(cls, def __new__(cls,
specification: str = None, specification: str = None,
compressor: str = None, compressor: str = None,
...@@ -131,14 +117,12 @@ class NullEncoding(Encoding): ...@@ -131,14 +117,12 @@ class NullEncoding(Encoding):
class LosslessEncoding(Encoding): class LosslessEncoding(Encoding):
def __init__(self, backend: str, compression_level: int): def __init__(self, backend: str, compression_level: int):
super().__init__()
self.backend = backend if backend is not None else rules.LOSSLESS_DEFAULT_BACKEND self.backend = backend if backend is not None else rules.LOSSLESS_DEFAULT_BACKEND
self.compression_level = compression_level if compression_level is not None \ self.compression_level = compression_level if compression_level is not None \
else rules.LOSSLESS_DEFAULT_COMPRESSION_LEVEL else rules.LOSSLESS_DEFAULT_COMPRESSION_LEVEL
self.check_validity() self.check_validity()
# Trying to convert it to a dictionary already here. self._kwargs = self.encoding()
self._kwargs = dict(self.encoding())
def check_validity(self) -> bool: def check_validity(self) -> bool:
if self.backend not in definitions.lossless_backends: if self.backend not in definitions.lossless_backends:
...@@ -164,16 +148,13 @@ class LosslessEncoding(Encoding): ...@@ -164,16 +148,13 @@ class LosslessEncoding(Encoding):
class LossyEncoding(Encoding): class LossyEncoding(Encoding):
def __init__(self, compressor: str, mode: str, parameter: Union[float, int]): def __init__(self, compressor: str, mode: str, parameter: Union[float, int]):
super().__init__()
self.compressor = compressor self.compressor = compressor
self.mode = mode self.mode = mode
self.parameter = parameter self.parameter = parameter
self.check_validity() self.check_validity()
self._kwargs = self.encoding()
# Trying to convert it to a dictionary already here.
self._kwargs = dict(self.encoding())
def check_validity(self): def check_validity(self):
# Check compressor validity # Check compressor validity
...@@ -265,6 +246,48 @@ def parse_variable_specification(var_spec: str) -> Encoding: ...@@ -265,6 +246,48 @@ def parse_variable_specification(var_spec: str) -> Encoding:
raise InvalidCompressionSpecification(f"Invalid specification {var_spec!r}") raise InvalidCompressionSpecification(f"Invalid specification {var_spec!r}")
# class VariableEncoding(_Mapping):
# """
# Class to encapsulate compression specification parameters for a single variable.
#
# It stores the compressor, the mode and the parameter.
#
# It has a method to create a new instance from a specification string,
# a method to get the corresponding specification string from an existing object
# and a method to obtain the corresponding mapping expected by h5py.
#
# """
#
# def __init__(self, specification: Specification):
# # Init basic components
# self.specification = specification
#
# self._kwargs = self.filter_mapping()
#
# @staticmethod
# def from_string(string: str) -> 'VariableEncoding':
# specification = parse_variable_specification(string)
# """
# Method to create a specification object from a specification string
# """
# return VariableEncoding(specification)
#
# def to_string(self) -> str:
# """
# Method to obtain a specification string from a specification object
# """
# return self.specification.to_string()
#
# def filter_mapping(self) -> Mapping:
# """
# Method to get the corresponding FilterRefBase expected by h5py/xarray
# """
#
# return self.specification.encoding()
#
# def description(self):
# self.specification.description()
def get_variable_encoding( def get_variable_encoding(
specification: str = None, specification: str = None,
compressor: str = None, compressor: str = None,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment