Skip to content
Snippets Groups Projects
Commit e4518f5f authored by Oriol Tintó's avatar Oriol Tintó
Browse files

Account for the possibility that the conditions can't be fulfilled by a compressor.

parent fe1bc00d
No related branches found
No related tags found
1 merge request!18Several changes including chunking and expansion of examples.
...@@ -18,6 +18,7 @@ from enstools.compression.compressor import drop_variables ...@@ -18,6 +18,7 @@ from enstools.compression.compressor import drop_variables
from enstools.io import read from enstools.io import read
from .analysis_options import AnalysisOptions, AnalysisParameters from .analysis_options import AnalysisOptions, AnalysisParameters
from .analyze_data_array import analyze_data_array, ANALYSIS_DIAGNOSTIC_METRICS, COMPRESSION_RATIO_LABEL from .analyze_data_array import analyze_data_array, ANALYSIS_DIAGNOSTIC_METRICS, COMPRESSION_RATIO_LABEL
from ..errors import ConditionsNotFulfilledError
logger = logging.getLogger("enstools.compression.analysis") logger = logging.getLogger("enstools.compression.analysis")
...@@ -67,7 +68,8 @@ def select_optimal_encoding_based_on_compression_ratio(encodings: dict, metrics: ...@@ -67,7 +68,8 @@ def select_optimal_encoding_based_on_compression_ratio(encodings: dict, metrics:
for variable in variables: for variable in variables:
best_compression_ratio = 0 best_compression_ratio = 0
for combination in combinations: for combination in combinations:
if metrics[combination][variable][COMPRESSION_RATIO_LABEL] > best_compression_ratio: if variable in metrics[combination] and \
metrics[combination][variable][COMPRESSION_RATIO_LABEL] > best_compression_ratio:
best_compression_ratio = metrics[combination][variable][COMPRESSION_RATIO_LABEL] best_compression_ratio = metrics[combination][variable][COMPRESSION_RATIO_LABEL]
best_combination[variable] = combination best_combination[variable] = combination
...@@ -145,18 +147,21 @@ def find_encodings_for_all_combinations(dataset: xarray.Dataset, options: Analys ...@@ -145,18 +147,21 @@ def find_encodings_for_all_combinations(dataset: xarray.Dataset, options: Analys
combination_metrics[var] = {COMPRESSION_RATIO_LABEL: 1.0} combination_metrics[var] = {COMPRESSION_RATIO_LABEL: 1.0}
continue continue
variable_encoding, variable_metrics = analyze_data_array( try:
data_array=dataset[var], variable_encoding, variable_metrics = analyze_data_array(
options=AnalysisOptions(compressor, mode, thresholds=options.thresholds) data_array=dataset[var],
) options=AnalysisOptions(compressor, mode, thresholds=options.thresholds)
combination_encoding[var] = variable_encoding )
combination_metrics[var] = variable_metrics combination_encoding[var] = variable_encoding
# (dataset, variable_name, thresholds, compressor_name, mode) combination_metrics[var] = variable_metrics
logger.debug("%s %s CR:%.1f", # (dataset, variable_name, thresholds, compressor_name, mode)
var, logger.debug("%s %s CR:%.1f",
variable_encoding, var,
variable_metrics[COMPRESSION_RATIO_LABEL], variable_encoding,
) variable_metrics[COMPRESSION_RATIO_LABEL],
)
except ConditionsNotFulfilledError:
...
encodings[combination] = combination_encoding encodings[combination] = combination_encoding
metrics[combination] = combination_metrics metrics[combination] = combination_metrics
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment