Package dynamicio

A package for wrapping your I/O operations.

Expand source code 
"""A package for wrapping your I/O operations."""
import os
from contextlib import suppress

import pkg_resources
from magic_logger import logger

with suppress(Exception):
    __version__ = pkg_resources.get_distribution("dynamicio").version

from dynamicio.core import DynamicDataIO
from dynamicio.mixins import WithKafka, WithLocal, WithLocalBatch, WithPostgres, WithS3File, WithS3PathPrefix

os.environ["LC_CTYPE"] = "en_US.UTF"  # Set your locale to a unicode-compatible one


class UnifiedIO(WithS3File, WithS3PathPrefix, WithLocalBatch, WithLocal, WithKafka, WithPostgres, DynamicDataIO):  # type: ignore
    """A unified io composed of dynamicio.mixins."""


logging_config = {
    "version": 1,
    "disable_existing_loggers": True,
    "formatters": {
        "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"},
        "generic-metrics": {"format": "%(message)s"},
    },
    "handlers": {
        "default": {
            "level": "INFO",
            "formatter": "standard",
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stdout",  # Default is stderr
        },
        "metrics": {
            "level": "INFO",
            "formatter": "generic-metrics",
            "class": "logging.StreamHandler",
            "stream": "ext://sys.stdout",  # Default is stderr
        },
    },
    "loggers": {
        "": {"handlers": ["default"], "level": "INFO", "propagate": False},
        "dynamicio.metrics": {"handlers": ["metrics"], "level": "INFO", "propagate": False},
        "awscli": {
            "handlers": ["default"],
            "level": "INFO",
            "propagate": False,
        },
    },
}

if os.environ.get("DYNAMICIO_DO_NOT_CONFIGURE_LOGGING", "") != "1":
    logger.dict_config(logging_config)

Sub-modules

dynamicio.cli

Implements the dynamicio Command Line Interface (CLI).

dynamicio.config

Dynamicio config file handling routines.

dynamicio.core

Implements the DynamicDataIO class which provides functionality for data: loading; sinking, and; schema validation.

dynamicio.errors

Hosts exception implementations for different errors.

dynamicio.metrics

A module responsible for metrics generation and logging.

dynamicio.mixins

Default dynamicio mixins module

dynamicio.validations

Implements the Validator class responsible for various generic data validations and metrics generation.

Classes

class UnifiedIO (source_config: IOEnvironment, apply_schema_validations: bool = False, log_schema_metrics: bool = False, show_casting_warnings: bool = False, **options: MutableMapping[str, Any])

A unified io composed of dynamicio.mixins.

Class constructor.

Args

source_config
Configuration to use when reading/writing data from/to a source
apply_schema_validations
Applies schema validations on either read() or write()
log_schema_metrics
Logs schema metrics on either read() or write()
show_casting_warnings
Logs casting warnings on either read() or write() if set to True
options
Any additional kwargs that may be used throughout the lifecycle of the object
Expand source code 
class UnifiedIO(WithS3File, WithS3PathPrefix, WithLocalBatch, WithLocal, WithKafka, WithPostgres, DynamicDataIO):  # type: ignore
    """A unified io composed of dynamicio.mixins."""

Ancestors

Class variables

var schema : DataframeSchema
var sources_config : S3DataEnvironment

Inherited members