Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a property for each LoggerDestination to Logger #2577

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions composer/loggers/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@

if TYPE_CHECKING:
from composer.core import State
from composer.loggers import (CometMLLogger, ConsoleLogger, FileLogger, InMemoryLogger, MLFlowLogger,
MosaicMLLogger, ProgressBarLogger, RemoteUploaderDownloader, SlackLogger,
TensorboardLogger, WandBLogger)
from composer.loggers.logger_destination import LoggerDestination

__all__ = ['LoggerDestination', 'Logger', 'format_log_data_value']
Expand Down Expand Up @@ -147,6 +150,117 @@ def has_file_upload_destination(self) -> bool:
return True
return False

@property
def wandb(self) -> Optional[Sequence['WandBLogger']]:
"""Returns instances of the :class:`~.WandBLogger` destination, if it/they exist.

Returns:
Optional[Sequence[WandBLogger]]: The WandBLogger instance(s), if it/they exist.

"""
from composer.loggers.wandb_logger import WandBLogger
return [destination for destination in self.destinations if isinstance(destination, WandBLogger)]

@property
def mlflow(self) -> Optional[Sequence['MLFlowLogger']]:
"""Returns instances of the :class:`~.MLFlowLogger` destination, if it/they exist.

Returns:
Optional[Sequence[MLFlowLogger]]: The MLFlowLogger instance(s), if it/they exist.
"""
from composer.loggers.mlflow_logger import MLFlowLogger
return [destination for destination in self.destinations if isinstance(destination, MLFlowLogger)]

@property
def remote_uploader_downloader(self) -> Optional[Sequence['RemoteUploaderDownloader']]:
"""Returns instances of the :class:`~.RemoteUploaderDownloader` destination, if it/they exist.

Returns:
Optional[Union[RemoteUploaderDownloader, Sequence[RemoteUploaderDownloader]]]: The RemoteUploaderDownloader instance(s), if it/they exist.
"""
from composer.loggers.remote_uploader_downloader import RemoteUploaderDownloader
return [destination for destination in self.destinations if isinstance(destination, RemoteUploaderDownloader)]

@property
def mosaicml(self) -> Optional[Sequence['MosaicMLLogger']]:
"""Returns instances of the :class:`~.MosaicMLLogger` destination, if it/they exist.

Returns:
Optional[Union[MosaicMLLogger, Sequence[MosaicMLLogger]]]: The MosaicMLLogger instance(s), if it/they exist.
"""
from composer.loggers.mosaicml_logger import MosaicMLLogger
return [destination for destination in self.destinations if isinstance(destination, MosaicMLLogger)]

@property
def slack(self) -> Optional[Sequence['SlackLogger']]:
"""Returns instances of the :class:`~.SlackLogger` destination, if it/they exist.

Returns:
Optional[Union[SlackLogger, Sequence[SlackLogger]]]: The SlackLogger instance(s), if it/they exist.
"""
from composer.loggers.slack_logger import SlackLogger
return [destination for destination in self.destinations if isinstance(destination, SlackLogger)]

@property
def console(self) -> Optional[Sequence['ConsoleLogger']]:
"""Returns instances of the :class:`~.ConsoleLogger` destination, if it/they exist.

Returns:
Optional[Union[ConsoleLogger, Sequence[ConsoleLogger]]]: The ConsoleLogger instance(s), if it/they exist.
"""
from composer.loggers.console_logger import ConsoleLogger
return [destination for destination in self.destinations if isinstance(destination, ConsoleLogger)]

@property
def file(self) -> Optional[Sequence['FileLogger']]:
"""Returns instances of the :class:`~.FileLogger` destination, if it/they exist.

Returns:
Optional[Union[FileLogger, Sequence[FileLogger]]]: The FileLogger instance(s), if it/they exist.
"""
from composer.loggers.file_logger import FileLogger
return [destination for destination in self.destinations if isinstance(destination, FileLogger)]

@property
def in_memory(self) -> Optional[Sequence['InMemoryLogger']]:
"""Returns instances of the :class:`~.InMemoryLogger` destination, if it/they exist.

Returns:
Optional[Union[InMemoryLogger, Sequence[InMemoryLogger]]]: The InMemoryLogger instance(s), if it/they exist.
"""
from composer.loggers.in_memory_logger import InMemoryLogger
return [destination for destination in self.destinations if isinstance(destination, InMemoryLogger)]

@property
def progress_bar(self) -> Optional[Sequence['ProgressBarLogger']]:
"""Returns instances of the :class:`~.ProgressBarLogger` destination, if it/they exist.

Returns:
Optional[Union[ProgressBarLogger, Sequence[ProgressBarLogger]]]: The ProgressBarLogger instance(s), if it/they exist.
"""
from composer.loggers.progress_bar_logger import ProgressBarLogger
return [destination for destination in self.destinations if isinstance(destination, ProgressBarLogger)]

@property
def tensorboard(self) -> Optional[Sequence['TensorboardLogger']]:
"""Returns instances of the :class:`~.TensorboardLogger` destination, if it/they exist.

Returns:
Optional[Union[TensorboardLogger, Sequence[TensorboardLogger]]]: The TensorboardLogger instance(s), if it/they exist.
"""
from composer.loggers.tensorboard_logger import TensorboardLogger
return [destination for destination in self.destinations if isinstance(destination, TensorboardLogger)]

@property
def cometml(self) -> Optional[Sequence['CometMLLogger']]:
"""Returns instances of the :class:`~.CometMLLogger` destination, if it/they exist.

Returns:
Optional[Union[CometMLLogger, Sequence[CometMLLogger]]]: The CometMLLogger instance(s), if it/they exist.
"""
from composer.loggers.cometml_logger import CometMLLogger
return [destination for destination in self.destinations if isinstance(destination, CometMLLogger)]


def format_log_data_value(data: Any) -> str:
"""Recursively formats a given log data value into a string.
Expand Down
34 changes: 33 additions & 1 deletion tests/loggers/test_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,41 @@
# SPDX-License-Identifier: Apache-2.0

import pathlib
from typing import List

import pytest

from composer.core.state import State
from composer.loggers import Logger, LoggerDestination
from composer.loggers import (ConsoleLogger, FileLogger, InMemoryLogger, Logger, LoggerDestination, MLFlowLogger,
MosaicMLLogger, ProgressBarLogger, RemoteUploaderDownloader, SlackLogger,
TensorboardLogger, WandBLogger)


@pytest.mark.parametrize('num_dest_instances_per_type', [0, 1, 2])
def test_logger_properties(dummy_state: State, num_dest_instances_per_type: int):
logger_destination_classes = (ConsoleLogger, FileLogger, InMemoryLogger, MLFlowLogger, MosaicMLLogger,
ProgressBarLogger, SlackLogger, TensorboardLogger, WandBLogger)
destinations = []

for _ in range(num_dest_instances_per_type):
for logger_destination_type_cls in logger_destination_classes:
logger_dest = logger_destination_type_cls()
destinations.append(logger_dest)
destinations.append(RemoteUploaderDownloader(bucket_uri='foo'))
logger = Logger(state=dummy_state, destinations=destinations)
for logger_destination_property_name in [
'wandb', 'mlflow', 'mosaicml', 'tensorboard', 'slack', 'console', 'progress_bar', 'file', 'in_memory',
'remote_uploader_downloader'
]:
assert hasattr(logger, logger_destination_property_name)
if num_dest_instances_per_type == 0:
assert getattr(logger, logger_destination_property_name) is None
else:
assert getattr(logger, logger_destination_property_name) is not None
if num_dest_instances_per_type == 1:
assert not isinstance(getattr(logger, logger_destination_property_name), List)
else:
assert isinstance(getattr(logger, logger_destination_property_name), List)


def test_logger_file_upload(dummy_state: State):
Expand Down
Loading