How to Inspect Function and Class Signatures in Python?#
Motivation#
There are two motivations for why we want to inspect function and class signatures in Python.
Code Introspection in Open Source Projects: When we are working on open source projects, we often need to inspect the function and class signatures of the codebase. This is especially true when we are working on a large codebase with many functions and classes. In this case, we need to inspect the function and class signatures to understand how the codebase is structured and how the functions and classes are used.
Sometimes there are nested abstractions, a child class \(\mathcal{C}_N\) that inherits from a parent class \(\mathcal{C}_{N-1}\), which in turn inherits from another parent class \(\mathcal{C}_{N-2}\), and so on. Sometimes the child class does not immediately show what types of arguments the constructor can take. In this case, we need to inspect the parent classes to understand the constructor signature of the child class.
Agent-Based Function Calling: In agent-based programming, where automated agents are tasked with executing functions or interacting with libraries, the risk of ‘hallucination’—where an agent attempts to invoke non-existent methods or improperly structured calls from a external module/library—is a notable concern. By equipping agents with the capability to query the actual signatures of libraries’ classes or functions, we can significantly mitigate this risk. This ensures that agents operate based on accurate, real-time information, thereby improving the reliability and effectiveness of automated tasks.
Construct Hypothetical Function, Child and Parent Classes#
class ParentClass:
"""This is the parent class."""
parent_class_attr = "a parent class attribute"
def __init__(self, parent_instance_attr: str) -> None:
self.parent_instance_attr = parent_instance_attr
def parent_method(self) -> str:
"""This is a method in the parent class."""
return "Parent method called"
class ChildClass(ParentClass):
"""This is a subclass of ParentClass."""
# Class attribute
class_attr = "a class attribute"
# Private and protected attributes
_protected_attr = "a protected attribute"
__private_attr = "a private attribute"
def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:
"""Initialize the instance."""
super().__init__(parent_instance_attr)
# Instance attribute
self.instance_attr = instance_attr
self.instance_not_in_constructor_attr = "an instance attribute not in the constructor"
self._private_instance_attr = "a private instance attribute"
@property
def read_only_attr(self) -> str:
"""This is a read-only attribute."""
return "You can read me, but you cannot change me."
def instance_method(self, arg: str) -> str:
"""This is an instance method."""
return f"Instance method called with argument: {arg}"
@classmethod
def class_method(cls, arg: str) -> str:
"""This is a class method."""
return f"Class method called with argument: {arg}"
@staticmethod
def static_method(arg: str) -> str:
"""This is a static method."""
return f"Static method called with argument: {arg}"
def __str__(self) -> str:
"""Return a string representation of the instance."""
return f"MyClass(instance_attr={self.instance_attr})"
instance_child = ChildClass(instance_attr="an instance attribute", parent_instance_attr="a parent instance attribute")
class_child = ChildClass
instance_parent = ParentClass(parent_instance_attr="a parent instance attribute")
class_parent = ParentClass
def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:
return a, b, c, d, e, kwargs
Inspect All Members#
@overload
def get_members_of_function_or_method(
func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None
) -> List[Tuple[str, Any]]: ...
@overload
def get_members_of_function_or_method(
func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None
) -> List[Tuple[str, Any]]: ...
def get_members_of_function_or_method(
func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None
) -> List[Tuple[str, Any]]:
return inspect.getmembers(func_or_class, predicate)
def loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:
if filter is not None:
members = [member for member in members if filter in member[0]]
for member in members:
name, value = member
print(f"{name}: {value}")
Our initial goal is to get all signatures and type annotations of a class or function. We can use the inspect module to achieve this. The getmembers function returns all members of a class or module. We can then filter out the functions and classes and inspect their signatures.
However, for our purpose, it may be overkill since it retrieves all members
within a module, the scope is very broad, for example, inspecting just the func
defined will also return all __globals__, which may not be what we want.
func_all_members = get_members_of_function_or_method(func, predicate=None)
loop_through_members(func_all_members)
__annotate__: <function func.__annotate__ at 0x7f86547f6820>
__annotations__: {'a': <class 'int'>, 'b': <class 'str'>, 'c': typing.List[int], 'd': typing.Tuple[str, str], 'e': int | str, 'kwargs': typing.Any, 'return': <class 'str'>}
__builtins__: {'__name__': 'builtins', '__doc__': "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", '__package__': '', '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': ModuleSpec(name='builtins', loader=<class '_frozen_importlib.BuiltinImporter'>, origin='built-in'), '__build_class__': <built-in function __build_class__>, '__import__': <built-in function __import__>, 'abs': <built-in function abs>, 'all': <built-in function all>, 'any': <built-in function any>, 'ascii': <built-in function ascii>, 'bin': <built-in function bin>, 'breakpoint': <built-in function breakpoint>, 'callable': <built-in function callable>, 'chr': <built-in function chr>, 'compile': <built-in function compile>, 'delattr': <built-in function delattr>, 'dir': <built-in function dir>, 'divmod': <built-in function divmod>, 'eval': <built-in function eval>, 'exec': <built-in function exec>, 'format': <built-in function format>, 'getattr': <built-in function getattr>, 'globals': <built-in function globals>, 'hasattr': <built-in function hasattr>, 'hash': <built-in function hash>, 'hex': <built-in function hex>, 'id': <built-in function id>, 'input': <bound method Kernel.raw_input of <ipykernel.ipkernel.IPythonKernel object at 0x7f8687cb5550>>, 'isinstance': <built-in function isinstance>, 'issubclass': <built-in function issubclass>, 'iter': <built-in function iter>, 'aiter': <built-in function aiter>, 'len': <built-in function len>, 'locals': <built-in function locals>, 'max': <built-in function max>, 'min': <built-in function min>, 'next': <built-in function next>, 'anext': <built-in function anext>, 'oct': <built-in function oct>, 'ord': <built-in function ord>, 'pow': <built-in function pow>, 'print': <built-in function print>, 'repr': <built-in function repr>, 'round': <built-in function round>, 'setattr': <built-in function setattr>, 'sorted': <built-in function sorted>, 'sum': <built-in function sum>, 'vars': <built-in function vars>, 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': <class 'bool'>, 'memoryview': <class 'memoryview'>, 'bytearray': <class 'bytearray'>, 'bytes': <class 'bytes'>, 'classmethod': <class 'classmethod'>, 'complex': <class 'complex'>, 'dict': <class 'dict'>, 'enumerate': <class 'enumerate'>, 'filter': <class 'filter'>, 'float': <class 'float'>, 'frozenset': <class 'frozenset'>, 'property': <class 'property'>, 'int': <class 'int'>, 'list': <class 'list'>, 'map': <class 'map'>, 'object': <class 'object'>, 'range': <class 'range'>, 'reversed': <class 'reversed'>, 'set': <class 'set'>, 'slice': <class 'slice'>, 'staticmethod': <class 'staticmethod'>, 'str': <class 'str'>, 'super': <class 'super'>, 'tuple': <class 'tuple'>, 'type': <class 'type'>, 'zip': <class 'zip'>, '__debug__': True, 'BaseException': <class 'BaseException'>, 'BaseExceptionGroup': <class 'BaseExceptionGroup'>, 'Exception': <class 'Exception'>, 'GeneratorExit': <class 'GeneratorExit'>, 'KeyboardInterrupt': <class 'KeyboardInterrupt'>, 'SystemExit': <class 'SystemExit'>, 'ArithmeticError': <class 'ArithmeticError'>, 'AssertionError': <class 'AssertionError'>, 'AttributeError': <class 'AttributeError'>, 'BufferError': <class 'BufferError'>, 'EOFError': <class 'EOFError'>, 'ImportError': <class 'ImportError'>, 'LookupError': <class 'LookupError'>, 'MemoryError': <class 'MemoryError'>, 'NameError': <class 'NameError'>, 'OSError': <class 'OSError'>, 'ReferenceError': <class 'ReferenceError'>, 'RuntimeError': <class 'RuntimeError'>, 'StopAsyncIteration': <class 'StopAsyncIteration'>, 'StopIteration': <class 'StopIteration'>, 'SyntaxError': <class 'SyntaxError'>, 'SystemError': <class 'SystemError'>, 'TypeError': <class 'TypeError'>, 'ValueError': <class 'ValueError'>, 'Warning': <class 'Warning'>, 'FloatingPointError': <class 'FloatingPointError'>, 'OverflowError': <class 'OverflowError'>, 'ZeroDivisionError': <class 'ZeroDivisionError'>, 'BytesWarning': <class 'BytesWarning'>, 'DeprecationWarning': <class 'DeprecationWarning'>, 'EncodingWarning': <class 'EncodingWarning'>, 'FutureWarning': <class 'FutureWarning'>, 'ImportWarning': <class 'ImportWarning'>, 'PendingDeprecationWarning': <class 'PendingDeprecationWarning'>, 'ResourceWarning': <class 'ResourceWarning'>, 'RuntimeWarning': <class 'RuntimeWarning'>, 'SyntaxWarning': <class 'SyntaxWarning'>, 'UnicodeWarning': <class 'UnicodeWarning'>, 'UserWarning': <class 'UserWarning'>, 'BlockingIOError': <class 'BlockingIOError'>, 'ChildProcessError': <class 'ChildProcessError'>, 'ConnectionError': <class 'ConnectionError'>, 'FileExistsError': <class 'FileExistsError'>, 'FileNotFoundError': <class 'FileNotFoundError'>, 'InterruptedError': <class 'InterruptedError'>, 'IsADirectoryError': <class 'IsADirectoryError'>, 'NotADirectoryError': <class 'NotADirectoryError'>, 'PermissionError': <class 'PermissionError'>, 'ProcessLookupError': <class 'ProcessLookupError'>, 'TimeoutError': <class 'TimeoutError'>, 'IndentationError': <class 'IndentationError'>, '_IncompleteInputError': <class '_IncompleteInputError'>, 'IndexError': <class 'IndexError'>, 'KeyError': <class 'KeyError'>, 'ModuleNotFoundError': <class 'ModuleNotFoundError'>, 'NotImplementedError': <class 'NotImplementedError'>, 'PythonFinalizationError': <class 'PythonFinalizationError'>, 'RecursionError': <class 'RecursionError'>, 'UnboundLocalError': <class 'UnboundLocalError'>, 'UnicodeError': <class 'UnicodeError'>, 'BrokenPipeError': <class 'BrokenPipeError'>, 'ConnectionAbortedError': <class 'ConnectionAbortedError'>, 'ConnectionRefusedError': <class 'ConnectionRefusedError'>, 'ConnectionResetError': <class 'ConnectionResetError'>, 'TabError': <class 'TabError'>, 'UnicodeDecodeError': <class 'UnicodeDecodeError'>, 'UnicodeEncodeError': <class 'UnicodeEncodeError'>, 'UnicodeTranslateError': <class 'UnicodeTranslateError'>, 'ExceptionGroup': <class 'ExceptionGroup'>, 'EnvironmentError': <class 'OSError'>, 'IOError': <class 'OSError'>, 'open': <built-in function open>, 'copyright': Copyright (c) 2001 Python Software Foundation.
All Rights Reserved.
Copyright (c) 2000 BeOpen.com.
All Rights Reserved.
Copyright (c) 1995-2001 Corporation for National Research Initiatives.
All Rights Reserved.
Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.
All Rights Reserved., 'credits': Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software
Foundation, and a cast of thousands for supporting Python
development. See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object., 'execfile': <function execfile at 0x7f8684bed220>, 'runfile': <function runfile at 0x7f8684b07690>, '__IPYTHON__': True, 'display': <function display at 0x7f8688d1e820>, 'get_ipython': <bound method InteractiveShell.get_ipython of <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7f86849c74d0>>}
__call__: <method-wrapper '__call__' of function object at 0x7f86547f6560>
__class__: <class 'function'>
__closure__: None
__code__: <code object func at 0x7f85640e10d0, file "/tmp/ipykernel_3325/2139551385.py", line 1>
__defaults__: None
__delattr__: <method-wrapper '__delattr__' of function object at 0x7f86547f6560>
__dict__: {}
__dir__: <built-in method __dir__ of function object at 0x7f86547f6560>
__doc__: None
__eq__: <method-wrapper '__eq__' of function object at 0x7f86547f6560>
__format__: <built-in method __format__ of function object at 0x7f86547f6560>
__ge__: <method-wrapper '__ge__' of function object at 0x7f86547f6560>
__get__: <method-wrapper '__get__' of function object at 0x7f86547f6560>
__getattribute__: <method-wrapper '__getattribute__' of function object at 0x7f86547f6560>
__getstate__: <built-in method __getstate__ of function object at 0x7f86547f6560>
__globals__: {'__name__': '__main__', '__doc__': 'Automatically created module for IPython interactive environment', '__package__': None, '__loader__': None, '__spec__': None, '__builtin__': <module 'builtins' (built-in)>, '__builtins__': <module 'builtins' (built-in)>, '_ih': ['', 'import inspect\nfrom dataclasses import field, make_dataclass\nfrom inspect import Parameter, Signature\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload\n\nfrom pydantic import BaseModel\nfrom rich.pretty import pprint\nfrom transformers import GPT2LMHeadModel, Trainer, TrainingArguments', 'class ParentClass:\n """This is the parent class."""\n\n parent_class_attr = "a parent class attribute"\n\n def __init__(self, parent_instance_attr: str) -> None:\n self.parent_instance_attr = parent_instance_attr\n\n def parent_method(self) -> str:\n """This is a method in the parent class."""\n return "Parent method called"\n\n\nclass ChildClass(ParentClass):\n """This is a subclass of ParentClass."""\n\n # Class attribute\n class_attr = "a class attribute"\n\n # Private and protected attributes\n _protected_attr = "a protected attribute"\n __private_attr = "a private attribute"\n\n def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:\n """Initialize the instance."""\n super().__init__(parent_instance_attr)\n # Instance attribute\n self.instance_attr = instance_attr\n self.instance_not_in_constructor_attr = "an instance attribute not in the constructor"\n self._private_instance_attr = "a private instance attribute"\n\n @property\n def read_only_attr(self) -> str:\n """This is a read-only attribute."""\n return "You can read me, but you cannot change me."\n\n def instance_method(self, arg: str) -> str:\n """This is an instance method."""\n return f"Instance method called with argument: {arg}"\n\n @classmethod\n def class_method(cls, arg: str) -> str:\n """This is a class method."""\n return f"Class method called with argument: {arg}"\n\n @staticmethod\n def static_method(arg: str) -> str:\n """This is a static method."""\n return f"Static method called with argument: {arg}"\n\n def __str__(self) -> str:\n """Return a string representation of the instance."""\n return f"MyClass(instance_attr={self.instance_attr})"', 'instance_child = ChildClass(instance_attr="an instance attribute", parent_instance_attr="a parent instance attribute")\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr="a parent instance attribute")\nclass_parent = ParentClass', 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', '@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f"{name}: {value}")', 'func_all_members = get_members_of_function_or_method(func, predicate=None)\nloop_through_members(func_all_members)'], '_oh': {}, '_dh': [PosixPath('/home/runner/work/omniverse/omniverse/omniverse/playbook')], 'In': ['', 'import inspect\nfrom dataclasses import field, make_dataclass\nfrom inspect import Parameter, Signature\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload\n\nfrom pydantic import BaseModel\nfrom rich.pretty import pprint\nfrom transformers import GPT2LMHeadModel, Trainer, TrainingArguments', 'class ParentClass:\n """This is the parent class."""\n\n parent_class_attr = "a parent class attribute"\n\n def __init__(self, parent_instance_attr: str) -> None:\n self.parent_instance_attr = parent_instance_attr\n\n def parent_method(self) -> str:\n """This is a method in the parent class."""\n return "Parent method called"\n\n\nclass ChildClass(ParentClass):\n """This is a subclass of ParentClass."""\n\n # Class attribute\n class_attr = "a class attribute"\n\n # Private and protected attributes\n _protected_attr = "a protected attribute"\n __private_attr = "a private attribute"\n\n def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:\n """Initialize the instance."""\n super().__init__(parent_instance_attr)\n # Instance attribute\n self.instance_attr = instance_attr\n self.instance_not_in_constructor_attr = "an instance attribute not in the constructor"\n self._private_instance_attr = "a private instance attribute"\n\n @property\n def read_only_attr(self) -> str:\n """This is a read-only attribute."""\n return "You can read me, but you cannot change me."\n\n def instance_method(self, arg: str) -> str:\n """This is an instance method."""\n return f"Instance method called with argument: {arg}"\n\n @classmethod\n def class_method(cls, arg: str) -> str:\n """This is a class method."""\n return f"Class method called with argument: {arg}"\n\n @staticmethod\n def static_method(arg: str) -> str:\n """This is a static method."""\n return f"Static method called with argument: {arg}"\n\n def __str__(self) -> str:\n """Return a string representation of the instance."""\n return f"MyClass(instance_attr={self.instance_attr})"', 'instance_child = ChildClass(instance_attr="an instance attribute", parent_instance_attr="a parent instance attribute")\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr="a parent instance attribute")\nclass_parent = ParentClass', 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', '@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f"{name}: {value}")', 'func_all_members = get_members_of_function_or_method(func, predicate=None)\nloop_through_members(func_all_members)'], 'Out': {}, 'get_ipython': <bound method InteractiveShell.get_ipython of <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7f86849c74d0>>, 'exit': <IPython.core.autocall.ZMQExitAutocall object at 0x7f86847a1160>, 'quit': <IPython.core.autocall.ZMQExitAutocall object at 0x7f86847a1160>, 'open': <function open at 0x7f8688b7f690>, '_': '', '__': '', '___': '', '_i': '@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f"{name}: {value}")', '_ii': 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', '_iii': 'instance_child = ChildClass(instance_attr="an instance attribute", parent_instance_attr="a parent instance attribute")\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr="a parent instance attribute")\nclass_parent = ParentClass', '_i1': 'import inspect\nfrom dataclasses import field, make_dataclass\nfrom inspect import Parameter, Signature\nfrom typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union, _GenericAlias, get_type_hints, overload\n\nfrom pydantic import BaseModel\nfrom rich.pretty import pprint\nfrom transformers import GPT2LMHeadModel, Trainer, TrainingArguments', 'inspect': <module 'inspect' from '/home/runner/.local/share/uv/python/cpython-3.14.6-linux-x86_64-gnu/lib/python3.14/inspect.py'>, 'field': <function field at 0x7f8689e497a0>, 'make_dataclass': <function make_dataclass at 0x7f8689934720>, 'Parameter': <class 'inspect.Parameter'>, 'Signature': <class 'inspect.Signature'>, 'Any': typing.Any, 'Callable': typing.Callable, 'Dict': typing.Dict, 'List': typing.List, 'Optional': typing.Optional, 'Set': typing.Set, 'Tuple': typing.Tuple, 'Type': typing.Type, 'Union': <class 'typing.Union'>, '_GenericAlias': <class 'typing._GenericAlias'>, 'get_type_hints': <function get_type_hints at 0x7f868a2997a0>, 'overload': <function overload at 0x7f868a299f30>, 'BaseModel': <class 'pydantic.main.BaseModel'>, 'pprint': <function pprint at 0x7f867ced9b10>, 'GPT2LMHeadModel': <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, 'Trainer': <class 'transformers.trainer.Trainer'>, 'TrainingArguments': <class 'transformers.training_args.TrainingArguments'>, '_i2': 'class ParentClass:\n """This is the parent class."""\n\n parent_class_attr = "a parent class attribute"\n\n def __init__(self, parent_instance_attr: str) -> None:\n self.parent_instance_attr = parent_instance_attr\n\n def parent_method(self) -> str:\n """This is a method in the parent class."""\n return "Parent method called"\n\n\nclass ChildClass(ParentClass):\n """This is a subclass of ParentClass."""\n\n # Class attribute\n class_attr = "a class attribute"\n\n # Private and protected attributes\n _protected_attr = "a protected attribute"\n __private_attr = "a private attribute"\n\n def __init__(self, instance_attr: str, parent_instance_attr: str) -> None:\n """Initialize the instance."""\n super().__init__(parent_instance_attr)\n # Instance attribute\n self.instance_attr = instance_attr\n self.instance_not_in_constructor_attr = "an instance attribute not in the constructor"\n self._private_instance_attr = "a private instance attribute"\n\n @property\n def read_only_attr(self) -> str:\n """This is a read-only attribute."""\n return "You can read me, but you cannot change me."\n\n def instance_method(self, arg: str) -> str:\n """This is an instance method."""\n return f"Instance method called with argument: {arg}"\n\n @classmethod\n def class_method(cls, arg: str) -> str:\n """This is a class method."""\n return f"Class method called with argument: {arg}"\n\n @staticmethod\n def static_method(arg: str) -> str:\n """This is a static method."""\n return f"Static method called with argument: {arg}"\n\n def __str__(self) -> str:\n """Return a string representation of the instance."""\n return f"MyClass(instance_attr={self.instance_attr})"', 'ParentClass': <class '__main__.ParentClass'>, 'ChildClass': <class '__main__.ChildClass'>, '_i3': 'instance_child = ChildClass(instance_attr="an instance attribute", parent_instance_attr="a parent instance attribute")\nclass_child = ChildClass\n\ninstance_parent = ParentClass(parent_instance_attr="a parent instance attribute")\nclass_parent = ParentClass', 'instance_child': <__main__.ChildClass object at 0x7f856403cd70>, 'class_child': <class '__main__.ChildClass'>, 'instance_parent': <__main__.ParentClass object at 0x7f856403cec0>, 'class_parent': <class '__main__.ParentClass'>, '_i4': 'def func(a: int, b: str, c: List[int], d: Tuple[str, str], e: Union[int, str], **kwargs: Any) -> str:\n return a, b, c, d, e, kwargs', 'func': <function func at 0x7f86547f6560>, '_i5': '@overload\ndef get_members_of_function_or_method(\n func_or_class: Type[object], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\n@overload\ndef get_members_of_function_or_method(\n func_or_class: Callable[..., Any], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]: ...\n\n\ndef get_members_of_function_or_method(\n func_or_class: Union[Type[object], Callable[..., Any]], predicate: Optional[Callable[[Any], bool]] = None\n) -> List[Tuple[str, Any]]:\n return inspect.getmembers(func_or_class, predicate)\n\n\ndef loop_through_members(members: List[Tuple[str, Any]], filter: Optional[str] = None) -> None:\n if filter is not None:\n members = [member for member in members if filter in member[0]]\n for member in members:\n name, value = member\n print(f"{name}: {value}")', 'get_members_of_function_or_method': <function get_members_of_function_or_method at 0x7f86547f6400>, 'loop_through_members': <function loop_through_members at 0x7f86547f62a0>, '_i6': 'func_all_members = get_members_of_function_or_method(func, predicate=None)\nloop_through_members(func_all_members)', 'func_all_members': [('__annotate__', <function func.__annotate__ at 0x7f86547f6820>), ('__annotations__', {'a': <class 'int'>, 'b': <class 'str'>, 'c': typing.List[int], 'd': typing.Tuple[str, str], 'e': int | str, 'kwargs': typing.Any, 'return': <class 'str'>}), ('__builtins__', {'__name__': 'builtins', '__doc__': "Built-in functions, types, exceptions, and other objects.\n\nThis module provides direct access to all 'built-in'\nidentifiers of Python; for example, builtins.len is\nthe full name for the built-in function len().\n\nThis module is not normally accessed explicitly by most\napplications, but can be useful in modules that provide\nobjects with the same name as a built-in value, but in\nwhich the built-in of that name is also needed.", '__package__': '', '__loader__': <class '_frozen_importlib.BuiltinImporter'>, '__spec__': ModuleSpec(name='builtins', loader=<class '_frozen_importlib.BuiltinImporter'>, origin='built-in'), '__build_class__': <built-in function __build_class__>, '__import__': <built-in function __import__>, 'abs': <built-in function abs>, 'all': <built-in function all>, 'any': <built-in function any>, 'ascii': <built-in function ascii>, 'bin': <built-in function bin>, 'breakpoint': <built-in function breakpoint>, 'callable': <built-in function callable>, 'chr': <built-in function chr>, 'compile': <built-in function compile>, 'delattr': <built-in function delattr>, 'dir': <built-in function dir>, 'divmod': <built-in function divmod>, 'eval': <built-in function eval>, 'exec': <built-in function exec>, 'format': <built-in function format>, 'getattr': <built-in function getattr>, 'globals': <built-in function globals>, 'hasattr': <built-in function hasattr>, 'hash': <built-in function hash>, 'hex': <built-in function hex>, 'id': <built-in function id>, 'input': <bound method Kernel.raw_input of <ipykernel.ipkernel.IPythonKernel object at 0x7f8687cb5550>>, 'isinstance': <built-in function isinstance>, 'issubclass': <built-in function issubclass>, 'iter': <built-in function iter>, 'aiter': <built-in function aiter>, 'len': <built-in function len>, 'locals': <built-in function locals>, 'max': <built-in function max>, 'min': <built-in function min>, 'next': <built-in function next>, 'anext': <built-in function anext>, 'oct': <built-in function oct>, 'ord': <built-in function ord>, 'pow': <built-in function pow>, 'print': <built-in function print>, 'repr': <built-in function repr>, 'round': <built-in function round>, 'setattr': <built-in function setattr>, 'sorted': <built-in function sorted>, 'sum': <built-in function sum>, 'vars': <built-in function vars>, 'None': None, 'Ellipsis': Ellipsis, 'NotImplemented': NotImplemented, 'False': False, 'True': True, 'bool': <class 'bool'>, 'memoryview': <class 'memoryview'>, 'bytearray': <class 'bytearray'>, 'bytes': <class 'bytes'>, 'classmethod': <class 'classmethod'>, 'complex': <class 'complex'>, 'dict': <class 'dict'>, 'enumerate': <class 'enumerate'>, 'filter': <class 'filter'>, 'float': <class 'float'>, 'frozenset': <class 'frozenset'>, 'property': <class 'property'>, 'int': <class 'int'>, 'list': <class 'list'>, 'map': <class 'map'>, 'object': <class 'object'>, 'range': <class 'range'>, 'reversed': <class 'reversed'>, 'set': <class 'set'>, 'slice': <class 'slice'>, 'staticmethod': <class 'staticmethod'>, 'str': <class 'str'>, 'super': <class 'super'>, 'tuple': <class 'tuple'>, 'type': <class 'type'>, 'zip': <class 'zip'>, '__debug__': True, 'BaseException': <class 'BaseException'>, 'BaseExceptionGroup': <class 'BaseExceptionGroup'>, 'Exception': <class 'Exception'>, 'GeneratorExit': <class 'GeneratorExit'>, 'KeyboardInterrupt': <class 'KeyboardInterrupt'>, 'SystemExit': <class 'SystemExit'>, 'ArithmeticError': <class 'ArithmeticError'>, 'AssertionError': <class 'AssertionError'>, 'AttributeError': <class 'AttributeError'>, 'BufferError': <class 'BufferError'>, 'EOFError': <class 'EOFError'>, 'ImportError': <class 'ImportError'>, 'LookupError': <class 'LookupError'>, 'MemoryError': <class 'MemoryError'>, 'NameError': <class 'NameError'>, 'OSError': <class 'OSError'>, 'ReferenceError': <class 'ReferenceError'>, 'RuntimeError': <class 'RuntimeError'>, 'StopAsyncIteration': <class 'StopAsyncIteration'>, 'StopIteration': <class 'StopIteration'>, 'SyntaxError': <class 'SyntaxError'>, 'SystemError': <class 'SystemError'>, 'TypeError': <class 'TypeError'>, 'ValueError': <class 'ValueError'>, 'Warning': <class 'Warning'>, 'FloatingPointError': <class 'FloatingPointError'>, 'OverflowError': <class 'OverflowError'>, 'ZeroDivisionError': <class 'ZeroDivisionError'>, 'BytesWarning': <class 'BytesWarning'>, 'DeprecationWarning': <class 'DeprecationWarning'>, 'EncodingWarning': <class 'EncodingWarning'>, 'FutureWarning': <class 'FutureWarning'>, 'ImportWarning': <class 'ImportWarning'>, 'PendingDeprecationWarning': <class 'PendingDeprecationWarning'>, 'ResourceWarning': <class 'ResourceWarning'>, 'RuntimeWarning': <class 'RuntimeWarning'>, 'SyntaxWarning': <class 'SyntaxWarning'>, 'UnicodeWarning': <class 'UnicodeWarning'>, 'UserWarning': <class 'UserWarning'>, 'BlockingIOError': <class 'BlockingIOError'>, 'ChildProcessError': <class 'ChildProcessError'>, 'ConnectionError': <class 'ConnectionError'>, 'FileExistsError': <class 'FileExistsError'>, 'FileNotFoundError': <class 'FileNotFoundError'>, 'InterruptedError': <class 'InterruptedError'>, 'IsADirectoryError': <class 'IsADirectoryError'>, 'NotADirectoryError': <class 'NotADirectoryError'>, 'PermissionError': <class 'PermissionError'>, 'ProcessLookupError': <class 'ProcessLookupError'>, 'TimeoutError': <class 'TimeoutError'>, 'IndentationError': <class 'IndentationError'>, '_IncompleteInputError': <class '_IncompleteInputError'>, 'IndexError': <class 'IndexError'>, 'KeyError': <class 'KeyError'>, 'ModuleNotFoundError': <class 'ModuleNotFoundError'>, 'NotImplementedError': <class 'NotImplementedError'>, 'PythonFinalizationError': <class 'PythonFinalizationError'>, 'RecursionError': <class 'RecursionError'>, 'UnboundLocalError': <class 'UnboundLocalError'>, 'UnicodeError': <class 'UnicodeError'>, 'BrokenPipeError': <class 'BrokenPipeError'>, 'ConnectionAbortedError': <class 'ConnectionAbortedError'>, 'ConnectionRefusedError': <class 'ConnectionRefusedError'>, 'ConnectionResetError': <class 'ConnectionResetError'>, 'TabError': <class 'TabError'>, 'UnicodeDecodeError': <class 'UnicodeDecodeError'>, 'UnicodeEncodeError': <class 'UnicodeEncodeError'>, 'UnicodeTranslateError': <class 'UnicodeTranslateError'>, 'ExceptionGroup': <class 'ExceptionGroup'>, 'EnvironmentError': <class 'OSError'>, 'IOError': <class 'OSError'>, 'open': <built-in function open>, 'copyright': Copyright (c) 2001 Python Software Foundation.
All Rights Reserved.
Copyright (c) 2000 BeOpen.com.
All Rights Reserved.
Copyright (c) 1995-2001 Corporation for National Research Initiatives.
All Rights Reserved.
Copyright (c) 1991-1995 Stichting Mathematisch Centrum, Amsterdam.
All Rights Reserved., 'credits': Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software
Foundation, and a cast of thousands for supporting Python
development. See www.python.org for more information., 'license': Type license() to see the full license text, 'help': Type help() for interactive help, or help(object) for help about object., 'execfile': <function execfile at 0x7f8684bed220>, 'runfile': <function runfile at 0x7f8684b07690>, '__IPYTHON__': True, 'display': <function display at 0x7f8688d1e820>, 'get_ipython': <bound method InteractiveShell.get_ipython of <ipykernel.zmqshell.ZMQInteractiveShell object at 0x7f86849c74d0>>}), ('__call__', <method-wrapper '__call__' of function object at 0x7f86547f6560>), ('__class__', <class 'function'>), ('__closure__', None), ('__code__', <code object func at 0x7f85640e10d0, file "/tmp/ipykernel_3325/2139551385.py", line 1>), ('__defaults__', None), ('__delattr__', <method-wrapper '__delattr__' of function object at 0x7f86547f6560>), ('__dict__', {}), ('__dir__', <built-in method __dir__ of function object at 0x7f86547f6560>), ('__doc__', None), ('__eq__', <method-wrapper '__eq__' of function object at 0x7f86547f6560>), ('__format__', <built-in method __format__ of function object at 0x7f86547f6560>), ('__ge__', <method-wrapper '__ge__' of function object at 0x7f86547f6560>), ('__get__', <method-wrapper '__get__' of function object at 0x7f86547f6560>), ('__getattribute__', <method-wrapper '__getattribute__' of function object at 0x7f86547f6560>), ('__getstate__', <built-in method __getstate__ of function object at 0x7f86547f6560>), ('__globals__', {...}), ('__gt__', <method-wrapper '__gt__' of function object at 0x7f86547f6560>), ('__hash__', <method-wrapper '__hash__' of function object at 0x7f86547f6560>), ('__init__', <method-wrapper '__init__' of function object at 0x7f86547f6560>), ('__init_subclass__', <built-in method __init_subclass__ of type object at 0x15a7bb8>), ('__kwdefaults__', None), ('__le__', <method-wrapper '__le__' of function object at 0x7f86547f6560>), ('__lt__', <method-wrapper '__lt__' of function object at 0x7f86547f6560>), ('__module__', '__main__'), ('__name__', 'func'), ('__ne__', <method-wrapper '__ne__' of function object at 0x7f86547f6560>), ('__new__', <built-in method __new__ of type object at 0x15a7bb8>), ('__qualname__', 'func'), ('__reduce__', <built-in method __reduce__ of function object at 0x7f86547f6560>), ('__reduce_ex__', <built-in method __reduce_ex__ of function object at 0x7f86547f6560>), ('__repr__', <method-wrapper '__repr__' of function object at 0x7f86547f6560>), ('__setattr__', <method-wrapper '__setattr__' of function object at 0x7f86547f6560>), ('__sizeof__', <built-in method __sizeof__ of function object at 0x7f86547f6560>), ('__str__', <method-wrapper '__str__' of function object at 0x7f86547f6560>), ('__subclasshook__', <built-in method __subclasshook__ of type object at 0x15a7bb8>), ('__type_params__', ())]}
__gt__: <method-wrapper '__gt__' of function object at 0x7f86547f6560>
__hash__: <method-wrapper '__hash__' of function object at 0x7f86547f6560>
__init__: <method-wrapper '__init__' of function object at 0x7f86547f6560>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x15a7bb8>
__kwdefaults__: None
__le__: <method-wrapper '__le__' of function object at 0x7f86547f6560>
__lt__: <method-wrapper '__lt__' of function object at 0x7f86547f6560>
__module__: __main__
__name__: func
__ne__: <method-wrapper '__ne__' of function object at 0x7f86547f6560>
__new__: <built-in method __new__ of type object at 0x15a7bb8>
__qualname__: func
__reduce__: <built-in method __reduce__ of function object at 0x7f86547f6560>
__reduce_ex__: <built-in method __reduce_ex__ of function object at 0x7f86547f6560>
__repr__: <method-wrapper '__repr__' of function object at 0x7f86547f6560>
__setattr__: <method-wrapper '__setattr__' of function object at 0x7f86547f6560>
__sizeof__: <built-in method __sizeof__ of function object at 0x7f86547f6560>
__str__: <method-wrapper '__str__' of function object at 0x7f86547f6560>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x15a7bb8>
__type_params__: ()
And to get the signature, we can just filter '__annotations__'.
loop_through_members(func_all_members, filter="__annotations__")
__annotations__: {'a': <class 'int'>, 'b': <class 'str'>, 'c': typing.List[int], 'd': typing.Tuple[str, str], 'e': int | str, 'kwargs': typing.Any, 'return': <class 'str'>}
class_child_all_members = get_members_of_function_or_method(class_child, predicate=None)
loop_through_members(class_child_all_members)
_ChildClass__private_attr: a private attribute
__class__: <class 'type'>
__delattr__: <slot wrapper '__delattr__' of 'object' objects>
__dict__: {'__module__': '__main__', '__firstlineno__': 14, '__doc__': 'This is a subclass of ParentClass.', 'class_attr': 'a class attribute', '_protected_attr': 'a protected attribute', '_ChildClass__private_attr': 'a private attribute', '__init__': <function ChildClass.__init__ at 0x7f86547f7110>, 'read_only_attr': <property object at 0x7f8654727dd0>, 'instance_method': <function ChildClass.instance_method at 0x7f86547f6e50>, 'class_method': <classmethod(<function ChildClass.class_method at 0x7f86547f6cf0>)>, 'static_method': <staticmethod(<function ChildClass.static_method at 0x7f86547f6980>)>, '__str__': <function ChildClass.__str__ at 0x7f86547f6a30>, '__static_attributes__': ('_private_instance_attr', 'instance_attr', 'instance_not_in_constructor_attr')}
__dir__: <method '__dir__' of 'object' objects>
__doc__: This is a subclass of ParentClass.
__eq__: <slot wrapper '__eq__' of 'object' objects>
__firstlineno__: 14
__format__: <method '__format__' of 'object' objects>
__ge__: <slot wrapper '__ge__' of 'object' objects>
__getattribute__: <slot wrapper '__getattribute__' of 'object' objects>
__getstate__: <method '__getstate__' of 'object' objects>
__gt__: <slot wrapper '__gt__' of 'object' objects>
__hash__: <slot wrapper '__hash__' of 'object' objects>
__init__: <function ChildClass.__init__ at 0x7f86547f7110>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x3d40d5b0>
__le__: <slot wrapper '__le__' of 'object' objects>
__lt__: <slot wrapper '__lt__' of 'object' objects>
__module__: __main__
__ne__: <slot wrapper '__ne__' of 'object' objects>
__new__: <built-in method __new__ of type object at 0x15b3a00>
__reduce__: <method '__reduce__' of 'object' objects>
__reduce_ex__: <method '__reduce_ex__' of 'object' objects>
__repr__: <slot wrapper '__repr__' of 'object' objects>
__setattr__: <slot wrapper '__setattr__' of 'object' objects>
__sizeof__: <method '__sizeof__' of 'object' objects>
__static_attributes__: ('_private_instance_attr', 'instance_attr', 'instance_not_in_constructor_attr')
__str__: <function ChildClass.__str__ at 0x7f86547f6a30>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x3d40d5b0>
__weakref__: <attribute '__weakref__' of 'ParentClass' objects>
_protected_attr: a protected attribute
class_attr: a class attribute
class_method: <bound method ChildClass.class_method of <class '__main__.ChildClass'>>
instance_method: <function ChildClass.instance_method at 0x7f86547f6e50>
parent_class_attr: a parent class attribute
parent_method: <function ParentClass.parent_method at 0x7f86547f7270>
read_only_attr: <property object at 0x7f8654727dd0>
static_method: <function ChildClass.static_method at 0x7f86547f6980>
instance_child_all_members = get_members_of_function_or_method(instance_child, predicate=None)
loop_through_members(instance_child_all_members)
_ChildClass__private_attr: a private attribute
__class__: <class '__main__.ChildClass'>
__delattr__: <method-wrapper '__delattr__' of ChildClass object at 0x7f856403cd70>
__dict__: {'parent_instance_attr': 'a parent instance attribute', 'instance_attr': 'an instance attribute', 'instance_not_in_constructor_attr': 'an instance attribute not in the constructor', '_private_instance_attr': 'a private instance attribute'}
__dir__: <built-in method __dir__ of ChildClass object at 0x7f856403cd70>
__doc__: This is a subclass of ParentClass.
__eq__: <method-wrapper '__eq__' of ChildClass object at 0x7f856403cd70>
__firstlineno__: 14
__format__: <built-in method __format__ of ChildClass object at 0x7f856403cd70>
__ge__: <method-wrapper '__ge__' of ChildClass object at 0x7f856403cd70>
__getattribute__: <method-wrapper '__getattribute__' of ChildClass object at 0x7f856403cd70>
__getstate__: <built-in method __getstate__ of ChildClass object at 0x7f856403cd70>
__gt__: <method-wrapper '__gt__' of ChildClass object at 0x7f856403cd70>
__hash__: <method-wrapper '__hash__' of ChildClass object at 0x7f856403cd70>
__init__: <bound method ChildClass.__init__ of <__main__.ChildClass object at 0x7f856403cd70>>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x3d40d5b0>
__le__: <method-wrapper '__le__' of ChildClass object at 0x7f856403cd70>
__lt__: <method-wrapper '__lt__' of ChildClass object at 0x7f856403cd70>
__module__: __main__
__ne__: <method-wrapper '__ne__' of ChildClass object at 0x7f856403cd70>
__new__: <built-in method __new__ of type object at 0x15b3a00>
__reduce__: <built-in method __reduce__ of ChildClass object at 0x7f856403cd70>
__reduce_ex__: <built-in method __reduce_ex__ of ChildClass object at 0x7f856403cd70>
__repr__: <method-wrapper '__repr__' of ChildClass object at 0x7f856403cd70>
__setattr__: <method-wrapper '__setattr__' of ChildClass object at 0x7f856403cd70>
__sizeof__: <built-in method __sizeof__ of ChildClass object at 0x7f856403cd70>
__static_attributes__: ('_private_instance_attr', 'instance_attr', 'instance_not_in_constructor_attr')
__str__: <bound method ChildClass.__str__ of <__main__.ChildClass object at 0x7f856403cd70>>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x3d40d5b0>
__weakref__: None
_private_instance_attr: a private instance attribute
_protected_attr: a protected attribute
class_attr: a class attribute
class_method: <bound method ChildClass.class_method of <class '__main__.ChildClass'>>
instance_attr: an instance attribute
instance_method: <bound method ChildClass.instance_method of <__main__.ChildClass object at 0x7f856403cd70>>
instance_not_in_constructor_attr: an instance attribute not in the constructor
parent_class_attr: a parent class attribute
parent_instance_attr: a parent instance attribute
parent_method: <bound method ParentClass.parent_method of <__main__.ChildClass object at 0x7f856403cd70>>
read_only_attr: You can read me, but you cannot change me.
static_method: <function ChildClass.static_method at 0x7f86547f6980>
trainer_all_members = get_members_of_function_or_method(Trainer, predicate=None)
loop_through_members(trainer_all_members)
__backends: ['torch', 'accelerate']
__class__: <class 'type'>
__delattr__: <slot wrapper '__delattr__' of 'object' objects>
__dict__: {'__module__': 'transformers.trainer', '__firstlineno__': 251, '__doc__': "\nTrainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers.\n\nArgs:\n model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*):\n The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed.\n\n <Tip>\n\n [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use\n your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers\n models.\n\n </Tip>\n\n args ([`TrainingArguments`], *optional*):\n The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the\n `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided.\n data_collator (`DataCollator`, *optional*):\n The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will\n default to [`default_data_collator`] if no `processing_class` is provided, an instance of\n [`DataCollatorWithPadding`] otherwise if the processing_class is a feature extractor or tokenizer.\n train_dataset (`torch.utils.data.Dataset` | `torch.utils.data.IterableDataset` | `datasets.Dataset`, *optional*):\n The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the\n `model.forward()` method are automatically removed.\n\n Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a\n distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a\n `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will\n manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally\n sets the seed of the RNGs used.\n eval_dataset (`torch.utils.data.Dataset` | dict[str, `torch.utils.data.Dataset`] | `datasets.Dataset`, *optional*):\n The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the\n `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each\n dataset prepending the dictionary key to the metric name.\n processing_class (`PreTrainedTokenizerBase` or `BaseImageProcessor` or `FeatureExtractionMixin` or `ProcessorMixin`, *optional*):\n Processing class used to process the data. If provided, will be used to automatically process the inputs\n for the model, and it will be saved along the model to make it easier to rerun an interrupted training or\n reuse the fine-tuned model.\n model_init (`Callable[[], PreTrainedModel]`, *optional*):\n A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start\n from a new instance of the model as given by this function.\n\n The function may have zero argument, or a single one containing the optuna/Ray Tune trial object, to\n be able to choose different architectures according to hyperparameters (such as layer count, sizes of\n inner layers, dropout probabilities etc).\n compute_loss_func (`Callable`, *optional*):\n A function that accepts the raw model outputs, labels, and the number of items in the entire accumulated\n batch (batch_size * gradient_accumulation_steps) and returns the loss. For example, see the default [loss function](https://github.com/huggingface/transformers/blob/052e652d6d53c2b26ffde87e039b723949a53493/src/transformers/trainer.py#L3618) used by [`Trainer`].\n compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):\n The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return\n a dictionary string to metric values. *Note* When passing TrainingArgs with `batch_eval_metrics` set to\n `True`, your compute_metrics function must take a boolean `compute_result` argument. This will be triggered\n after the last eval batch to signal that the function needs to calculate and return the global summary\n statistics rather than accumulating the batch-level statistics\n callbacks (List of [`TrainerCallback`], *optional*):\n A list of callbacks to customize the training loop. Will add those to the list of default callbacks\n detailed in [here](callback).\n\n If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method.\n optimizers (`tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`):\n A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your\n model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.\n optimizer_cls_and_kwargs (`tuple[Type[torch.optim.Optimizer], dict[str, Any]]`, *optional*):\n A tuple containing the optimizer class and keyword arguments to use.\n Overrides `optim` and `optim_args` in `args`. Incompatible with the `optimizers` argument.\n\n Unlike `optimizers`, this argument avoids the need to place model parameters on the correct devices before initializing the Trainer.\n preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*):\n A function that preprocess the logits right before caching them at each evaluation step. Must take two\n tensors, the logits and the labels, and return the logits once processed as desired. The modifications made\n by this function will be reflected in the predictions received by `compute_metrics`.\n\n Note that the labels (second parameter) will be `None` if the dataset does not have them.\n\nImportant attributes:\n\n - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`]\n subclass.\n - **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the\n original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`,\n the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner\n model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.\n - **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from\n data parallelism, this means some of the model layers are split on different GPUs).\n - **place_model_on_device** -- Whether or not to automatically place the model on the device. Defaults to\n `True` unless model parallel, DeepSpeed, FSDP, full fp16/bf16 eval, or SageMaker MP is active. Can be\n overridden by subclassing `TrainingArguments` and overriding the `place_model_on_device` property.\n - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while\n in `train`)\n\n", 'get_learning_rates': <function get_learning_rates at 0x7f85650c1d20>, 'get_num_trainable_parameters': <function get_num_trainable_parameters at 0x7f85650c1bc0>, 'get_optimizer_group': <function get_optimizer_group at 0x7f85650c1e80>, 'log_metrics': <function log_metrics at 0x7f85650c1900>, 'metrics_format': <function metrics_format at 0x7f85650c1850>, 'save_metrics': <function save_metrics at 0x7f85650c19b0>, 'save_state': <function save_state at 0x7f85650c1a60>, '__init__': <function Trainer.__init__ at 0x7f8564299430>, '_validate_args': <function Trainer._validate_args at 0x7f85641957a0>, '_build_accelerator_args': <function Trainer._build_accelerator_args at 0x7f8564195900>, 'create_accelerator_and_postprocess': <function Trainer.create_accelerator_and_postprocess at 0x7f8564195a60>, 'get_train_dataloader': <function Trainer.get_train_dataloader at 0x7f8564195bc0>, 'get_eval_dataloader': <function Trainer.get_eval_dataloader at 0x7f8564195d20>, 'get_test_dataloader': <function Trainer.get_test_dataloader at 0x7f8564195e80>, 'num_examples': <function Trainer.num_examples at 0x7f8564195fe0>, '_get_dataloader': <function Trainer._get_dataloader at 0x7f8564196140>, '_get_train_sampler': <function Trainer._get_train_sampler at 0x7f85641962a0>, '_get_eval_sampler': <function Trainer._get_eval_sampler at 0x7f8564196400>, '_set_signature_columns_if_needed': <function Trainer._set_signature_columns_if_needed at 0x7f8564196560>, '_remove_unused_columns': <function Trainer._remove_unused_columns at 0x7f85641966c0>, '_get_collator_with_removed_columns': <function Trainer._get_collator_with_removed_columns at 0x7f8564196820>, 'create_optimizer_and_scheduler': <function Trainer.create_optimizer_and_scheduler at 0x7f8564196980>, 'create_optimizer': <function Trainer.create_optimizer at 0x7f8564196ae0>, 'create_scheduler': <function Trainer.create_scheduler at 0x7f8564196c40>, 'get_optimizer_cls_and_kwargs': <staticmethod(<function Trainer.get_optimizer_cls_and_kwargs at 0x7f8564196da0>)>, 'get_decay_parameter_names': <function Trainer.get_decay_parameter_names at 0x7f8564103270>, '_get_learning_rate': <function Trainer._get_learning_rate at 0x7f8564102770>, 'train': <function Trainer.train at 0x7f8564196f00>, '_inner_training_loop': <function Trainer._inner_training_loop at 0x7f8564197060>, '_init_training_state': <function Trainer._init_training_state at 0x7f85641971c0>, '_prepare_for_training': <function Trainer._prepare_for_training at 0x7f8564197270>, '_run_epoch': <function Trainer._run_epoch at 0x7f8564197320>, '_finalize_training': <function Trainer._finalize_training at 0x7f85641973d0>, 'training_step': <function Trainer.training_step at 0x7f8564197530>, 'compute_loss': <function Trainer.compute_loss at 0x7f8564197690>, 'compute_loss_context_manager': <function Trainer.compute_loss_context_manager at 0x7f85641977f0>, 'autocast_smart_context_manager': <function Trainer.autocast_smart_context_manager at 0x7f8564197950>, '_maybe_log_save_evaluate': <function Trainer._maybe_log_save_evaluate at 0x7f8564197ab0>, 'get_batch_samples': <function Trainer.get_batch_samples at 0x7f8564197c10>, '_get_num_items_in_batch': <function Trainer._get_num_items_in_batch at 0x7f8564197d70>, '_prepare_input': <function Trainer._prepare_input at 0x7f8564197ed0>, '_prepare_inputs': <function Trainer._prepare_inputs at 0x7f85642100f0>, '_prepare_context_parallel_inputs': <function Trainer._prepare_context_parallel_inputs at 0x7f8564210250>, 'set_initial_training_values': <function Trainer.set_initial_training_values at 0x7f85642103b0>, 'get_total_train_batch_size': <function Trainer.get_total_train_batch_size at 0x7f8564210510>, 'get_sp_size': <function Trainer.get_sp_size at 0x7f8564210670>, 'get_cp_size': <function Trainer.get_cp_size at 0x7f85642107d0>, 'get_tp_size': <function Trainer.get_tp_size at 0x7f8564210930>, '_wrap_model': <function Trainer._wrap_model at 0x7f8564210a90>, '_update_auto_batch_size': <function Trainer._update_auto_batch_size at 0x7f8564210b40>, '_track_num_input_tokens': <function Trainer._track_num_input_tokens at 0x7f8564210bf0>, '_clip_grad_norm': <function Trainer._clip_grad_norm at 0x7f8564210ca0>, '_get_grad_norm': <function Trainer._get_grad_norm at 0x7f8564210d50>, 'evaluate': <function Trainer.evaluate at 0x7f8564210eb0>, 'evaluation_loop': <function Trainer.evaluation_loop at 0x7f8564211010>, 'predict': <function Trainer.predict at 0x7f8564211170>, 'prediction_step': <function Trainer.prediction_step at 0x7f85642112d0>, '_evaluate': <function Trainer._evaluate at 0x7f8564211430>, '_get_output_dir': <function Trainer._get_output_dir at 0x7f8564211590>, '_save_checkpoint': <function Trainer._save_checkpoint at 0x7f85642116f0>, '_determine_best_metric': <function Trainer._determine_best_metric at 0x7f8564211850>, '_save_rng_state': <function Trainer._save_rng_state at 0x7f85642119b0>, '_save_optimizer_and_scheduler': <function Trainer._save_optimizer_and_scheduler at 0x7f8564211b10>, '_save_scaler': <function Trainer._save_scaler at 0x7f8564211c70>, '_load_from_checkpoint': <function Trainer._load_from_checkpoint at 0x7f8564211dd0>, '_load_best_model': <function Trainer._load_best_model at 0x7f8564211f30>, '_load_rng_state': <function Trainer._load_rng_state at 0x7f8564212090>, '_load_optimizer_and_scheduler': <function Trainer._load_optimizer_and_scheduler at 0x7f85642121f0>, '_load_scaler': <function Trainer._load_scaler at 0x7f8564212350>, '_load_callback_state': <function Trainer._load_callback_state at 0x7f85642124b0>, '_issue_warnings_after_load': <function Trainer._issue_warnings_after_load at 0x7f8564212610>, 'save_model': <function Trainer.save_model at 0x7f8564212770>, '_save': <function Trainer._save at 0x7f85642128d0>, 'log': <function Trainer.log at 0x7f8564212a30>, 'store_flos': <function Trainer.store_flos at 0x7f8564212b90>, 'floating_point_ops': <function Trainer.floating_point_ops at 0x7f8564212cf0>, 'init_hf_repo': <function Trainer.init_hf_repo at 0x7f8564212e50>, 'create_model_card': <function Trainer.create_model_card at 0x7f8564212fb0>, 'push_to_hub': <function Trainer.push_to_hub at 0x7f8564213110>, '_push_from_checkpoint': <function Trainer._push_from_checkpoint at 0x7f8564213270>, '_finish_current_push': <function Trainer._finish_current_push at 0x7f85642133d0>, 'hyperparameter_search': <function Trainer.hyperparameter_search at 0x7f8564213530>, 'call_model_init': <function Trainer.call_model_init at 0x7f8564213690>, '_hp_search_setup': <function Trainer._hp_search_setup at 0x7f85642137f0>, '_report_to_hp_search': <function Trainer._report_to_hp_search at 0x7f8564213950>, '_tune_save_checkpoint': <function Trainer._tune_save_checkpoint at 0x7f8564213ab0>, 'add_callback': <function Trainer.add_callback at 0x7f8564213c10>, 'pop_callback': <function Trainer.pop_callback at 0x7f8564213d70>, 'remove_callback': <function Trainer.remove_callback at 0x7f8564213ed0>, 'is_local_process_zero': <function Trainer.is_local_process_zero at 0x7f85642080f0>, 'is_world_process_zero': <function Trainer.is_world_process_zero at 0x7f8564208250>, '_move_model_to_device': <function Trainer._move_model_to_device at 0x7f85642083b0>, '__static_attributes__': ('_attn_mask_causal_checked', '_created_lr_scheduler', '_eval_dataloaders', '_globalstep_last_logged', '_initial_num_input_tokens_seen', '_loggers_initialized', '_loss_shifts_labels', '_memory_tracker', '_signature_columns', '_total_loss_scalar', '_tr_loss', '_train_batch_size', '_trial', 'accelerator', 'args', 'callback_handler', 'can_return_loss', 'compute_loss_func', 'compute_metrics', 'compute_objective', 'control', 'current_flos', 'current_gradient_accumulation_steps', 'data_collator', 'deepspeed', 'eval_dataset', 'gather_function', 'hp_name', 'hp_search_backend', 'hp_space', 'hub_model_id', 'is_deepspeed_enabled', 'is_fsdp_enabled', 'is_fsdp_xla_enabled', 'is_fsdp_xla_v1_enabled', 'is_fsdp_xla_v2_enabled', 'is_in_train', 'is_model_parallel', 'label_names', 'label_smoother', 'lr_scheduler', 'model', 'model_accepts_loss_kwargs', 'model_init', 'model_preparation_time', 'model_wrapped', 'neftune_hook_handle', 'neftune_noise_alpha', 'objective', 'optimizer', 'optimizer_cls_and_kwargs', 'place_model_on_device', 'preprocess_logits_for_metrics', 'processing_class', 'push_in_progress', 'state', 'train_dataset'), '__dict__': <attribute '__dict__' of 'Trainer' objects>, '__weakref__': <attribute '__weakref__' of 'Trainer' objects>, '__backends': ['torch', 'accelerate']}
__dir__: <method '__dir__' of 'object' objects>
__doc__:
Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for 🤗 Transformers.
Args:
model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*):
The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed.
<Tip>
[`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use
your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers
models.
</Tip>
args ([`TrainingArguments`], *optional*):
The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the
`output_dir` set to a directory named *tmp_trainer* in the current directory if not provided.
data_collator (`DataCollator`, *optional*):
The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will
default to [`default_data_collator`] if no `processing_class` is provided, an instance of
[`DataCollatorWithPadding`] otherwise if the processing_class is a feature extractor or tokenizer.
train_dataset (`torch.utils.data.Dataset` | `torch.utils.data.IterableDataset` | `datasets.Dataset`, *optional*):
The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the
`model.forward()` method are automatically removed.
Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a
distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a
`torch.Generator` for the randomization that must be identical on all processes (and the Trainer will
manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally
sets the seed of the RNGs used.
eval_dataset (`torch.utils.data.Dataset` | dict[str, `torch.utils.data.Dataset`] | `datasets.Dataset`, *optional*):
The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the
`model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each
dataset prepending the dictionary key to the metric name.
processing_class (`PreTrainedTokenizerBase` or `BaseImageProcessor` or `FeatureExtractionMixin` or `ProcessorMixin`, *optional*):
Processing class used to process the data. If provided, will be used to automatically process the inputs
for the model, and it will be saved along the model to make it easier to rerun an interrupted training or
reuse the fine-tuned model.
model_init (`Callable[[], PreTrainedModel]`, *optional*):
A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start
from a new instance of the model as given by this function.
The function may have zero argument, or a single one containing the optuna/Ray Tune trial object, to
be able to choose different architectures according to hyperparameters (such as layer count, sizes of
inner layers, dropout probabilities etc).
compute_loss_func (`Callable`, *optional*):
A function that accepts the raw model outputs, labels, and the number of items in the entire accumulated
batch (batch_size * gradient_accumulation_steps) and returns the loss. For example, see the default [loss function](https://github.com/huggingface/transformers/blob/052e652d6d53c2b26ffde87e039b723949a53493/src/transformers/trainer.py#L3618) used by [`Trainer`].
compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):
The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return
a dictionary string to metric values. *Note* When passing TrainingArgs with `batch_eval_metrics` set to
`True`, your compute_metrics function must take a boolean `compute_result` argument. This will be triggered
after the last eval batch to signal that the function needs to calculate and return the global summary
statistics rather than accumulating the batch-level statistics
callbacks (List of [`TrainerCallback`], *optional*):
A list of callbacks to customize the training loop. Will add those to the list of default callbacks
detailed in [here](callback).
If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method.
optimizers (`tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`):
A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your
model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.
optimizer_cls_and_kwargs (`tuple[Type[torch.optim.Optimizer], dict[str, Any]]`, *optional*):
A tuple containing the optimizer class and keyword arguments to use.
Overrides `optim` and `optim_args` in `args`. Incompatible with the `optimizers` argument.
Unlike `optimizers`, this argument avoids the need to place model parameters on the correct devices before initializing the Trainer.
preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*):
A function that preprocess the logits right before caching them at each evaluation step. Must take two
tensors, the logits and the labels, and return the logits once processed as desired. The modifications made
by this function will be reflected in the predictions received by `compute_metrics`.
Note that the labels (second parameter) will be `None` if the dataset does not have them.
Important attributes:
- **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`]
subclass.
- **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the
original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`,
the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner
model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.
- **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from
data parallelism, this means some of the model layers are split on different GPUs).
- **place_model_on_device** -- Whether or not to automatically place the model on the device. Defaults to
`True` unless model parallel, DeepSpeed, FSDP, full fp16/bf16 eval, or SageMaker MP is active. Can be
overridden by subclassing `TrainingArguments` and overriding the `place_model_on_device` property.
- **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while
in `train`)
__eq__: <slot wrapper '__eq__' of 'object' objects>
__firstlineno__: 251
__format__: <method '__format__' of 'object' objects>
__ge__: <slot wrapper '__ge__' of 'object' objects>
__getattribute__: <slot wrapper '__getattribute__' of 'object' objects>
__getstate__: <method '__getstate__' of 'object' objects>
__gt__: <slot wrapper '__gt__' of 'object' objects>
__hash__: <slot wrapper '__hash__' of 'object' objects>
__init__: <function Trainer.__init__ at 0x7f8564299430>
__init_subclass__: <built-in method __init_subclass__ of type object at 0x3da96c80>
__le__: <slot wrapper '__le__' of 'object' objects>
__lt__: <slot wrapper '__lt__' of 'object' objects>
__module__: transformers.trainer
__ne__: <slot wrapper '__ne__' of 'object' objects>
__new__: <built-in method __new__ of type object at 0x15b3a00>
__reduce__: <method '__reduce__' of 'object' objects>
__reduce_ex__: <method '__reduce_ex__' of 'object' objects>
__repr__: <slot wrapper '__repr__' of 'object' objects>
__setattr__: <slot wrapper '__setattr__' of 'object' objects>
__sizeof__: <method '__sizeof__' of 'object' objects>
__static_attributes__: ('_attn_mask_causal_checked', '_created_lr_scheduler', '_eval_dataloaders', '_globalstep_last_logged', '_initial_num_input_tokens_seen', '_loggers_initialized', '_loss_shifts_labels', '_memory_tracker', '_signature_columns', '_total_loss_scalar', '_tr_loss', '_train_batch_size', '_trial', 'accelerator', 'args', 'callback_handler', 'can_return_loss', 'compute_loss_func', 'compute_metrics', 'compute_objective', 'control', 'current_flos', 'current_gradient_accumulation_steps', 'data_collator', 'deepspeed', 'eval_dataset', 'gather_function', 'hp_name', 'hp_search_backend', 'hp_space', 'hub_model_id', 'is_deepspeed_enabled', 'is_fsdp_enabled', 'is_fsdp_xla_enabled', 'is_fsdp_xla_v1_enabled', 'is_fsdp_xla_v2_enabled', 'is_in_train', 'is_model_parallel', 'label_names', 'label_smoother', 'lr_scheduler', 'model', 'model_accepts_loss_kwargs', 'model_init', 'model_preparation_time', 'model_wrapped', 'neftune_hook_handle', 'neftune_noise_alpha', 'objective', 'optimizer', 'optimizer_cls_and_kwargs', 'place_model_on_device', 'preprocess_logits_for_metrics', 'processing_class', 'push_in_progress', 'state', 'train_dataset')
__str__: <slot wrapper '__str__' of 'object' objects>
__subclasshook__: <built-in method __subclasshook__ of type object at 0x3da96c80>
__weakref__: <attribute '__weakref__' of 'Trainer' objects>
_build_accelerator_args: <function Trainer._build_accelerator_args at 0x7f8564195900>
_clip_grad_norm: <function Trainer._clip_grad_norm at 0x7f8564210ca0>
_determine_best_metric: <function Trainer._determine_best_metric at 0x7f8564211850>
_evaluate: <function Trainer._evaluate at 0x7f8564211430>
_finalize_training: <function Trainer._finalize_training at 0x7f85641973d0>
_finish_current_push: <function Trainer._finish_current_push at 0x7f85642133d0>
_get_collator_with_removed_columns: <function Trainer._get_collator_with_removed_columns at 0x7f8564196820>
_get_dataloader: <function Trainer._get_dataloader at 0x7f8564196140>
_get_eval_sampler: <function Trainer._get_eval_sampler at 0x7f8564196400>
_get_grad_norm: <function Trainer._get_grad_norm at 0x7f8564210d50>
_get_learning_rate: <function Trainer._get_learning_rate at 0x7f8564102770>
_get_num_items_in_batch: <function Trainer._get_num_items_in_batch at 0x7f8564197d70>
_get_output_dir: <function Trainer._get_output_dir at 0x7f8564211590>
_get_train_sampler: <function Trainer._get_train_sampler at 0x7f85641962a0>
_hp_search_setup: <function Trainer._hp_search_setup at 0x7f85642137f0>
_init_training_state: <function Trainer._init_training_state at 0x7f85641971c0>
_inner_training_loop: <function Trainer._inner_training_loop at 0x7f8564197060>
_issue_warnings_after_load: <function Trainer._issue_warnings_after_load at 0x7f8564212610>
_load_best_model: <function Trainer._load_best_model at 0x7f8564211f30>
_load_callback_state: <function Trainer._load_callback_state at 0x7f85642124b0>
_load_from_checkpoint: <function Trainer._load_from_checkpoint at 0x7f8564211dd0>
_load_optimizer_and_scheduler: <function Trainer._load_optimizer_and_scheduler at 0x7f85642121f0>
_load_rng_state: <function Trainer._load_rng_state at 0x7f8564212090>
_load_scaler: <function Trainer._load_scaler at 0x7f8564212350>
_maybe_log_save_evaluate: <function Trainer._maybe_log_save_evaluate at 0x7f8564197ab0>
_move_model_to_device: <function Trainer._move_model_to_device at 0x7f85642083b0>
_prepare_context_parallel_inputs: <function Trainer._prepare_context_parallel_inputs at 0x7f8564210250>
_prepare_for_training: <function Trainer._prepare_for_training at 0x7f8564197270>
_prepare_input: <function Trainer._prepare_input at 0x7f8564197ed0>
_prepare_inputs: <function Trainer._prepare_inputs at 0x7f85642100f0>
_push_from_checkpoint: <function Trainer._push_from_checkpoint at 0x7f8564213270>
_remove_unused_columns: <function Trainer._remove_unused_columns at 0x7f85641966c0>
_report_to_hp_search: <function Trainer._report_to_hp_search at 0x7f8564213950>
_run_epoch: <function Trainer._run_epoch at 0x7f8564197320>
_save: <function Trainer._save at 0x7f85642128d0>
_save_checkpoint: <function Trainer._save_checkpoint at 0x7f85642116f0>
_save_optimizer_and_scheduler: <function Trainer._save_optimizer_and_scheduler at 0x7f8564211b10>
_save_rng_state: <function Trainer._save_rng_state at 0x7f85642119b0>
_save_scaler: <function Trainer._save_scaler at 0x7f8564211c70>
_set_signature_columns_if_needed: <function Trainer._set_signature_columns_if_needed at 0x7f8564196560>
_track_num_input_tokens: <function Trainer._track_num_input_tokens at 0x7f8564210bf0>
_tune_save_checkpoint: <function Trainer._tune_save_checkpoint at 0x7f8564213ab0>
_update_auto_batch_size: <function Trainer._update_auto_batch_size at 0x7f8564210b40>
_validate_args: <function Trainer._validate_args at 0x7f85641957a0>
_wrap_model: <function Trainer._wrap_model at 0x7f8564210a90>
add_callback: <function Trainer.add_callback at 0x7f8564213c10>
autocast_smart_context_manager: <function Trainer.autocast_smart_context_manager at 0x7f8564197950>
call_model_init: <function Trainer.call_model_init at 0x7f8564213690>
compute_loss: <function Trainer.compute_loss at 0x7f8564197690>
compute_loss_context_manager: <function Trainer.compute_loss_context_manager at 0x7f85641977f0>
create_accelerator_and_postprocess: <function Trainer.create_accelerator_and_postprocess at 0x7f8564195a60>
create_model_card: <function Trainer.create_model_card at 0x7f8564212fb0>
create_optimizer: <function Trainer.create_optimizer at 0x7f8564196ae0>
create_optimizer_and_scheduler: <function Trainer.create_optimizer_and_scheduler at 0x7f8564196980>
create_scheduler: <function Trainer.create_scheduler at 0x7f8564196c40>
evaluate: <function Trainer.evaluate at 0x7f8564210eb0>
evaluation_loop: <function Trainer.evaluation_loop at 0x7f8564211010>
floating_point_ops: <function Trainer.floating_point_ops at 0x7f8564212cf0>
get_batch_samples: <function Trainer.get_batch_samples at 0x7f8564197c10>
get_cp_size: <function Trainer.get_cp_size at 0x7f85642107d0>
get_decay_parameter_names: <function Trainer.get_decay_parameter_names at 0x7f8564103270>
get_eval_dataloader: <function Trainer.get_eval_dataloader at 0x7f8564195d20>
get_learning_rates: <function get_learning_rates at 0x7f85650c1d20>
get_num_trainable_parameters: <function get_num_trainable_parameters at 0x7f85650c1bc0>
get_optimizer_cls_and_kwargs: <function Trainer.get_optimizer_cls_and_kwargs at 0x7f8564196da0>
get_optimizer_group: <function get_optimizer_group at 0x7f85650c1e80>
get_sp_size: <function Trainer.get_sp_size at 0x7f8564210670>
get_test_dataloader: <function Trainer.get_test_dataloader at 0x7f8564195e80>
get_total_train_batch_size: <function Trainer.get_total_train_batch_size at 0x7f8564210510>
get_tp_size: <function Trainer.get_tp_size at 0x7f8564210930>
get_train_dataloader: <function Trainer.get_train_dataloader at 0x7f8564195bc0>
hyperparameter_search: <function Trainer.hyperparameter_search at 0x7f8564213530>
init_hf_repo: <function Trainer.init_hf_repo at 0x7f8564212e50>
is_local_process_zero: <function Trainer.is_local_process_zero at 0x7f85642080f0>
is_world_process_zero: <function Trainer.is_world_process_zero at 0x7f8564208250>
log: <function Trainer.log at 0x7f8564212a30>
log_metrics: <function log_metrics at 0x7f85650c1900>
metrics_format: <function metrics_format at 0x7f85650c1850>
num_examples: <function Trainer.num_examples at 0x7f8564195fe0>
pop_callback: <function Trainer.pop_callback at 0x7f8564213d70>
predict: <function Trainer.predict at 0x7f8564211170>
prediction_step: <function Trainer.prediction_step at 0x7f85642112d0>
push_to_hub: <function Trainer.push_to_hub at 0x7f8564213110>
remove_callback: <function Trainer.remove_callback at 0x7f8564213ed0>
save_metrics: <function save_metrics at 0x7f85650c19b0>
save_model: <function Trainer.save_model at 0x7f8564212770>
save_state: <function save_state at 0x7f85650c1a60>
set_initial_training_values: <function Trainer.set_initial_training_values at 0x7f85642103b0>
store_flos: <function Trainer.store_flos at 0x7f8564212b90>
train: <function Trainer.train at 0x7f8564196f00>
training_step: <function Trainer.training_step at 0x7f8564197530>
Retrieve All Methods of a Class#
There are a few ways to do it.
Using __dict__#
child_class_methods_using_dict = list(ChildClass.__dict__.keys())
pprint(sorted(child_class_methods_using_dict))
assert "parent_method" not in child_class_methods_using_dict
assert "read_only_attr" in child_class_methods_using_dict
assert "class_method" in child_class_methods_using_dict
[ │ '_ChildClass__private_attr', │ '__doc__', │ '__firstlineno__', │ '__init__', │ '__module__', │ '__static_attributes__', │ '__str__', │ '_protected_attr', │ 'class_attr', │ 'class_method', │ 'instance_method', │ 'read_only_attr', │ 'static_method' ]
Notice that the parent class methods are not included!
pprint(instance_child.__class__.__dict__.keys() == ChildClass.__dict__.keys())
True
Using vars#
vars and __dict__ are equivalent, but people are preferring the former due
to some efficiency reasons, which can be found in this post.
child_class_methods_using_vars = list(vars(ChildClass).keys())
pprint(sorted(child_class_methods_using_vars))
assert "parent_method" not in child_class_methods_using_vars
assert "read_only_attr" in child_class_methods_using_vars
assert "class_method" in child_class_methods_using_vars
assert set(child_class_methods_using_dict) == set(child_class_methods_using_vars)
[ │ '_ChildClass__private_attr', │ '__doc__', │ '__firstlineno__', │ '__init__', │ '__module__', │ '__static_attributes__', │ '__str__', │ '_protected_attr', │ 'class_attr', │ 'class_method', │ 'instance_method', │ 'read_only_attr', │ 'static_method' ]
Using dir#
To include the base/parent class methods, we can use dir instead.
child_class_methods_using_dir = list(dir(ChildClass))
pprint(sorted(child_class_methods_using_dir))
assert "parent_method" in child_class_methods_using_dir
assert "read_only_attr" in child_class_methods_using_dir
assert "class_method" in child_class_methods_using_dir
[ │ '_ChildClass__private_attr', │ '__class__', │ '__delattr__', │ '__dict__', │ '__dir__', │ '__doc__', │ '__eq__', │ '__firstlineno__', │ '__format__', │ '__ge__', │ '__getattribute__', │ '__getstate__', │ '__gt__', │ '__hash__', │ '__init__', │ '__init_subclass__', │ '__le__', │ '__lt__', │ '__module__', │ '__ne__', │ '__new__', │ '__reduce__', │ '__reduce_ex__', │ '__repr__', │ '__setattr__', │ '__sizeof__', │ '__static_attributes__', │ '__str__', │ '__subclasshook__', │ '__weakref__', │ '_protected_attr', │ 'class_attr', │ 'class_method', │ 'instance_method', │ 'parent_class_attr', │ 'parent_method', │ 'read_only_attr', │ 'static_method' ]
Using inspect.getmembers#
We use inspect.getmembers to get all members of a class, and then filter out
via the predicate inspect.isroutine, a stronger filter than inspect.isfunction
or inspect.ismethod.
We attach the source code of inspect.isroutine here for reference.
def isroutine(object):
"""Return true if the object is any kind of function or method."""
return (isbuiltin(object)
or isfunction(object)
or ismethod(object)
or ismethoddescriptor(object))
predicate = inspect.isroutine
child_class_methods_using_getmembers = list(get_members_of_function_or_method(ChildClass, predicate=predicate))
pprint(sorted(child_class_methods_using_getmembers))
[ │ ('__delattr__', <slot wrapper '__delattr__' of 'object' objects>), │ ('__dir__', <method '__dir__' of 'object' objects>), │ ('__eq__', <slot wrapper '__eq__' of 'object' objects>), │ ('__format__', <method '__format__' of 'object' objects>), │ ('__ge__', <slot wrapper '__ge__' of 'object' objects>), │ ('__getattribute__', <slot wrapper '__getattribute__' of 'object' objects>), │ ('__getstate__', <method '__getstate__' of 'object' objects>), │ ('__gt__', <slot wrapper '__gt__' of 'object' objects>), │ ('__hash__', <slot wrapper '__hash__' of 'object' objects>), │ ('__init__', <function ChildClass.__init__ at 0x7f86547f7110>), │ ('__init_subclass__', <built-in method __init_subclass__ of type object at 0x3d40d5b0>), │ ('__le__', <slot wrapper '__le__' of 'object' objects>), │ ('__lt__', <slot wrapper '__lt__' of 'object' objects>), │ ('__ne__', <slot wrapper '__ne__' of 'object' objects>), │ ('__new__', <built-in method __new__ of type object at 0x15b3a00>), │ ('__reduce__', <method '__reduce__' of 'object' objects>), │ ('__reduce_ex__', <method '__reduce_ex__' of 'object' objects>), │ ('__repr__', <slot wrapper '__repr__' of 'object' objects>), │ ('__setattr__', <slot wrapper '__setattr__' of 'object' objects>), │ ('__sizeof__', <method '__sizeof__' of 'object' objects>), │ ('__str__', <function ChildClass.__str__ at 0x7f86547f6a30>), │ ('__subclasshook__', <built-in method __subclasshook__ of type object at 0x3d40d5b0>), │ ('class_method', <bound method ChildClass.class_method of <class '__main__.ChildClass'>>), │ ('instance_method', <function ChildClass.instance_method at 0x7f86547f6e50>), │ ('parent_method', <function ParentClass.parent_method at 0x7f86547f7270>), │ ('static_method', <function ChildClass.static_method at 0x7f86547f6980>) ]
Of course, the reason to retrieve all methods is a convenience if we want to inspect all methods at once. And if we can obtain all methods, we can then iteratively inspect each method’s signature.
Method Resolution Order#
The above examples do not take into account complicated cases, such as when the class is a subclass of multiple classes, in which case if you just print out the methods of the class, you will have a hard time to know which methods are from which class. You can do so via more filtering, but this is beyond the scope of this notebook.
predicate = inspect.isroutine
GPT2LMHeadModel_methods_using_getmembers = list(get_members_of_function_or_method(GPT2LMHeadModel, predicate=predicate))
pprint(sorted(GPT2LMHeadModel_methods_using_getmembers))
[ │ ('__call__', <function Module._wrapped_call_impl at 0x7f8596627950>), │ ('__delattr__', <function Module.__delattr__ at 0x7f8596628040>), │ ('__dir__', <function Module.__dir__ at 0x7f859662a350>), │ ('__eq__', <slot wrapper '__eq__' of 'object' objects>), │ ('__format__', <method '__format__' of 'object' objects>), │ ('__ge__', <slot wrapper '__ge__' of 'object' objects>), │ ('__getattr__', <function Module.__getattr__ at 0x7f8596627cc0>), │ ('__getattribute__', <slot wrapper '__getattribute__' of 'object' objects>), │ ('__getstate__', <function Module.__getstate__ at 0x7f8596627ab0>), │ ('__gt__', <slot wrapper '__gt__' of 'object' objects>), │ ('__hash__', <slot wrapper '__hash__' of 'object' objects>), │ ('__init__', <function GPT2LMHeadModel.__init__ at 0x7f85654c19b0>), │ ( │ │ '__init_subclass__', │ │ <bound method PreTrainedModel.__init_subclass__ of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('__le__', <slot wrapper '__le__' of 'object' objects>), │ ('__lt__', <slot wrapper '__lt__' of 'object' objects>), │ ('__ne__', <slot wrapper '__ne__' of 'object' objects>), │ ('__new__', <built-in method __new__ of type object at 0x15b3a00>), │ ('__reduce__', <method '__reduce__' of 'object' objects>), │ ('__reduce_ex__', <method '__reduce_ex__' of 'object' objects>), │ ('__repr__', <function Module.__repr__ at 0x7f859662a2a0>), │ ('__setattr__', <function Module.__setattr__ at 0x7f8596627e20>), │ ('__setstate__', <function Module.__setstate__ at 0x7f8596627b60>), │ ('__sizeof__', <method '__sizeof__' of 'object' objects>), │ ('__str__', <slot wrapper '__str__' of 'object' objects>), │ ('__subclasshook__', <built-in method __subclasshook__ of type object at 0x3d2a2fc0>), │ ('_adjust_bias', <function PreTrainedModel._adjust_bias at 0x7f85655978a0>), │ ( │ │ '_adjust_missing_and_unexpected_keys', │ │ <function PreTrainedModel._adjust_missing_and_unexpected_keys at 0x7f856559a6c0> │ ), │ ('_apply', <function Module._apply at 0x7f8596625900>), │ ('_assisted_decoding', <function GenerationMixin._assisted_decoding at 0x7f85677a00f0>), │ ( │ │ '_backward_compatibility_gradient_checkpointing', │ │ <function PreTrainedModel._backward_compatibility_gradient_checkpointing at 0x7f8565595430> │ ), │ ('_beam_search', <function GenerationMixin._beam_search at 0x7f856779fed0>), │ ( │ │ '_beam_search_has_unfinished_sequences', │ │ <function GenerationMixin._beam_search_has_unfinished_sequences at 0x7f856779f950> │ ), │ ('_call_impl', <function Module._call_impl at 0x7f8596627a00>), │ ( │ │ '_can_set_attn_implementation', │ │ <bound method PreTrainedModel._can_set_attn_implementation of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ '_can_set_experts_implementation', │ │ <bound method PreTrainedModel._can_set_experts_implementation of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ '_check_and_adjust_attn_implementation', │ │ <function PreTrainedModel._check_and_adjust_attn_implementation at 0x7f8565596140> │ ), │ ( │ │ '_check_and_adjust_experts_implementation', │ │ <function PreTrainedModel._check_and_adjust_experts_implementation at 0x7f85655962a0> │ ), │ ('_check_early_stop_heuristic', <function GenerationMixin._check_early_stop_heuristic at 0x7f856779f7f0>), │ ( │ │ '_copy_lm_head_original_to_resized', │ │ <function PreTrainedModel._copy_lm_head_original_to_resized at 0x7f85655981a0> │ ), │ ('_default_compile_config', <function PreTrainedModel._default_compile_config at 0x7f856559a090>), │ ('_dispatch_accelerate_model', <function PeftAdapterMixin._dispatch_accelerate_model at 0x7f856566dc70>), │ ('_expand_inputs_for_generation', <function GenerationMixin._expand_inputs_for_generation at 0x7f856779ceb0>), │ ( │ │ '_extract_generation_mode_kwargs', │ │ <function GenerationMixin._extract_generation_mode_kwargs at 0x7f856779ec40> │ ), │ ('_finalize_model_loading', <function PreTrainedModel._finalize_model_loading at 0x7f85655996f0>), │ ('_flash_attn_can_dispatch', <function PreTrainedModel._flash_attn_can_dispatch at 0x7f8565595bc0>), │ ('_flash_attn_import_error', <function PreTrainedModel._flash_attn_import_error at 0x7f8565595a60>), │ ('_flatten_beam_dim', <function GenerationMixin._flatten_beam_dim at 0x7f856779f3d0>), │ ('_flex_attn_can_dispatch', <function PreTrainedModel._flex_attn_can_dispatch at 0x7f8565595fe0>), │ ( │ │ '_from_config', │ │ <bound method PreTrainedModel._from_config of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('_gather_beams', <function GenerationMixin._gather_beams at 0x7f856779f690>), │ ('_get_backward_hooks', <function Module._get_backward_hooks at 0x7f8596627320>), │ ('_get_backward_pre_hooks', <function Module._get_backward_pre_hooks at 0x7f85966273d0>), │ ('_get_candidate_generator', <function GenerationMixin._get_candidate_generator at 0x7f856779d170>), │ ('_get_deprecated_gen_repo', <function GenerationMixin._get_deprecated_gen_repo at 0x7f856779eae0>), │ ('_get_dtype_plan', <function PreTrainedModel._get_dtype_plan at 0x7f8565599170>), │ ('_get_files_timestamps', <function PushToHubMixin._get_files_timestamps at 0x7f8654958460>), │ ('_get_logits_processor', <function GenerationMixin._get_logits_processor at 0x7f856779d2d0>), │ ('_get_name', <function Module._get_name at 0x7f8596629fe0>), │ ('_get_resized_embeddings', <function PreTrainedModel._get_resized_embeddings at 0x7f8565597c10>), │ ('_get_resized_lm_head', <function PreTrainedModel._get_resized_lm_head at 0x7f8565597d70>), │ ( │ │ '_get_running_beams_for_next_iteration', │ │ <function GenerationMixin._get_running_beams_for_next_iteration at 0x7f856779fc10> │ ), │ ('_get_static_cache_init_shape', <function GenerationMixin._get_static_cache_init_shape at 0x7f856779df30>), │ ('_get_stopping_criteria', <function GenerationMixin._get_stopping_criteria at 0x7f856779d430>), │ ('_get_top_k_continuations', <function GenerationMixin._get_top_k_continuations at 0x7f856779fab0>), │ ('_grouped_mm_can_dispatch', <function PreTrainedModel._grouped_mm_can_dispatch at 0x7f8565595e80>), │ ('_has_unfinished_sequences', <function GenerationMixin._has_unfinished_sequences at 0x7f856779efb0>), │ ( │ │ '_init_added_embeddings_weights_with_mean', │ │ <function PreTrainedModel._init_added_embeddings_weights_with_mean at 0x7f8565597e20> │ ), │ ( │ │ '_init_added_lm_head_bias_with_mean', │ │ <function PreTrainedModel._init_added_lm_head_bias_with_mean at 0x7f85655980f0> │ ), │ ( │ │ '_init_added_lm_head_weights_with_mean', │ │ <function PreTrainedModel._init_added_lm_head_weights_with_mean at 0x7f8565598040> │ ), │ ('_init_weights', <function GPT2PreTrainedModel._init_weights at 0x7f85654c0eb0>), │ ('_initialize_missing_keys', <function PreTrainedModel._initialize_missing_keys at 0x7f856559a560>), │ ('_initialize_weights', <function PreTrainedModel._initialize_weights at 0x7f8565597320>), │ ('_load_from_state_dict', <function Module._load_from_state_dict at 0x7f8596628a90>), │ ('_load_pretrained_model', <function PreTrainedModel._load_pretrained_model at 0x7f8565599590>), │ ( │ │ '_maybe_initialize_input_ids_for_generation', │ │ <function GenerationMixin._maybe_initialize_input_ids_for_generation at 0x7f856779c880> │ ), │ ( │ │ '_maybe_warn_non_full_backward_hook', │ │ <function Module._maybe_warn_non_full_backward_hook at 0x7f8596627530> │ ), │ ( │ │ '_merge_criteria_processor_list', │ │ <function GenerationMixin._merge_criteria_processor_list at 0x7f856779d590> │ ), │ ( │ │ '_move_missing_keys_from_meta_to_device', │ │ <function PreTrainedModel._move_missing_keys_from_meta_to_device at 0x7f856559a400> │ ), │ ('_named_members', <function Module._named_members at 0x7f8596628d50>), │ ('_optimize_model_for_decode', <function GenerationMixin._optimize_model_for_decode at 0x7f856779e980>), │ ('_prefill', <function GenerationMixin._prefill at 0x7f85677a0250>), │ ( │ │ '_prepare_attention_mask_for_generation', │ │ <function GenerationMixin._prepare_attention_mask_for_generation at 0x7f856779ca90> │ ), │ ('_prepare_cache_for_generation', <function GenerationMixin._prepare_cache_for_generation at 0x7f856779e350>), │ ( │ │ '_prepare_decoder_input_ids_for_generation', │ │ <function GenerationMixin._prepare_decoder_input_ids_for_generation at 0x7f856779cd50> │ ), │ ( │ │ '_prepare_encoder_decoder_kwargs_for_generation', │ │ <function GenerationMixin._prepare_encoder_decoder_kwargs_for_generation at 0x7f856779cbf0> │ ), │ ('_prepare_generated_length', <function GenerationMixin._prepare_generated_length at 0x7f856779dc70>), │ ('_prepare_generation_config', <function GenerationMixin._prepare_generation_config at 0x7f856779ddd0>), │ ('_prepare_model_inputs', <function GenerationMixin._prepare_model_inputs at 0x7f856779c720>), │ ( │ │ '_prepare_position_ids_for_generation', │ │ <function GenerationMixin._prepare_position_ids_for_generation at 0x7f856779c930> │ ), │ ('_prepare_special_tokens', <function GenerationMixin._prepare_special_tokens at 0x7f856779e610>), │ ('_prepare_static_cache', <function GenerationMixin._prepare_static_cache at 0x7f856779e090>), │ ( │ │ '_register_load_state_dict_pre_hook', │ │ <function Module._register_load_state_dict_pre_hook at 0x7f85966287d0> │ ), │ ('_register_state_dict_hook', <function Module._register_state_dict_hook at 0x7f85966280f0>), │ ('_replicate_for_data_parallel', <function Module._replicate_for_data_parallel at 0x7f859662a400>), │ ('_resize_token_embeddings', <function PreTrainedModel._resize_token_embeddings at 0x7f8565597ab0>), │ ('_sample', <function GenerationMixin._sample at 0x7f856779f270>), │ ('_save_to_state_dict', <function Module._save_to_state_dict at 0x7f85966283b0>), │ ('_sdpa_can_dispatch', <function PreTrainedModel._sdpa_can_dispatch at 0x7f8565595d20>), │ ('_set_gradient_checkpointing', <function PreTrainedModel._set_gradient_checkpointing at 0x7f8565598720>), │ ('_slow_forward', <function Module._slow_forward at 0x7f85966278a0>), │ ( │ │ '_supports_default_dynamic_cache', │ │ <bound method GenerationMixin._supports_default_dynamic_cache of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('_supports_logits_to_keep', <function GenerationMixin._supports_logits_to_keep at 0x7f856779e4b0>), │ ('_unflatten_beam_dim', <function GenerationMixin._unflatten_beam_dim at 0x7f856779f530>), │ ('_update_finished_beams', <function GenerationMixin._update_finished_beams at 0x7f856779fd70>), │ ( │ │ '_update_model_kwargs_for_generation', │ │ <function GenerationMixin._update_model_kwargs_for_generation at 0x7f856779d010> │ ), │ ('_upload_modified_files', <function PushToHubMixin._upload_modified_files at 0x7f86549585c0>), │ ('_valid_auto_compile_criteria', <function GenerationMixin._valid_auto_compile_criteria at 0x7f856779e770>), │ ('_validate_generated_length', <function GenerationMixin._validate_generated_length at 0x7f856779db10>), │ ('_validate_generation_mode', <function GenerationMixin._validate_generation_mode at 0x7f856779d850>), │ ('_validate_model_kwargs', <function GenerationMixin._validate_model_kwargs at 0x7f856779d9b0>), │ ('_wrapped_call_impl', <function Module._wrapped_call_impl at 0x7f8596627950>), │ ('active_adapters', <function PeftAdapterMixin.active_adapters at 0x7f856566d9b0>), │ ('add_adapter', <function PeftAdapterMixin.add_adapter at 0x7f856566d430>), │ ('add_model_tags', <function PreTrainedModel.add_model_tags at 0x7f8565595590>), │ ('add_module', <function Module.add_module at 0x7f8596624eb0>), │ ('adjust_generation_fn', <function GenerationMixin.adjust_generation_fn at 0x7f856779c300>), │ ('apply', <function Module.apply at 0x7f8596625a60>), │ ('bfloat16', <function Module.bfloat16 at 0x7f8596626820>), │ ('buffers', <function Module.buffers at 0x7f8596629170>), │ ( │ │ 'can_generate', │ │ <bound method PreTrainedModel.can_generate of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('children', <function Module.children at 0x7f8596629430>), │ ('compile', <function Module.compile at 0x7f859662a560>), │ ('compute_transition_scores', <function GenerationMixin.compute_transition_scores at 0x7f856779d6f0>), │ ( │ │ 'continuous_batching_context_manager', │ │ <function ContinuousMixin.continuous_batching_context_manager at 0x7f85677794e0> │ ), │ ('cpu', <function Module.cpu at 0x7f8596626140>), │ ( │ │ 'create_extended_attention_mask_for_decoder', │ │ <function ModuleUtilsMixin.create_extended_attention_mask_for_decoder at 0x7f856558bd70> │ ), │ ('cuda', <function Module.cuda at 0x7f8565598ca0>), │ ('delete_adapter', <function PeftAdapterMixin.delete_adapter at 0x7f856566ddd0>), │ ('dequantize', <function PreTrainedModel.dequantize at 0x7f8565595380>), │ ( │ │ 'destroy_cached_continuous_batching_manager', │ │ <function ContinuousMixin.destroy_cached_continuous_batching_manager at 0x7f8567779220> │ ), │ ('disable_adapters', <function PeftAdapterMixin.disable_adapters at 0x7f856566d6f0>), │ ('disable_input_require_grads', <function PreTrainedModel.disable_input_require_grads at 0x7f8565596c40>), │ ('double', <function Module.double at 0x7f8596626560>), │ ('enable_adapters', <function PeftAdapterMixin.enable_adapters at 0x7f856566d850>), │ ('enable_input_require_grads', <function PreTrainedModel.enable_input_require_grads at 0x7f8565596b90>), │ ('enable_peft_hotswap', <function PeftAdapterMixin.enable_peft_hotswap at 0x7f856566d2d0>), │ ('eval', <function PreTrainedModel.eval at 0x7f856559ac40>), │ ('extra_repr', <function Module.extra_repr at 0x7f859662a140>), │ ('float', <function PreTrainedModel.float at 0x7f8565598eb0>), │ ('forward', <function GPT2LMHeadModel.forward at 0x7f85654c1d20>), │ ( │ │ 'from_pretrained', │ │ <bound method PreTrainedModel.from_pretrained of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('generate', <function GenerationMixin.generate at 0x7f856779ee50>), │ ('generate_batch', <function ContinuousMixin.generate_batch at 0x7f85677796f0>), │ ('get_adapter_state_dict', <function PeftAdapterMixin.get_adapter_state_dict at 0x7f856566db10>), │ ('get_buffer', <function Module.get_buffer at 0x7f8596625590>), │ ('get_compiled_call', <function PreTrainedModel.get_compiled_call at 0x7f856559a1f0>), │ ( │ │ 'get_correct_attn_implementation', │ │ <function PreTrainedModel.get_correct_attn_implementation at 0x7f8565596400> │ ), │ ( │ │ 'get_correct_experts_implementation', │ │ <function PreTrainedModel.get_correct_experts_implementation at 0x7f8565596560> │ ), │ ('get_decoder', <function PreTrainedModel.get_decoder at 0x7f8565596fb0>), │ ('get_encoder', <function PreTrainedModel.get_encoder at 0x7f8565596da0>), │ ( │ │ 'get_expanded_tied_weights_keys', │ │ <function PreTrainedModel.get_expanded_tied_weights_keys at 0x7f8565597690> │ ), │ ('get_extended_attention_mask', <function ModuleUtilsMixin.get_extended_attention_mask at 0x7f856558bed0>), │ ('get_extra_state', <function Module.get_extra_state at 0x7f85966256f0>), │ ( │ │ 'get_init_context', │ │ <bound method PreTrainedModel.get_init_context of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('get_input_embeddings', <function EmbeddingAccessMixin.get_input_embeddings at 0x7f8565594250>), │ ('get_memory_footprint', <function PreTrainedModel.get_memory_footprint at 0x7f8565598bf0>), │ ('get_output_embeddings', <function EmbeddingAccessMixin.get_output_embeddings at 0x7f8565594460>), │ ('get_parameter', <function Module.get_parameter at 0x7f8596625430>), │ ('get_parameter_or_buffer', <function PreTrainedModel.get_parameter_or_buffer at 0x7f856559a8d0>), │ ('get_position_embeddings', <function PreTrainedModel.get_position_embeddings at 0x7f8565598460>), │ ('get_submodule', <function Module.get_submodule at 0x7f8596625170>), │ ( │ │ 'gradient_checkpointing_disable', │ │ <function PreTrainedModel.gradient_checkpointing_disable at 0x7f85655987d0> │ ), │ ('gradient_checkpointing_enable', <function PreTrainedModel.gradient_checkpointing_enable at 0x7f85655985c0>), │ ('half', <function PreTrainedModel.half at 0x7f8565598e00>), │ ('heal_tokens', <function GenerationMixin.heal_tokens at 0x7f856779f110>), │ ('init_continuous_batching', <function ContinuousMixin.init_continuous_batching at 0x7f85677790c0>), │ ('init_weights', <function PreTrainedModel.init_weights at 0x7f8565598510>), │ ('initialize_weights', <function PreTrainedModel.initialize_weights at 0x7f8565597530>), │ ('invert_attention_mask', <function ModuleUtilsMixin.invert_attention_mask at 0x7f856558bcc0>), │ ('ipu', <function Module.ipu at 0x7f8596625d20>), │ ( │ │ 'is_backend_compatible', │ │ <bound method PreTrainedModel.is_backend_compatible of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ 'is_custom_code', │ │ <bound method PreTrainedModel.is_custom_code of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ( │ │ 'is_remote_code', │ │ <bound method PreTrainedModel.is_remote_code of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('load_adapter', <function PeftAdapterMixin.load_adapter at 0x7f856566d170>), │ ('load_custom_generate', <function GenerationMixin.load_custom_generate at 0x7f856779c460>), │ ('load_state_dict', <function Module.load_state_dict at 0x7f8596628bf0>), │ ( │ │ 'mark_tied_weights_as_initialized', │ │ <function PreTrainedModel.mark_tied_weights_as_initialized at 0x7f856559a770> │ ), │ ('modules', <function Module.modules at 0x7f85966296f0>), │ ('mtia', <function Module.mtia at 0x7f8596625fe0>), │ ('named_buffers', <function Module.named_buffers at 0x7f85966292d0>), │ ('named_children', <function Module.named_children at 0x7f8596629590>), │ ('named_modules', <function Module.named_modules at 0x7f8596629850>), │ ('named_non_persistent_buffers', <function PreTrainedModel.named_non_persistent_buffers at 0x7f856559aa30>), │ ('named_parameters', <function Module.named_parameters at 0x7f8596629010>), │ ('num_parameters', <function ModuleUtilsMixin.num_parameters at 0x7f85655940f0>), │ ('parameters', <function Module.parameters at 0x7f8596628eb0>), │ ('post_init', <function PreTrainedModel.post_init at 0x7f8565594bf0>), │ ('prepare_inputs_for_generation', <function GenerationMixin.prepare_inputs_for_generation at 0x7f856779c5c0>), │ ('push_to_hub', <function PushToHubMixin.push_to_hub at 0x7f856558b8a0>), │ ('register_backward_hook', <function Module.register_backward_hook at 0x7f8596627110>), │ ('register_buffer', <function Module.register_buffer at 0x7f8596624bf0>), │ ( │ │ 'register_for_auto_class', │ │ <bound method PreTrainedModel.register_for_auto_class of <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>> │ ), │ ('register_forward_hook', <function Module.register_forward_hook at 0x7f85966277f0>), │ ('register_forward_pre_hook', <function Module.register_forward_pre_hook at 0x7f8596627690>), │ ('register_full_backward_hook', <function Module.register_full_backward_hook at 0x7f8596627270>), │ ('register_full_backward_pre_hook', <function Module.register_full_backward_pre_hook at 0x7f8596626fb0>), │ ( │ │ 'register_load_state_dict_post_hook', │ │ <function Module.register_load_state_dict_post_hook at 0x7f8596628930> │ ), │ ('register_load_state_dict_pre_hook', <function Module.register_load_state_dict_pre_hook at 0x7f8596628880>), │ ('register_module', <function Module.register_module at 0x7f8596625010>), │ ('register_parameter', <function Module.register_parameter at 0x7f8596624d50>), │ ('register_state_dict_post_hook', <function Module.register_state_dict_post_hook at 0x7f85966281a0>), │ ('register_state_dict_pre_hook', <function Module.register_state_dict_pre_hook at 0x7f8596628250>), │ ('requires_grad_', <function Module.requires_grad_ at 0x7f8596629c70>), │ ('resize_position_embeddings', <function PreTrainedModel.resize_position_embeddings at 0x7f8565598300>), │ ('resize_token_embeddings', <function PreTrainedModel.resize_token_embeddings at 0x7f8565597a00>), │ ('retrieve_modules_from_names', <function PreTrainedModel.retrieve_modules_from_names at 0x7f85655997a0>), │ ('save_pretrained', <function PreTrainedModel.save_pretrained at 0x7f8565598a90>), │ ('set_adapter', <function PeftAdapterMixin.set_adapter at 0x7f856566d590>), │ ('set_attn_implementation', <function PreTrainedModel.set_attn_implementation at 0x7f8565596980>), │ ('set_decoder', <function PreTrainedModel.set_decoder at 0x7f8565597060>), │ ('set_encoder', <function PreTrainedModel.set_encoder at 0x7f8565596f00>), │ ('set_experts_implementation', <function PreTrainedModel.set_experts_implementation at 0x7f8565596ae0>), │ ('set_extra_state', <function Module.set_extra_state at 0x7f8596625850>), │ ('set_input_embeddings', <function EmbeddingAccessMixin.set_input_embeddings at 0x7f85655943b0>), │ ('set_output_embeddings', <function EmbeddingAccessMixin.set_output_embeddings at 0x7f8565594510>), │ ('set_submodule', <function Module.set_submodule at 0x7f85966252d0>), │ ('set_use_kernels', <function PreTrainedModel.set_use_kernels at 0x7f85655992d0>), │ ('share_memory', <function Module.share_memory at 0x7f8596629f30>), │ ('state_dict', <function Module.state_dict at 0x7f8596628720>), │ ('tie_weights', <function PreTrainedModel.tie_weights at 0x7f85655977f0>), │ ('to', <function Module.to at 0x7f8565598d50>), │ ('to_empty', <function Module.to_empty at 0x7f8596626980>), │ ('train', <function PreTrainedModel.train at 0x7f856559ab90>), │ ('type', <function Module.type at 0x7f85966262a0>), │ ( │ │ 'warn_if_padding_and_no_attention_mask', │ │ <function PreTrainedModel.warn_if_padding_and_no_attention_mask at 0x7f8565599900> │ ), │ ('xpu', <function Module.xpu at 0x7f8596625e80>), │ ('zero_grad', <function Module.zero_grad at 0x7f8596629dd0>) ]
You can get the method resolution order (MRO) of a class via cls.__mro__.
inspect.getmro(GPT2LMHeadModel)
(transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel,
transformers.models.gpt2.modeling_gpt2.GPT2PreTrainedModel,
transformers.modeling_utils.PreTrainedModel,
torch.nn.modules.module.Module,
transformers.modeling_utils.EmbeddingAccessMixin,
transformers.modeling_utils.ModuleUtilsMixin,
transformers.utils.hub.PushToHubMixin,
transformers.integrations.peft.PeftAdapterMixin,
transformers.generation.utils.GenerationMixin,
transformers.generation.continuous_batching.continuous_api.ContinuousMixin,
object)
A pseudocode to get all signatures of a class via MRO is as follows:
def get_all_args(cls: Type[object]) -> Dict[str, inspect.Parameter]:
mro = inspect.getmro(cls)
all_args = {}
for base_class in mro[::-1]: # reverse to start from topmost class
if base_class is object: # skip the base 'object' class
continue
sig = inspect.signature(base_class.__init__)
all_args.update(sig.parameters)
return all_args
Get Class and Instance Attributes#
pprint(list(class_child.__dict__.keys())) # class attributes
pprint(list(instance_child.__dict__.keys())) # instance attributes
[ │ '__module__', │ '__firstlineno__', │ '__doc__', │ 'class_attr', │ '_protected_attr', │ '_ChildClass__private_attr', │ '__init__', │ 'read_only_attr', │ 'instance_method', │ 'class_method', │ 'static_method', │ '__str__', │ '__static_attributes__' ]
['parent_instance_attr', 'instance_attr', 'instance_not_in_constructor_attr', '_private_instance_attr']
union_class_and_instance_attributes = list(set(class_child.__dict__.keys()).union(set(instance_child.__dict__.keys())))
pprint(union_class_and_instance_attributes)
[ │ 'static_method', │ 'parent_instance_attr', │ 'class_attr', │ '__str__', │ '__static_attributes__', │ '__doc__', │ 'instance_not_in_constructor_attr', │ '__module__', │ '_protected_attr', │ 'instance_attr', │ '__init__', │ '_ChildClass__private_attr', │ 'read_only_attr', │ 'class_method', │ '_private_instance_attr', │ '__firstlineno__', │ 'instance_method' ]
Get Signature and Type Annotations of a Function#
func_sig: Signature = inspect.signature(func)
pprint(func_sig.parameters)
pprint(func_sig.return_annotation)
mappingproxy({ │ 'a': <Parameter "a: int">, │ 'b': <Parameter "b: str">, │ 'c': <Parameter "c: List[int]">, │ 'd': <Parameter "d: Tuple[str, str]">, │ 'e': <Parameter "e: int | str">, │ 'kwargs': <Parameter "**kwargs: Any"> })
<class 'str'>
Here are the 4 key properties of the Parameter object
of the Signature object.
@property
def name(self):
return self._name
@property
def default(self):
return self._default
@property
def annotation(self):
return self._annotation
@property
def kind(self):
return self._kind
We will also use get_type_hints to get the type hints of a function
instead of using the annotations property of inspect.Signature. The reason
can be found in the docstring of get_type_hints:
def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
"""Return type hints for an object.
This is often the same as obj.__annotations__, but it handles
forward references encoded as string literals, adds Optional[t] if a
default value equal to None is set and recursively replaces all
'Annotated[T, ...]' with 'T' (unless 'include_extras=True').
...
"""
def no_type_hints(a, b, c, d, e, **kwargs):
return a, b, c, d, e, kwargs
get_type_hints(no_type_hints), inspect.signature(no_type_hints)
({}, <Signature (a, b, c, d, e, **kwargs)>)
How to know if a parameter is optional or not? We can use the inspect.Parameter.empty
property.
for name, value in inspect.signature(func).parameters.items():
print(value.default)
print(value.default is inspect.Parameter.empty)
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
<class 'inspect._empty'>
True
We will also use __mro__ to get the method resolution order of a class
because __bases__ only returns the immediate parent class.
ChildClass.__bases__, GPT2LMHeadModel.__bases__[0].__bases__[0].__bases__
((__main__.ParentClass,),
(torch.nn.modules.module.Module,
transformers.modeling_utils.EmbeddingAccessMixin,
transformers.modeling_utils.ModuleUtilsMixin,
transformers.utils.hub.PushToHubMixin,
transformers.integrations.peft.PeftAdapterMixin))
list(reversed(inspect.getmro(GPT2LMHeadModel)))
[object,
transformers.generation.continuous_batching.continuous_api.ContinuousMixin,
transformers.generation.utils.GenerationMixin,
transformers.integrations.peft.PeftAdapterMixin,
transformers.utils.hub.PushToHubMixin,
transformers.modeling_utils.ModuleUtilsMixin,
transformers.modeling_utils.EmbeddingAccessMixin,
torch.nn.modules.module.Module,
transformers.modeling_utils.PreTrainedModel,
transformers.models.gpt2.modeling_gpt2.GPT2PreTrainedModel,
transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel]
def get_base_classes(cls: Type[Any], include_self: bool = False) -> Set[Type[Any]]:
"""
Get the base classes of a class and all its base classes.
"""
return set(cls.__mro__[0:-1] if include_self else cls.__mro__[1:-1])
pprint(get_base_classes(GPT2LMHeadModel, include_self=True))
{ │ <class 'transformers.modeling_utils.EmbeddingAccessMixin'>, │ <class 'transformers.utils.hub.PushToHubMixin'>, │ <class 'transformers.integrations.peft.PeftAdapterMixin'>, │ <class 'transformers.models.gpt2.modeling_gpt2.GPT2PreTrainedModel'>, │ <class 'transformers.modeling_utils.ModuleUtilsMixin'>, │ <class 'transformers.generation.utils.GenerationMixin'>, │ <class 'torch.nn.modules.module.Module'>, │ <class 'transformers.generation.continuous_batching.continuous_api.ContinuousMixin'>, │ <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, │ <class 'transformers.modeling_utils.PreTrainedModel'> }
def get_default(param: Parameter) -> Any:
"""Return the parameter's default value or None if not specified."""
return param.default if param.default is not param.empty else None
def get_field_annotations(func_or_method: Callable[..., Any]) -> Tuple[List[Tuple[str, Any, Any]], Dict[str, Any]]:
if not inspect.isroutine(func_or_method):
raise ValueError("Expected a function or method")
required_fields = []
optional_fields = []
annotations = {}
try:
sig: Signature = inspect.signature(func_or_method)
type_hints: Dict[str, Any] = get_type_hints(func_or_method)
except ValueError:
raise ValueError("Object does not support signature or type hints extraction.") from None
for name, param in sig.parameters.items():
if name == "self":
continue
type_hint = type_hints.get(name, Any)
annotations[name] = type_hint
if param.default is param.empty:
required_fields.append((name, type_hint, Ellipsis))
else:
default_value = param.default
optional_fields.append((name, type_hint, default_value))
fields = required_fields + optional_fields
return fields, annotations
# TODO: Tuple[str, Any, Any] should be Tuple[str, Any, ellipsis]
def get_constructor_field_annotations(
cls: Type[Any], include_bases: bool = True
) -> Tuple[List[Tuple[str, Any, Any]], Dict[str, Any]]:
fields = []
annotations = {}
classes_to_inspect = [cls] + list(get_base_classes(cls, include_self=False)) if include_bases else [cls]
for c in reversed(classes_to_inspect): # Reverse to respect MRO
if hasattr(c, "__init__"):
class_fields, class_annotations = get_field_annotations(c.__init__)
# Update fields and annotations with those from the current class,
# avoiding duplicates.
for field in class_fields:
if field[0] not in annotations:
fields.append(field) # noqa: PERF401
annotations.update(class_annotations)
return fields, annotations
fields, annotations = get_constructor_field_annotations(TrainingArguments, include_bases=False)
for field in fields:
assert len(field) == 3
print(f"{field[0]}, {field[1]}, {field[2]}")
assert get_field_annotations(TrainingArguments.__init__) == (fields, annotations)
output_dir, str | None, None
per_device_train_batch_size, <class 'int'>, 8
num_train_epochs, <class 'float'>, 3.0
max_steps, <class 'int'>, -1
learning_rate, <class 'float'>, 5e-05
lr_scheduler_type, transformers.trainer_utils.SchedulerType | str, linear
lr_scheduler_kwargs, dict | str | None, None
warmup_steps, <class 'float'>, 0
optim, transformers.training_args.OptimizerNames | str, adamw_torch_fused
optim_args, str | None, None
weight_decay, <class 'float'>, 0.0
adam_beta1, <class 'float'>, 0.9
adam_beta2, <class 'float'>, 0.999
adam_epsilon, <class 'float'>, 1e-08
optim_target_modules, None | str | list[str], None
gradient_accumulation_steps, <class 'int'>, 1
average_tokens_across_devices, <class 'bool'>, True
max_grad_norm, <class 'float'>, 1.0
label_smoothing_factor, <class 'float'>, 0.0
bf16, <class 'bool'>, False
fp16, <class 'bool'>, False
bf16_full_eval, <class 'bool'>, False
fp16_full_eval, <class 'bool'>, False
tf32, bool | None, None
gradient_checkpointing, <class 'bool'>, False
gradient_checkpointing_kwargs, dict[str, typing.Any] | str | None, None
torch_compile, <class 'bool'>, False
torch_compile_backend, str | None, None
torch_compile_mode, str | None, None
use_liger_kernel, <class 'bool'>, False
liger_kernel_config, dict[str, bool] | None, None
use_cache, <class 'bool'>, False
neftune_noise_alpha, float | None, None
torch_empty_cache_steps, int | None, None
auto_find_batch_size, <class 'bool'>, False
logging_strategy, transformers.trainer_utils.IntervalStrategy | str, steps
logging_steps, <class 'float'>, 500
logging_first_step, <class 'bool'>, False
log_on_each_node, <class 'bool'>, True
logging_nan_inf_filter, <class 'bool'>, True
include_num_input_tokens_seen, str | bool, no
log_level, <class 'str'>, passive
log_level_replica, <class 'str'>, warning
disable_tqdm, bool | None, None
report_to, None | str | list[str], none
run_name, str | None, None
project, <class 'str'>, huggingface
trackio_space_id, str | None, None
trackio_bucket_id, str | None, None
trackio_static_space_id, str | None | typing.Literal[False], None
eval_strategy, transformers.trainer_utils.IntervalStrategy | str, no
eval_steps, float | None, None
eval_delay, <class 'float'>, 0
per_device_eval_batch_size, <class 'int'>, 8
prediction_loss_only, <class 'bool'>, False
eval_on_start, <class 'bool'>, False
eval_do_concat_batches, <class 'bool'>, True
eval_use_gather_object, <class 'bool'>, False
eval_accumulation_steps, int | None, None
include_for_metrics, list[str], <factory>
batch_eval_metrics, <class 'bool'>, False
save_only_model, <class 'bool'>, False
save_strategy, transformers.trainer_utils.SaveStrategy | str, steps
save_steps, <class 'float'>, 500
save_on_each_node, <class 'bool'>, False
save_total_limit, int | None, None
enable_jit_checkpoint, <class 'bool'>, False
push_to_hub, <class 'bool'>, False
hub_token, str | None, None
hub_private_repo, bool | None, None
hub_model_id, str | None, None
hub_strategy, transformers.trainer_utils.HubStrategy | str, every_save
hub_always_push, <class 'bool'>, False
hub_revision, str | None, None
load_best_model_at_end, <class 'bool'>, False
metric_for_best_model, str | None, None
greater_is_better, bool | None, None
ignore_data_skip, <class 'bool'>, False
restore_callback_states_from_checkpoint, <class 'bool'>, False
full_determinism, <class 'bool'>, False
seed, <class 'int'>, 42
data_seed, int | None, None
use_cpu, <class 'bool'>, False
accelerator_config, dict | str | None, None
parallelism_config, typing.Any | None, None
dataloader_drop_last, <class 'bool'>, False
dataloader_num_workers, <class 'int'>, 0
dataloader_pin_memory, <class 'bool'>, True
dataloader_persistent_workers, <class 'bool'>, False
dataloader_prefetch_factor, int | None, None
remove_unused_columns, <class 'bool'>, True
label_names, list[str] | None, None
train_sampling_strategy, <class 'str'>, random
length_column_name, <class 'str'>, length
ddp_find_unused_parameters, bool | None, None
ddp_bucket_cap_mb, int | None, None
ddp_broadcast_buffers, bool | None, None
ddp_static_graph, bool | None, None
ddp_backend, str | None, None
ddp_timeout, <class 'int'>, 1800
fsdp, str | None, None
fsdp_config, dict[str, typing.Any] | str | None, None
deepspeed, dict | str | None, None
debug, str | list[transformers.debug_utils.DebugOption],
skip_memory_metrics, <class 'bool'>, True
do_train, <class 'bool'>, False
do_eval, <class 'bool'>, False
do_predict, <class 'bool'>, False
resume_from_checkpoint, str | None, None
warmup_ratio, float | None, None
logging_dir, str | None, None
local_rank, <class 'int'>, -1
Warning: it does not play too well with dataclass and pydantic classes
because they have more complex bells and whistles. However, because of the perks
of dataclass and pydantic, we can just use
property
like model_fields to get all fields and their types.
As we can see from above, we did not handle lr_scheduler_kwargs well:
lr_scheduler_kwargs, typing.Optional[typing.Dict], <factory>
where <factory> is the default value of the parameter. But it is actually
referring to the default_factory of the dataclass field, which can be a default
dict etc.
def type_hint_to_str(type_hint: Any) -> str:
"""
Convert a type hint into its string representation.
"""
if hasattr(type_hint, "__name__"):
return type_hint.__name__
elif hasattr(type_hint, "_name") and type_hint._name is not None:
return str(type_hint._name)
elif type(type_hint) == _GenericAlias: # For Python 3.8+
# Handles complex types, e.g., List[int], Union[str, int]
origin = type_hint_to_str(type_hint.__origin__)
args = ", ".join(type_hint_to_str(arg) for arg in type_hint.__args__)
return f"{origin}[{args}]"
else:
# Fallback for unhandled types
return str(type_hint)
def create_config_class_str(fields: List[Tuple[str, Any, Any]]) -> str:
lines = ["class Config:"]
if not fields:
lines.append(" ...")
else:
init_params = ["self"]
init_body = []
for name, type_hint, default in fields:
type_hint_str = type_hint_to_str(type_hint)
if default is Ellipsis: # Required argument
param_str = f"{name}: {type_hint_str}"
elif default is field:
param_str = f"{name}: {type_hint_str} = field(default_factory=dict)"
else:
default_repr = repr(default) if default is not None else "None"
param_str = f"{name}: {type_hint_str} = {default_repr}"
init_params.append(param_str)
init_body.append(f" self.{name} = {name}")
lines.append(f" def __init__({', '.join(init_params)}):")
lines.extend(init_body)
return "\n".join(lines)
config_class_str = create_config_class_str(fields)
print(config_class_str)
class Config:
def __init__(self, output_dir: Union = None, per_device_train_batch_size: int = 8, num_train_epochs: float = 3.0, max_steps: int = -1, learning_rate: float = 5e-05, lr_scheduler_type: Union = 'linear', lr_scheduler_kwargs: Union = None, warmup_steps: float = 0, optim: Union = 'adamw_torch_fused', optim_args: Union = None, weight_decay: float = 0.0, adam_beta1: float = 0.9, adam_beta2: float = 0.999, adam_epsilon: float = 1e-08, optim_target_modules: Union = None, gradient_accumulation_steps: int = 1, average_tokens_across_devices: bool = True, max_grad_norm: float = 1.0, label_smoothing_factor: float = 0.0, bf16: bool = False, fp16: bool = False, bf16_full_eval: bool = False, fp16_full_eval: bool = False, tf32: Union = None, gradient_checkpointing: bool = False, gradient_checkpointing_kwargs: Union = None, torch_compile: bool = False, torch_compile_backend: Union = None, torch_compile_mode: Union = None, use_liger_kernel: bool = False, liger_kernel_config: Union = None, use_cache: bool = False, neftune_noise_alpha: Union = None, torch_empty_cache_steps: Union = None, auto_find_batch_size: bool = False, logging_strategy: Union = 'steps', logging_steps: float = 500, logging_first_step: bool = False, log_on_each_node: bool = True, logging_nan_inf_filter: bool = True, include_num_input_tokens_seen: Union = 'no', log_level: str = 'passive', log_level_replica: str = 'warning', disable_tqdm: Union = None, report_to: Union = 'none', run_name: Union = None, project: str = 'huggingface', trackio_space_id: Union = None, trackio_bucket_id: Union = None, trackio_static_space_id: Union = None, eval_strategy: Union = 'no', eval_steps: Union = None, eval_delay: float = 0, per_device_eval_batch_size: int = 8, prediction_loss_only: bool = False, eval_on_start: bool = False, eval_do_concat_batches: bool = True, eval_use_gather_object: bool = False, eval_accumulation_steps: Union = None, include_for_metrics: list = <factory>, batch_eval_metrics: bool = False, save_only_model: bool = False, save_strategy: Union = 'steps', save_steps: float = 500, save_on_each_node: bool = False, save_total_limit: Union = None, enable_jit_checkpoint: bool = False, push_to_hub: bool = False, hub_token: Union = None, hub_private_repo: Union = None, hub_model_id: Union = None, hub_strategy: Union = 'every_save', hub_always_push: bool = False, hub_revision: Union = None, load_best_model_at_end: bool = False, metric_for_best_model: Union = None, greater_is_better: Union = None, ignore_data_skip: bool = False, restore_callback_states_from_checkpoint: bool = False, full_determinism: bool = False, seed: int = 42, data_seed: Union = None, use_cpu: bool = False, accelerator_config: Union = None, parallelism_config: Union = None, dataloader_drop_last: bool = False, dataloader_num_workers: int = 0, dataloader_pin_memory: bool = True, dataloader_persistent_workers: bool = False, dataloader_prefetch_factor: Union = None, remove_unused_columns: bool = True, label_names: Union = None, train_sampling_strategy: str = 'random', length_column_name: str = 'length', ddp_find_unused_parameters: Union = None, ddp_bucket_cap_mb: Union = None, ddp_broadcast_buffers: Union = None, ddp_static_graph: Union = None, ddp_backend: Union = None, ddp_timeout: int = 1800, fsdp: Union = None, fsdp_config: Union = None, deepspeed: Union = None, debug: Union = '', skip_memory_metrics: bool = True, do_train: bool = False, do_eval: bool = False, do_predict: bool = False, resume_from_checkpoint: Union = None, warmup_ratio: Union = None, logging_dir: Union = None, local_rank: int = -1):
self.output_dir = output_dir
self.per_device_train_batch_size = per_device_train_batch_size
self.num_train_epochs = num_train_epochs
self.max_steps = max_steps
self.learning_rate = learning_rate
self.lr_scheduler_type = lr_scheduler_type
self.lr_scheduler_kwargs = lr_scheduler_kwargs
self.warmup_steps = warmup_steps
self.optim = optim
self.optim_args = optim_args
self.weight_decay = weight_decay
self.adam_beta1 = adam_beta1
self.adam_beta2 = adam_beta2
self.adam_epsilon = adam_epsilon
self.optim_target_modules = optim_target_modules
self.gradient_accumulation_steps = gradient_accumulation_steps
self.average_tokens_across_devices = average_tokens_across_devices
self.max_grad_norm = max_grad_norm
self.label_smoothing_factor = label_smoothing_factor
self.bf16 = bf16
self.fp16 = fp16
self.bf16_full_eval = bf16_full_eval
self.fp16_full_eval = fp16_full_eval
self.tf32 = tf32
self.gradient_checkpointing = gradient_checkpointing
self.gradient_checkpointing_kwargs = gradient_checkpointing_kwargs
self.torch_compile = torch_compile
self.torch_compile_backend = torch_compile_backend
self.torch_compile_mode = torch_compile_mode
self.use_liger_kernel = use_liger_kernel
self.liger_kernel_config = liger_kernel_config
self.use_cache = use_cache
self.neftune_noise_alpha = neftune_noise_alpha
self.torch_empty_cache_steps = torch_empty_cache_steps
self.auto_find_batch_size = auto_find_batch_size
self.logging_strategy = logging_strategy
self.logging_steps = logging_steps
self.logging_first_step = logging_first_step
self.log_on_each_node = log_on_each_node
self.logging_nan_inf_filter = logging_nan_inf_filter
self.include_num_input_tokens_seen = include_num_input_tokens_seen
self.log_level = log_level
self.log_level_replica = log_level_replica
self.disable_tqdm = disable_tqdm
self.report_to = report_to
self.run_name = run_name
self.project = project
self.trackio_space_id = trackio_space_id
self.trackio_bucket_id = trackio_bucket_id
self.trackio_static_space_id = trackio_static_space_id
self.eval_strategy = eval_strategy
self.eval_steps = eval_steps
self.eval_delay = eval_delay
self.per_device_eval_batch_size = per_device_eval_batch_size
self.prediction_loss_only = prediction_loss_only
self.eval_on_start = eval_on_start
self.eval_do_concat_batches = eval_do_concat_batches
self.eval_use_gather_object = eval_use_gather_object
self.eval_accumulation_steps = eval_accumulation_steps
self.include_for_metrics = include_for_metrics
self.batch_eval_metrics = batch_eval_metrics
self.save_only_model = save_only_model
self.save_strategy = save_strategy
self.save_steps = save_steps
self.save_on_each_node = save_on_each_node
self.save_total_limit = save_total_limit
self.enable_jit_checkpoint = enable_jit_checkpoint
self.push_to_hub = push_to_hub
self.hub_token = hub_token
self.hub_private_repo = hub_private_repo
self.hub_model_id = hub_model_id
self.hub_strategy = hub_strategy
self.hub_always_push = hub_always_push
self.hub_revision = hub_revision
self.load_best_model_at_end = load_best_model_at_end
self.metric_for_best_model = metric_for_best_model
self.greater_is_better = greater_is_better
self.ignore_data_skip = ignore_data_skip
self.restore_callback_states_from_checkpoint = restore_callback_states_from_checkpoint
self.full_determinism = full_determinism
self.seed = seed
self.data_seed = data_seed
self.use_cpu = use_cpu
self.accelerator_config = accelerator_config
self.parallelism_config = parallelism_config
self.dataloader_drop_last = dataloader_drop_last
self.dataloader_num_workers = dataloader_num_workers
self.dataloader_pin_memory = dataloader_pin_memory
self.dataloader_persistent_workers = dataloader_persistent_workers
self.dataloader_prefetch_factor = dataloader_prefetch_factor
self.remove_unused_columns = remove_unused_columns
self.label_names = label_names
self.train_sampling_strategy = train_sampling_strategy
self.length_column_name = length_column_name
self.ddp_find_unused_parameters = ddp_find_unused_parameters
self.ddp_bucket_cap_mb = ddp_bucket_cap_mb
self.ddp_broadcast_buffers = ddp_broadcast_buffers
self.ddp_static_graph = ddp_static_graph
self.ddp_backend = ddp_backend
self.ddp_timeout = ddp_timeout
self.fsdp = fsdp
self.fsdp_config = fsdp_config
self.deepspeed = deepspeed
self.debug = debug
self.skip_memory_metrics = skip_memory_metrics
self.do_train = do_train
self.do_eval = do_eval
self.do_predict = do_predict
self.resume_from_checkpoint = resume_from_checkpoint
self.warmup_ratio = warmup_ratio
self.logging_dir = logging_dir
self.local_rank = local_rank
Using this as is will yield a SyntaxError because of the <factory> issue
highlighted above. We can use on a “normal” class Trainer.
fields, annotations = get_constructor_field_annotations(Trainer, include_bases=False)
config_class_str = create_config_class_str(fields)
print(config_class_str)
class Config:
def __init__(self, model: Union = None, args: Union = None, data_collator: Union = None, train_dataset: Union = None, eval_dataset: Union = None, processing_class: Union = None, model_init: Union = None, compute_loss_func: Union = None, compute_metrics: Union = None, callbacks: Union = None, optimizers: tuple = (None, None), optimizer_cls_and_kwargs: Union = None, preprocess_logits_for_metrics: Union = None):
self.model = model
self.args = args
self.data_collator = data_collator
self.train_dataset = train_dataset
self.eval_dataset = eval_dataset
self.processing_class = processing_class
self.model_init = model_init
self.compute_loss_func = compute_loss_func
self.compute_metrics = compute_metrics
self.callbacks = callbacks
self.optimizers = optimizers
self.optimizer_cls_and_kwargs = optimizer_cls_and_kwargs
self.preprocess_logits_for_metrics = preprocess_logits_for_metrics
import transformers
import typing
import torch
from transformers import DataCollator
NoneType = type(None)
config_class_str = create_config_class_str(fields)
# Execute the generated class definition string
namespace = {}
exec(config_class_str, globals(), namespace)
# Extract the newly created class from the namespace
ConfigClass = namespace["Config"]
inspect.signature(ConfigClass.__init__)
<Signature (self, model: <class 'Union'> = None, args: <class 'Union'> = None, data_collator: <class 'Union'> = None, train_dataset: <class 'Union'> = None, eval_dataset: <class 'Union'> = None, processing_class: <class 'Union'> = None, model_init: <class 'Union'> = None, compute_loss_func: <class 'Union'> = None, compute_metrics: <class 'Union'> = None, callbacks: <class 'Union'> = None, optimizers: tuple = (None, None), optimizer_cls_and_kwargs: <class 'Union'> = None, preprocess_logits_for_metrics: <class 'Union'> = None)>