diff --git a/src/smolagents/types.py b/src/smolagents/types.py
index 0729b8c..d88293f 100644
--- a/src/smolagents/types.py
+++ b/src/smolagents/types.py
@@ -277,8 +277,8 @@ def handle_agent_output_types(output, output_type=None):
         # If the class does not have defined output, then we map according to the type
         for _k, _v in INSTANCE_TYPE_MAPPING.items():
             if isinstance(output, _k):
-                if not isinstance(
-                    output, object
+                if (
+                    _k is not object
                 ):  # avoid converting to audio if torch is not installed
                     return _v(output)
         return output