diff --git a/fastapi/routing.py b/fastapi/routing.py
index 55aa7794b..03588655e 100644
--- a/fastapi/routing.py
+++ b/fastapi/routing.py
@@ -21,6 +21,8 @@ from typing import (
     Tuple,
     Type,
     Union,
+    get_args,
+    get_origin,
 )
 
 from annotated_doc import Doc
@@ -122,6 +124,18 @@ def request_response(
     return app
 
 
+def _contains_response(annotation: Any) -> bool:
+    if lenient_issubclass(annotation, Response):
+        return True
+
+    args = get_args(annotation)
+    for arg in args:
+        if _contains_response(arg):
+            return True
+
+    return False
+
+
 # Copy of starlette.routing.websocket_session modified to include the
 # dependencies' AsyncExitStack
 def websocket_session(
@@ -549,7 +563,9 @@ class APIRoute(routing.Route):
                     not lenient_issubclass(response_model, BaseModel)
                     and not dataclasses.is_dataclass(response_model)
                 ):
-                    if return_annotation is not None:
+                    if return_annotation is not None and not _contains_response(
+                        return_annotation
+                    ):
                         inferred = infer_response_model_from_ast(endpoint)
                         if inferred:
                             response_model = inferred
diff --git a/fastapi/utils.py b/fastapi/utils.py
index 9554ea97a..0db7381c2 100644
--- a/fastapi/utils.py
+++ b/fastapi/utils.py
@@ -429,6 +429,12 @@ def infer_response_model_from_ast(
     if not fields:
         return None
 
+    # Don't create a model if all fields are Any - this provides no additional
+    # type information compared to Dict[str, Any] and would override explicit
+    # type annotations unnecessarily
+    if all(field_type is Any for field_type, _ in fields.values()):
+        return None
+
     if PYDANTIC_V2:
         from pydantic import create_model
     else: