diff --git a/fastapi/routing.py b/fastapi/routing.py
index 6485b1ad5..c4d9504c1 100644
--- a/fastapi/routing.py
+++ b/fastapi/routing.py
@@ -545,11 +545,16 @@ class APIRoute(routing.Route):
                 response_model = None
             else:
                 response_model = return_annotation
-        
-        if response_model is not None and not lenient_issubclass(response_model, BaseModel):
-            inferred = infer_response_model_from_ast(endpoint)
-            if inferred:
-                response_model = inferred
+                if (
+                    response_model is None
+                    or (
+                        not lenient_issubclass(response_model, BaseModel)
+                        and not dataclasses.is_dataclass(response_model)
+                    )
+                ):
+                    inferred = infer_response_model_from_ast(endpoint)
+                    if inferred:
+                        response_model = inferred
 
         self.response_model = response_model
         self.summary = summary