diff --git a/esphome/components/web_server_idf/__init__.py b/esphome/components/web_server_idf/__init__.py
index 73c51f8cb5..506e1c5c13 100644
--- a/esphome/components/web_server_idf/__init__.py
+++ b/esphome/components/web_server_idf/__init__.py
@@ -8,8 +8,6 @@ CONFIG_SCHEMA = cv.All(
     cv.only_with_esp_idf,
 )
 
-AUTO_LOAD = ["web_server"]
-
 
 async def to_code(config):
     # Increase the maximum supported size of headers section in HTTP request packet to be processed by the server
diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp
index 6bfc49c675..90fdf720cd 100644
--- a/esphome/components/web_server_idf/web_server_idf.cpp
+++ b/esphome/components/web_server_idf/web_server_idf.cpp
@@ -9,10 +9,12 @@
 
 #include "utils.h"
 
+#include "web_server_idf.h"
+
+#ifdef USE_WEBSERVER
 #include "esphome/components/web_server/web_server.h"
 #include "esphome/components/web_server/list_entities.h"
-
-#include "web_server_idf.h"
+#endif  // USE_WEBSERVER
 
 namespace esphome {
 namespace web_server_idf {
@@ -273,6 +275,7 @@ void AsyncResponseStream::printf(const char *fmt, ...) {
   this->print(str);
 }
 
+#ifdef USE_WEBSERVER
 AsyncEventSource::~AsyncEventSource() {
   for (auto *ses : this->sessions_) {
     delete ses;  // NOLINT(cppcoreguidelines-owning-memory)
@@ -511,6 +514,7 @@ void AsyncEventSourceResponse::deferrable_send_state(void *source, const char *e
     }
   }
 }
+#endif
 
 }  // namespace web_server_idf
 }  // namespace esphome
diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h
index 13a3ef168d..d883c0ca9b 100644
--- a/esphome/components/web_server_idf/web_server_idf.h
+++ b/esphome/components/web_server_idf/web_server_idf.h
@@ -1,6 +1,7 @@
 #pragma once
 #ifdef USE_ESP_IDF
 
+#include "esphome/core/defines.h"
 #include <esp_http_server.h>
 
 #include <functional>
@@ -12,10 +13,12 @@
 #include <vector>
 
 namespace esphome {
+#ifdef USE_WEBSERVER
 namespace web_server {
 class WebServer;
 class ListEntitiesIterator;
 };  // namespace web_server
+#endif
 namespace web_server_idf {
 
 #define F(string_literal) (string_literal)
@@ -220,6 +223,7 @@ class AsyncWebHandler {
   virtual bool isRequestHandlerTrivial() { return true; }
 };
 
+#ifdef USE_WEBSERVER
 class AsyncEventSource;
 class AsyncEventSourceResponse;
 
@@ -307,10 +311,13 @@ class AsyncEventSource : public AsyncWebHandler {
   connect_handler_t on_connect_{};
   esphome::web_server::WebServer *web_server_;
 };
+#endif  // USE_WEBSERVER
 
 class DefaultHeaders {
   friend class AsyncWebServerRequest;
+#ifdef USE_WEBSERVER
   friend class AsyncEventSourceResponse;
+#endif
 
  public:
   // NOLINTNEXTLINE(readability-identifier-naming)