use anyhow::Result;
use async_recursion::async_recursion;
use async_trait::async_trait;
use reqwest::Client;
use roxmltree::Document;
use std::collections::HashSet;
use std::future::Future;
use std::pin::Pin;
use std::time::Duration;

use crate::network::client::HttpClientConfig;
use crate::network::RateLimiter;
use crate::providers::Provider;

/// Max nesting depth for sitemap-index → sitemap recursion. A hostile or
/// misconfigured index can chain or cycle indefinitely; this bounds it.
const MAX_SITEMAP_DEPTH: usize = 10;

/// Overall cap on URLs collected from one domain's sitemaps, so a giant (or
/// adversarial) sitemap tree can't grow memory without bound.
const MAX_SITEMAP_URLS: usize = 1_000_000;

/// Cap on the raw bytes read from a single sitemap document. Without it, a
/// hostile or misconfigured endpoint could stream gigabytes into memory before
/// any URL parsing happens (the per-URL cap only bounds the *parsed* output).
const MAX_SITEMAP_BYTES: usize = 50 * 1024 * 1024;

/// Read a response body but stop after `max` bytes, so an unbounded (or
/// deliberately huge) document can't exhaust memory. Reads incrementally via
/// `chunk()` rather than buffering the whole body up front.
async fn read_body_capped(mut resp: reqwest::Response, max: usize) -> Result<String> {
    let mut buf: Vec<u8> = Vec::new();
    while let Some(chunk) = resp.chunk().await? {
        let remaining = max.saturating_sub(buf.len());
        if remaining == 0 {
            break;
        }
        if chunk.len() > remaining {
            buf.extend_from_slice(&chunk[..remaining]);
            break;
        }
        buf.extend_from_slice(&chunk);
    }
    Ok(String::from_utf8_lossy(&buf).into_owned())
}

#[derive(Clone)]
pub struct SitemapProvider {
    timeout: Duration,
    retries: u32,
    random_agent: bool,
    proxy: Option<String>,
    proxy_auth: Option<String>,
    insecure: bool,
    rate_limit: Option<RateLimiter>,
}

impl SitemapProvider {
    pub fn new() -> Self {
        Self {
            timeout: Duration::from_secs(30),
            retries: 3,
            random_agent: false,
            proxy: None,
            proxy_auth: None,
            insecure: false,
            rate_limit: None,
        }
    }

    fn client_config(&self) -> HttpClientConfig {
        HttpClientConfig {
            timeout: self.timeout.as_secs(),
            insecure: self.insecure,
            random_agent: self.random_agent,
            proxy: self.proxy.clone(),
            proxy_auth: self.proxy_auth.clone(),
        }
    }

    /// Build the HTTP client via the shared config so it always sends a
    /// User-Agent (a UA-less request is rejected with 400 by some servers).
    fn build_client(&self) -> Result<Client> {
        self.client_config().build_client()
    }

    /// Recursively fetch and parse a sitemap (or sitemap index).
    ///
    /// `visited` records already-fetched sitemap URLs to break cycles
    /// (`A → A`, `A → B → A`); `depth` bounds straight-line nesting; and the
    /// caller stops feeding work once [`MAX_SITEMAP_URLS`] is reached. Together
    /// these stop a malicious sitemap from hanging the run or exhausting memory.
    #[async_recursion]
    async fn parse_sitemap(
        client: &Client,
        sitemap_url: &str,
        depth: usize,
        visited: &mut HashSet<String>,
        limiter: Option<&RateLimiter>,
    ) -> Result<Vec<String>> {
        if depth > MAX_SITEMAP_DEPTH {
            return Ok(Vec::new());
        }
        // A sitemap URL we've already fetched in this walk is a cycle — skip it.
        if !visited.insert(sitemap_url.to_string()) {
            return Ok(Vec::new());
        }

        // Pace nested-sitemap fetches: a sitemap index can chain to many child
        // sitemaps, so honor --rate-limit before each request.
        if let Some(rl) = limiter {
            rl.acquire().await;
        }
        let resp = client.get(sitemap_url).send().await?;
        if !resp.status().is_success() {
            return Ok(Vec::new());
        }

        // Only a genuine text sitemap should fall back to line-based parsing.
        // Decide that from the URL suffix / Content-Type *before* consuming the
        // body, so an XML endpoint that returns an HTML error page isn't mined
        // for stray `http` lines.
        let is_text_sitemap = sitemap_url.to_ascii_lowercase().ends_with(".txt")
            || resp
                .headers()
                .get(reqwest::header::CONTENT_TYPE)
                .and_then(|v| v.to_str().ok())
                .map(|ct| ct.to_ascii_lowercase().contains("text/plain"))
                .unwrap_or(false);

        let content = read_body_capped(resp, MAX_SITEMAP_BYTES).await?;
        let mut urls = Vec::new();

        match Document::parse(&content) {
            Ok(doc) => {
                // Check if this is a sitemap index file
                let is_sitemap_index = doc.root_element().has_tag_name("sitemapindex");

                if is_sitemap_index {
                    // This is a sitemap index file, so we need to process each sitemap
                    for sitemap_node in doc.descendants().filter(|n| n.has_tag_name("sitemap")) {
                        if urls.len() >= MAX_SITEMAP_URLS {
                            break;
                        }
                        if let Some(loc_node) =
                            sitemap_node.descendants().find(|n| n.has_tag_name("loc"))
                        {
                            if let Some(nested_sitemap_url) = loc_node.text() {
                                // Recursively fetch and parse nested sitemaps.
                                // Box::pin the future to avoid infinitely sized futures.
                                let nested_urls = Box::pin(Self::parse_sitemap(
                                    client,
                                    nested_sitemap_url,
                                    depth + 1,
                                    visited,
                                    limiter,
                                ))
                                .await?;
                                urls.extend(nested_urls);
                            }
                        }
                    }
                } else {
                    // This is a regular sitemap file
                    for url_node in doc.descendants().filter(|n| n.has_tag_name("url")) {
                        if urls.len() >= MAX_SITEMAP_URLS {
                            break;
                        }
                        if let Some(loc_node) =
                            url_node.descendants().find(|n| n.has_tag_name("loc"))
                        {
                            if let Some(url) = loc_node.text() {
                                urls.push(url.to_string());
                            }
                        }
                    }
                }
            }
            Err(_) => {
                // XML parse failed. Only treat it as a plain-text URL list when
                // the source actually is text (a .txt sitemap or text/plain);
                // otherwise this was an HTML/error page and yields no URLs.
                if is_text_sitemap {
                    for line in content.lines() {
                        if urls.len() >= MAX_SITEMAP_URLS {
                            break;
                        }
                        let line = line.trim();
                        if line.starts_with("http") {
                            urls.push(line.to_string());
                        }
                    }
                }
            }
        }

        Ok(urls)
    }
}

#[async_trait]
impl Provider for SitemapProvider {
    fn clone_box(&self) -> Box<dyn Provider> {
        Box::new(self.clone())
    }

    fn fetch_urls<'a>(
        &'a self,
        domain: &'a str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'a>> {
        Box::pin(async move {
            let client = self.build_client()?;
            let limiter = self.rate_limit.as_ref();
            let mut urls = Vec::new();
            // Shared across all candidate locations so a sitemap reachable from
            // more than one entry point is fetched at most once.
            let mut visited = HashSet::new();

            // Try common sitemap locations
            let sitemap_urls = vec![
                format!("https://{}/sitemap.xml", domain),
                format!("https://{}/sitemap_index.xml", domain),
                format!("https://{}/sitemap.txt", domain),
                format!("http://{}/sitemap.xml", domain),
                format!("http://{}/sitemap_index.xml", domain),
                format!("http://{}/sitemap.txt", domain),
            ];

            for sitemap_url in sitemap_urls {
                // Pace the candidate-location probes too: this loop fires up to
                // six back-to-back requests at the target.
                if let Some(rl) = &limiter {
                    rl.acquire().await;
                }
                let resp = client.get(&sitemap_url).send().await;

                if let Ok(resp) = resp {
                    if resp.status().is_success() {
                        // Found a valid sitemap, parse it
                        let sitemap_urls =
                            Self::parse_sitemap(&client, &sitemap_url, 0, &mut visited, limiter)
                                .await?;
                        urls.extend(sitemap_urls);
                    }
                }
            }

            Ok(urls)
        })
    }

    fn with_subdomains(&mut self, _include: bool) {}
    fn with_proxy(&mut self, proxy: Option<String>) {
        self.proxy = proxy;
    }
    fn with_proxy_auth(&mut self, auth: Option<String>) {
        self.proxy_auth = auth;
    }
    fn with_timeout(&mut self, seconds: u64) {
        self.timeout = Duration::from_secs(seconds);
    }
    fn with_retries(&mut self, count: u32) {
        self.retries = count;
    }
    fn with_random_agent(&mut self, enabled: bool) {
        self.random_agent = enabled;
    }
    fn with_insecure(&mut self, enabled: bool) {
        self.insecure = enabled;
    }
    fn with_rate_limit(&mut self, rate_limit: Option<f32>) {
        self.rate_limit = RateLimiter::from_rate(rate_limit);
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use mockito::Server;

    #[test]
    fn test_new_provider() {
        let provider = SitemapProvider::new();
        assert_eq!(provider.timeout, Duration::from_secs(30));
        assert_eq!(provider.retries, 3);
        assert!(!provider.random_agent);
        assert_eq!(provider.proxy, None);
        assert_eq!(provider.proxy_auth, None);
        assert!(!provider.insecure);
        assert!(provider.rate_limit.is_none());
    }

    #[test]
    fn test_with_rate_limit() {
        let mut provider = SitemapProvider::new();
        provider.with_rate_limit(Some(2.5));
        assert!(provider.rate_limit.is_some());
        // A non-positive rate means "no limiting".
        provider.with_rate_limit(Some(0.0));
        assert!(provider.rate_limit.is_none());
    }

    #[tokio::test]
    async fn test_rate_limit_paces_probe_requests() {
        // fetch_urls fires up to six back-to-back candidate-location probes;
        // --rate-limit must pace them. Before this provider honored the limiter
        // `with_rate_limit` was a no-op and the probes raced out instantly.
        let server = Server::new_async().await;
        let host = server.host_with_port();

        let mut provider = SitemapProvider::new();
        provider.with_rate_limit(Some(20.0)); // 50ms minimum interval

        let start = std::time::Instant::now();
        // No mocks: every probe 404s/fails fast, but each acquire() still paces.
        let _ = provider.fetch_urls(&host).await;

        // Six probes => five enforced ~50ms gaps (~250ms). A no-op limiter would
        // finish in a few ms; allow generous scheduler slack below that signal.
        assert!(
            start.elapsed() >= Duration::from_millis(150),
            "rate limit did not pace the sitemap probe requests: {:?}",
            start.elapsed()
        );
    }

    #[test]
    fn test_with_proxy() {
        let mut provider = SitemapProvider::new();
        provider.with_proxy(Some("http://proxy.example.com:8080".to_string()));
        assert_eq!(
            provider.proxy,
            Some("http://proxy.example.com:8080".to_string())
        );
    }

    #[test]
    fn test_with_proxy_auth() {
        let mut provider = SitemapProvider::new();
        provider.with_proxy_auth(Some("user:pass".to_string()));
        assert_eq!(provider.proxy_auth, Some("user:pass".to_string()));
    }

    #[test]
    fn test_with_timeout() {
        let mut provider = SitemapProvider::new();
        provider.with_timeout(60);
        assert_eq!(provider.timeout, Duration::from_secs(60));
    }

    #[test]
    fn test_with_retries() {
        let mut provider = SitemapProvider::new();
        provider.with_retries(5);
        assert_eq!(provider.retries, 5);
    }

    #[test]
    fn test_with_random_agent() {
        let mut provider = SitemapProvider::new();
        provider.with_random_agent(true);
        assert!(provider.random_agent);
        provider.with_random_agent(false);
        assert!(!provider.random_agent);
    }

    #[test]
    fn test_with_insecure() {
        let mut provider = SitemapProvider::new();
        provider.with_insecure(true);
        assert!(provider.insecure);
    }

    #[test]
    fn test_clone_box() {
        let provider = SitemapProvider::new();
        let _cloned = provider.clone_box();
        // Testing the existence of cloned object
    }

    #[test]
    fn test_build_client() {
        let provider = SitemapProvider::new();
        let client_result = provider.build_client();
        assert!(client_result.is_ok());
    }

    #[tokio::test]
    async fn test_sitemap_xml_parsing() {
        // Sample sitemap XML content for testing
        let sitemap_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://example.com/</loc>
    <lastmod>2023-01-01</lastmod>
    <changefreq>daily</changefreq>
    <priority>1.0</priority>
  </url>
  <url>
    <loc>https://example.com/about</loc>
    <lastmod>2023-01-02</lastmod>
    <changefreq>weekly</changefreq>
    <priority>0.8</priority>
  </url>
</urlset>"#;

        // Parse the sample sitemap
        let doc = Document::parse(sitemap_xml).unwrap();
        let mut urls = Vec::new();

        for url_node in doc.descendants().filter(|n| n.has_tag_name("url")) {
            if let Some(loc_node) = url_node.descendants().find(|n| n.has_tag_name("loc")) {
                if let Some(url) = loc_node.text() {
                    urls.push(url.to_string());
                }
            }
        }

        // Verify extracted URLs
        assert_eq!(urls.len(), 2);
        assert!(urls.contains(&"https://example.com/".to_string()));
        assert!(urls.contains(&"https://example.com/about".to_string()));
    }

    #[tokio::test]
    async fn test_sitemap_index_parsing() {
        // Sample sitemap index XML content for testing
        let sitemap_index_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap>
    <loc>https://example.com/sitemap1.xml</loc>
    <lastmod>2023-01-01</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://example.com/sitemap2.xml</loc>
    <lastmod>2023-01-02</lastmod>
  </sitemap>
</sitemapindex>"#;

        // Parse the sample sitemap index
        let doc = Document::parse(sitemap_index_xml).unwrap();
        let mut sitemap_urls = Vec::new();

        for sitemap_node in doc.descendants().filter(|n| n.has_tag_name("sitemap")) {
            if let Some(loc_node) = sitemap_node.descendants().find(|n| n.has_tag_name("loc")) {
                if let Some(url) = loc_node.text() {
                    sitemap_urls.push(url.to_string());
                }
            }
        }

        // Verify extracted sitemap URLs
        assert_eq!(sitemap_urls.len(), 2);
        assert!(sitemap_urls.contains(&"https://example.com/sitemap1.xml".to_string()));
        assert!(sitemap_urls.contains(&"https://example.com/sitemap2.xml".to_string()));
    }

    #[tokio::test]
    async fn test_fetch_urls_sitemap_xml() {
        let mut server = Server::new_async().await;
        let sitemap_xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://example.com/page1</loc>
  </url>
  <url>
    <loc>https://example.com/page2</loc>
  </url>
</urlset>"#;

        let _m = server
            .mock("GET", "/sitemap.xml")
            .with_status(200)
            .with_header("content-type", "application/xml")
            .with_body(sitemap_xml)
            .create_async()
            .await;

        let provider = SitemapProvider::new();
        // remove "http://" prefix from host_with_port if present (mockito shouldn't have it, but just in case)
        let host = server.host_with_port();
        // fetch_urls expects domain without protocol
        let result = provider.fetch_urls(&host).await;

        assert!(result.is_ok());
        let urls = result.unwrap();
        assert_eq!(urls.len(), 2);
        assert!(urls.contains(&"https://example.com/page1".to_string()));
        assert!(urls.contains(&"https://example.com/page2".to_string()));
    }

    #[tokio::test]
    async fn test_fetch_urls_sitemap_index() {
        let mut server = Server::new_async().await;
        let host = server.host_with_port();

        // Sitemap index pointing to a nested sitemap
        // We use the server address to ensure it calls back to our mock
        let sitemap_index = format!(
            r#"<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap>
    <loc>http://{}/nested.xml</loc>
  </sitemap>
</sitemapindex>"#,
            host
        );

        let nested_sitemap = r#"<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://example.com/nested-page</loc>
  </url>
</urlset>"#;

        let _m1 = server
            .mock("GET", "/sitemap_index.xml")
            .with_status(200)
            .with_header("content-type", "application/xml")
            .with_body(sitemap_index)
            .create_async()
            .await;

        let _m2 = server
            .mock("GET", "/nested.xml")
            .with_status(200)
            .with_header("content-type", "application/xml")
            .with_body(nested_sitemap)
            .create_async()
            .await;

        let provider = SitemapProvider::new();
        // fetch_urls will try sitemap.xml first (which will 404/fail or 501), then sitemap_index.xml
        // Mockito returns 501 for unmocked requests by default, but it's fine as long as fetch_urls continues
        let result = provider.fetch_urls(&host).await;

        assert!(result.is_ok());
        let urls = result.unwrap();
        assert_eq!(urls.len(), 1);
        assert!(urls.contains(&"https://example.com/nested-page".to_string()));
    }

    #[tokio::test]
    async fn test_fetch_urls_sitemap_index_self_cycle_terminates() {
        // A sitemap index that references itself must not loop forever.
        let mut server = Server::new_async().await;
        let host = server.host_with_port();

        let self_ref_index = format!(
            r#"<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap><loc>http://{host}/sitemap_index.xml</loc></sitemap>
</sitemapindex>"#
        );

        // Allow many hits; the cycle guard should make far fewer (ideally 1-2).
        let _m = server
            .mock("GET", "/sitemap_index.xml")
            .with_status(200)
            .with_header("content-type", "application/xml")
            .with_body(self_ref_index)
            .expect_at_most(5)
            .create_async()
            .await;

        let provider = SitemapProvider::new();
        // Completing at all (not hanging) is the assertion.
        let result = provider.fetch_urls(&host).await;
        assert!(result.is_ok());
        assert!(result.unwrap().is_empty());
    }

    #[tokio::test]
    async fn test_fetch_urls_sitemap_txt() {
        let mut server = Server::new_async().await;
        let sitemap_txt = "https://example.com/page1\nhttps://example.com/page2";

        let _m = server
            .mock("GET", "/sitemap.txt")
            .with_status(200)
            .with_body(sitemap_txt)
            .create_async()
            .await;

        let provider = SitemapProvider::new();
        let host = server.host_with_port();
        let result = provider.fetch_urls(&host).await;

        assert!(result.is_ok());
        let urls = result.unwrap();
        assert_eq!(urls.len(), 2);
        assert!(urls.contains(&"https://example.com/page1".to_string()));
        assert!(urls.contains(&"https://example.com/page2".to_string()));
    }

    #[tokio::test]
    async fn test_fetch_urls_xml_html_error_page_not_mined() {
        let mut server = Server::new_async().await;
        let host = server.host_with_port();

        // A non-XML HTML error page served where a .xml sitemap was expected.
        // The unescaped '&' makes XML parsing fail; the http line must NOT be
        // harvested, because the source is not a text sitemap.
        let html = "<html>\n<p>error & oops</p>\nhttp://attacker.example/inject\n</html>";
        let _m = server
            .mock("GET", "/sitemap.xml")
            .with_status(200)
            .with_header("content-type", "text/html")
            .with_body(html)
            .create_async()
            .await;

        let provider = SitemapProvider::new();
        let urls = provider.fetch_urls(&host).await.unwrap();
        assert!(
            !urls.iter().any(|u| u.contains("attacker.example")),
            "HTML error page was mined for URLs: {urls:?}"
        );
    }

    #[tokio::test]
    async fn test_fetch_urls_not_found() {
        let server = Server::new_async().await;
        // No mocks created, so all requests will return 501 (Not Implemented) or fail connection

        let provider = SitemapProvider::new();
        let host = server.host_with_port();
        let result = provider.fetch_urls(&host).await;

        assert!(result.is_ok());
        let urls = result.unwrap();
        assert!(urls.is_empty());
    }
}
