use anyhow::Result;
use serde::Deserialize;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use tokio::sync::OnceCell;

use super::Provider;
use crate::network::client::{get_with_retry, HttpClientConfig};
use crate::network::RateLimiter;
use crate::progress::ProgressReporter;

/// Sentinel value that asks the provider to resolve the most recent Common
/// Crawl index at runtime via `collinfo.json`.
pub(crate) const LATEST_INDEX_ALIAS: &str = "latest";

/// Hard ceiling on the number of CDX index pages we will fetch for one domain,
/// mirroring the Wayback guard. At the index server's default block size this
/// covers far more captures than any real domain has.
const CC_MAX_PAGES: usize = 10_000;

/// Validate that a Common Crawl index identifier matches the expected
/// `CC-MAIN-YYYY-WW` shape before we splice it into a URL path. This guards
/// against a hostile or corrupted `collinfo.json` causing path manipulation.
fn is_valid_cc_index_id(id: &str) -> bool {
    // Expected: "CC-MAIN-YYYY-WW" — fixed prefix, 4-digit year, 2-digit week.
    let Some(rest) = id.strip_prefix("CC-MAIN-") else {
        return false;
    };
    let mut parts = rest.split('-');
    let (Some(year), Some(week), None) = (parts.next(), parts.next(), parts.next()) else {
        return false;
    };
    year.len() == 4
        && year.chars().all(|c| c.is_ascii_digit())
        && week.len() == 2
        && week.chars().all(|c| c.is_ascii_digit())
}

#[derive(Clone)]
pub struct CommonCrawlProvider {
    index: String,
    /// Cached resolution of `LATEST_INDEX_ALIAS`. Shared across clones so the
    /// `collinfo.json` lookup happens at most once per run.
    resolved_index: Arc<OnceCell<String>>,
    include_subdomains: bool,
    proxy: Option<String>,
    proxy_auth: Option<String>,
    timeout: u64,
    retries: u32,
    random_agent: bool,
    insecure: bool,
    rate_limit: Option<RateLimiter>,
    #[cfg(test)]
    base_url: String,
}

#[derive(Deserialize)]
struct CCRecord {
    url: String,
}

/// Response shape of a `&showNumPages=true` probe — the index server reports
/// how many block-paginated pages a query spans. Extra fields (pageSize,
/// blocks) are ignored.
#[derive(Deserialize)]
struct CCPageInfo {
    pages: usize,
}

#[derive(Deserialize)]
struct CollInfoEntry {
    id: String,
}

impl CommonCrawlProvider {
    #[allow(dead_code)]
    pub fn new() -> Self {
        CommonCrawlProvider {
            index: "CC-MAIN-2026-17".to_string(),
            resolved_index: Arc::new(OnceCell::new()),
            include_subdomains: false,
            proxy: None,
            proxy_auth: None,
            timeout: 10,
            retries: 3,
            random_agent: true,
            insecure: false,
            rate_limit: None,
            #[cfg(test)]
            base_url: "https://index.commoncrawl.org".to_string(),
        }
    }

    /// Creates a provider instance with a specific Common Crawl index.
    ///
    /// Pass [`LATEST_INDEX_ALIAS`] (`"latest"`) to defer resolution until the
    /// first fetch — the provider will then look up `collinfo.json` and use
    /// the most recent published index.
    pub fn with_index(index: String) -> Self {
        CommonCrawlProvider {
            index,
            resolved_index: Arc::new(OnceCell::new()),
            include_subdomains: false,
            proxy: None,
            proxy_auth: None,
            timeout: 10,
            retries: 3,
            random_agent: true,
            insecure: false,
            rate_limit: None,
            #[cfg(test)]
            base_url: "https://index.commoncrawl.org".to_string(),
        }
    }

    /// Build an `HttpClientConfig` from the current provider settings.
    fn client_config(&self) -> HttpClientConfig {
        HttpClientConfig {
            timeout: self.timeout,
            insecure: self.insecure,
            random_agent: self.random_agent,
            proxy: self.proxy.clone(),
            proxy_auth: self.proxy_auth.clone(),
        }
    }

    fn index_base_url(&self) -> &str {
        #[cfg(test)]
        {
            &self.base_url
        }
        #[cfg(not(test))]
        {
            "https://index.commoncrawl.org"
        }
    }

    /// Resolve `self.index`, fetching `collinfo.json` once if the user passed
    /// the `latest` alias. The resolved value is memoised across all clones.
    async fn effective_index(&self) -> Result<String> {
        if !self.index.eq_ignore_ascii_case(LATEST_INDEX_ALIAS) {
            return Ok(self.index.clone());
        }

        let cached = self
            .resolved_index
            .get_or_try_init(|| async {
                let url = format!("{}/collinfo.json", self.index_base_url());
                let client = self.client_config().build_client()?;
                let body = get_with_retry(&client, &url, self.retries).await?;
                let entries: Vec<CollInfoEntry> = serde_json::from_str(&body)?;
                let id = entries
                    .into_iter()
                    .next()
                    .map(|e| e.id)
                    .ok_or_else(|| anyhow::anyhow!("collinfo.json returned no entries"))?;
                if !is_valid_cc_index_id(&id) {
                    return Err(anyhow::anyhow!(
                        "collinfo.json returned an unexpected index id: {id:?}"
                    ));
                }
                Ok(id)
            })
            .await?;

        Ok(cached.clone())
    }

    /// Build the index query without pagination params. `output=json` streams
    /// one JSON record per line; `&page=N` / `&showNumPages=true` are appended
    /// per request.
    fn query_base(&self, index: &str, domain: &str) -> String {
        let base_url = self.index_base_url();
        if self.include_subdomains {
            format!("{base_url}/{index}-index?url=*.{domain}/*&output=json")
        } else {
            format!("{base_url}/{index}-index?url={domain}/*&output=json")
        }
    }
}

impl Provider for CommonCrawlProvider {
    fn clone_box(&self) -> Box<dyn Provider> {
        Box::new(self.clone())
    }

    fn fetch_urls<'a>(
        &'a self,
        domain: &'a str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'a>> {
        self.fetch_urls_with_progress(domain, None)
    }

    fn fetch_urls_with_progress<'a>(
        &'a self,
        domain: &'a str,
        reporter: Option<ProgressReporter>,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'a>> {
        Box::pin(async move {
            let index = self.effective_index().await?;
            let query_base = self.query_base(&index, domain);
            let client = self.client_config().build_client()?;
            let limiter = self.rate_limit.as_ref();

            if let Some(r) = &reporter {
                r.detail("fetching…");
            }

            // The Common Crawl index server block-paginates: a single request
            // returns only the first block (historically ~15k records). We must
            // ask how many pages the query spans via `&showNumPages=true` and
            // then walk every page, or large domains are silently truncated to
            // their first block.
            if let Some(rl) = &limiter {
                rl.acquire().await;
            }
            let count_url = format!("{query_base}&showNumPages=true");
            let pages = match get_with_retry(&client, &count_url, self.retries).await {
                Ok(body) => serde_json::from_str::<CCPageInfo>(body.trim())
                    .map(|info| info.pages)
                    // A 200 that isn't a page-count document: fall back to a
                    // single page rather than giving up.
                    .unwrap_or(1),
                // The index returns 404 for a domain with no captures. Don't
                // hard-fail the probe; fall through to a single page=0 fetch so
                // genuine "no data" stays an empty/`Err` result exactly as the
                // single-request implementation produced.
                Err(_) => 1,
            };

            if pages == 0 {
                return Ok(Vec::new());
            }
            let pages = pages.min(CC_MAX_PAGES);

            let mut urls = Vec::new();
            for page in 0..pages {
                if let Some(rl) = &limiter {
                    rl.acquire().await;
                }
                let page_url = format!("{query_base}&page={page}");
                match get_with_retry(&client, &page_url, self.retries).await {
                    Ok(text) => {
                        // Common Crawl returns one JSON object per line.
                        for line in text.lines() {
                            if let Ok(record) = serde_json::from_str::<CCRecord>(line) {
                                urls.push(record.url);
                            }
                        }
                        if let Some(r) = &reporter {
                            r.detail(format!("{} URLs…", urls.len()));
                        }
                    }
                    Err(e) => {
                        // Nothing collected yet (e.g. a not-found domain whose
                        // first page 404s) is a hard failure, matching the old
                        // single-request behaviour. A mid-pagination failure
                        // keeps what we have and flags the result as partial.
                        if urls.is_empty() {
                            return Err(e);
                        }
                        if let Some(r) = &reporter {
                            r.mark_partial();
                        }
                        break;
                    }
                }
            }

            // Remove duplicates
            urls.sort();
            urls.dedup();

            Ok(urls)
        })
    }

    // Implement new trait methods
    fn with_subdomains(&mut self, include: bool) {
        self.include_subdomains = include;
    }

    fn with_proxy(&mut self, proxy: Option<String>) {
        self.proxy = proxy;
    }

    fn with_proxy_auth(&mut self, auth: Option<String>) {
        self.proxy_auth = auth;
    }

    // New method implementations for the additional features
    fn with_timeout(&mut self, seconds: u64) {
        self.timeout = seconds;
    }

    fn with_retries(&mut self, count: u32) {
        self.retries = count;
    }

    fn with_random_agent(&mut self, enabled: bool) {
        self.random_agent = enabled;
    }

    fn with_insecure(&mut self, enabled: bool) {
        self.insecure = enabled;
    }

    fn with_rate_limit(&mut self, rate_limit: Option<f32>) {
        self.rate_limit = RateLimiter::from_rate(rate_limit);
    }
}

#[cfg(test)]
pub struct MockCommonCrawlProvider {
    mock_urls: Vec<String>,
    include_subdomains: bool,
}

#[cfg(test)]
impl MockCommonCrawlProvider {
    pub fn new(mock_urls: Vec<String>) -> Self {
        MockCommonCrawlProvider {
            mock_urls,
            include_subdomains: false,
        }
    }

    pub fn with_subdomains(&mut self, include: bool) -> &mut Self {
        self.include_subdomains = include;
        self
    }
}

#[cfg(test)]
impl Provider for MockCommonCrawlProvider {
    fn clone_box(&self) -> Box<dyn Provider> {
        Box::new(Self {
            mock_urls: self.mock_urls.clone(),
            include_subdomains: self.include_subdomains,
        })
    }

    fn fetch_urls<'a>(
        &'a self,
        _domain: &'a str,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<String>>> + Send + 'a>> {
        let urls = self.mock_urls.clone();
        Box::pin(async move { Ok(urls) })
    }

    fn with_subdomains(&mut self, include: bool) {
        self.include_subdomains = include;
    }

    fn with_proxy(&mut self, _proxy: Option<String>) {}
    fn with_proxy_auth(&mut self, _auth: Option<String>) {}
    fn with_timeout(&mut self, _seconds: u64) {}
    fn with_retries(&mut self, _count: u32) {}
    fn with_random_agent(&mut self, _enabled: bool) {}
    fn with_insecure(&mut self, _enabled: bool) {}
    fn with_rate_limit(&mut self, _rate_limit: Option<f32>) {}
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_new_provider() {
        let provider = CommonCrawlProvider::new();
        assert_eq!(provider.index, "CC-MAIN-2026-17");
        assert!(!provider.include_subdomains);
        assert_eq!(provider.proxy, None);
        assert_eq!(provider.proxy_auth, None);
        assert_eq!(provider.timeout, 10);
        assert_eq!(provider.retries, 3);
        assert!(provider.random_agent);
        assert!(!provider.insecure);
        assert!(provider.rate_limit.is_none());
        assert_eq!(provider.base_url, "https://index.commoncrawl.org");
    }

    #[test]
    fn test_with_index() {
        let index = "CC-MAIN-2023-06".to_string();
        let provider = CommonCrawlProvider::with_index(index.clone());
        assert_eq!(provider.index, index);
        assert_eq!(provider.base_url, "https://index.commoncrawl.org");
    }

    #[test]
    fn test_with_subdomains() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_subdomains(true);
        assert!(provider.include_subdomains);
    }

    #[test]
    fn test_with_proxy() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_proxy(Some("http://proxy.example.com:8080".to_string()));
        assert_eq!(
            provider.proxy,
            Some("http://proxy.example.com:8080".to_string())
        );
    }

    #[test]
    fn test_with_proxy_auth() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_proxy_auth(Some("user:pass".to_string()));
        assert_eq!(provider.proxy_auth, Some("user:pass".to_string()));
    }

    #[test]
    fn test_with_timeout() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_timeout(60);
        assert_eq!(provider.timeout, 60);
    }

    #[test]
    fn test_with_retries() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_retries(5);
        assert_eq!(provider.retries, 5);
    }

    #[test]
    fn test_with_random_agent() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_random_agent(false);
        assert!(!provider.random_agent);
    }

    #[test]
    fn test_with_insecure() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_insecure(true);
        assert!(provider.insecure);
    }

    #[test]
    fn test_with_rate_limit() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_rate_limit(Some(2.5));
        assert!(provider.rate_limit.is_some());
    }

    #[test]
    fn test_clone_box() {
        let provider = CommonCrawlProvider::new();
        let _cloned = provider.clone_box();
        // Just testing that cloning works without error
    }

    #[test]
    fn test_client_config() {
        let mut provider = CommonCrawlProvider::new();
        provider.with_timeout(60);
        provider.with_insecure(true);
        provider.with_random_agent(false);
        provider.with_proxy(Some("http://proxy:8080".to_string()));
        provider.with_proxy_auth(Some("user:pass".to_string()));

        let config = provider.client_config();
        assert_eq!(config.timeout, 60);
        assert!(config.insecure);
        assert!(!config.random_agent);
        assert_eq!(config.proxy, Some("http://proxy:8080".to_string()));
        assert_eq!(config.proxy_auth, Some("user:pass".to_string()));
    }

    #[tokio::test]
    #[ignore = "Skip tests that make actual network requests in CI"]
    async fn test_fetch_urls_builds_correct_url_without_subdomains() {
        // Create a mock provider with predefined results
        let urls = vec![
            "https://example.com/page1".to_string(),
            "https://example.com/page2".to_string(),
        ];
        let provider = MockCommonCrawlProvider::new(urls.clone());

        // Test fetching URLs
        let result = provider.fetch_urls("example.com").await;
        assert!(result.is_ok());

        let fetched_urls = result.unwrap();
        assert_eq!(fetched_urls.len(), 2);
        assert_eq!(fetched_urls[0], "https://example.com/page1");
        assert_eq!(fetched_urls[1], "https://example.com/page2");
    }

    #[tokio::test]
    #[ignore = "Skip tests that make actual network requests in CI"]
    async fn test_fetch_urls_builds_correct_url_with_subdomains() {
        // Create a mock provider with predefined results for subdomain test
        let urls = vec![
            "https://sub1.example.com/page1".to_string(),
            "https://sub2.example.com/page2".to_string(),
        ];
        let mut provider = MockCommonCrawlProvider::new(urls.clone());
        provider.with_subdomains(true);

        // Test fetching URLs with subdomains enabled
        let result = provider.fetch_urls("example.com").await;
        assert!(result.is_ok());

        let fetched_urls = result.unwrap();
        assert_eq!(fetched_urls.len(), 2);
        assert_eq!(fetched_urls[0], "https://sub1.example.com/page1");
        assert_eq!(fetched_urls[1], "https://sub2.example.com/page2");
    }

    #[test]
    fn test_cc_record_deserialize() {
        let json = r#"{"url":"https://example.com/test"}"#;
        let record: CCRecord = serde_json::from_str(json).unwrap();
        assert_eq!(record.url, "https://example.com/test");
    }

    #[test]
    fn test_url_construction_without_subdomains() {
        // This test just verifies that the URL is constructed correctly without making a network request
        let provider = CommonCrawlProvider::new();

        // Use private helper function to check URL formation
        let url = format!(
            "https://index.commoncrawl.org/{}-index?url={}/*&output=json",
            provider.index, "example.com"
        );

        assert_eq!(
            url,
            "https://index.commoncrawl.org/CC-MAIN-2026-17-index?url=example.com/*&output=json"
        );
    }

    #[test]
    fn test_url_construction_with_subdomains() {
        // This test just verifies that the URL is constructed correctly without making a network request
        let mut provider = CommonCrawlProvider::new();
        provider.with_subdomains(true);

        // Use private helper function to check URL formation
        let url = format!(
            "https://index.commoncrawl.org/{}-index?url=*.{}/*&output=json",
            provider.index, "example.com"
        );

        assert_eq!(
            url,
            "https://index.commoncrawl.org/CC-MAIN-2026-17-index?url=*.example.com/*&output=json"
        );
    }

    #[tokio::test]
    async fn test_fetch_urls_integration() {
        let mut server = mockito::Server::new_async().await;
        let _mock = server
            .mock("GET", "/CC-MAIN-2026-17-index")
            .match_query(mockito::Matcher::AllOf(vec![
                mockito::Matcher::UrlEncoded("url".into(), "example.com/*".into()),
                mockito::Matcher::UrlEncoded("output".into(), "json".into()),
            ]))
            .with_status(200)
            .with_body("{\"url\": \"https://example.com/page1\"}\n{\"url\": \"https://example.com/page2\"}\n{\"url\": \"https://example.com/page1\"}")
            .create_async()
            .await;

        let mut provider = CommonCrawlProvider::new();
        // Override base_url to point to the mock server
        provider.base_url = server.url();

        let result = provider.fetch_urls("example.com").await;
        assert!(result.is_ok());

        let urls = result.unwrap();
        // Should be deduped and sorted
        assert_eq!(urls.len(), 2);
        assert_eq!(urls[0], "https://example.com/page1");
        assert_eq!(urls[1], "https://example.com/page2");
    }

    #[tokio::test]
    async fn test_fetch_urls_paginates_all_pages() {
        let mut server = mockito::Server::new_async().await;

        // The server reports the query spans two pages.
        let probe = server
            .mock("GET", "/CC-MAIN-2026-17-index")
            .match_query(mockito::Matcher::UrlEncoded(
                "showNumPages".into(),
                "true".into(),
            ))
            .with_status(200)
            .with_body(r#"{"pages": 2, "pageSize": 5, "blocks": 9}"#)
            .expect(1)
            .create_async()
            .await;
        let page0 = server
            .mock("GET", "/CC-MAIN-2026-17-index")
            .match_query(mockito::Matcher::UrlEncoded("page".into(), "0".into()))
            .with_status(200)
            .with_body("{\"url\": \"https://example.com/a\"}")
            .expect(1)
            .create_async()
            .await;
        let page1 = server
            .mock("GET", "/CC-MAIN-2026-17-index")
            .match_query(mockito::Matcher::UrlEncoded("page".into(), "1".into()))
            .with_status(200)
            .with_body("{\"url\": \"https://example.com/b\"}")
            .expect(1)
            .create_async()
            .await;

        let mut provider = CommonCrawlProvider::new();
        provider.base_url = server.url();

        let urls = provider.fetch_urls("example.com").await.unwrap();
        assert_eq!(
            urls,
            vec![
                "https://example.com/a".to_string(),
                "https://example.com/b".to_string(),
            ]
        );
        probe.assert();
        page0.assert();
        page1.assert();
    }

    #[tokio::test]
    async fn test_latest_alias_resolves_via_collinfo() {
        let mut server = mockito::Server::new_async().await;

        let collinfo = server
            .mock("GET", "/collinfo.json")
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                r#"[
                    {"id": "CC-MAIN-2099-01", "name": "Latest"},
                    {"id": "CC-MAIN-2098-50", "name": "Previous"}
                ]"#,
            )
            .expect(1)
            .create_async()
            .await;

        // Each fetch now issues a showNumPages probe plus the page fetch, so two
        // fetch_urls calls hit the index endpoint four times. The probe body
        // isn't a page-count document, so the provider falls back to one page.
        let index_mock = server
            .mock("GET", "/CC-MAIN-2099-01-index")
            .match_query(mockito::Matcher::AllOf(vec![
                mockito::Matcher::UrlEncoded("url".into(), "example.com/*".into()),
                mockito::Matcher::UrlEncoded("output".into(), "json".into()),
            ]))
            .with_status(200)
            .with_body("{\"url\": \"https://example.com/a\"}")
            .expect(4)
            .create_async()
            .await;

        let mut provider = CommonCrawlProvider::with_index(LATEST_INDEX_ALIAS.to_string());
        provider.base_url = server.url();

        // First fetch triggers collinfo lookup; second reuses the cached value.
        let first = provider.fetch_urls("example.com").await.unwrap();
        let second = provider.fetch_urls("example.com").await.unwrap();

        assert_eq!(first, vec!["https://example.com/a".to_string()]);
        assert_eq!(second, first);

        collinfo.assert();
        index_mock.assert();
    }

    #[tokio::test]
    async fn test_latest_alias_is_case_insensitive() {
        let mut server = mockito::Server::new_async().await;

        let _collinfo = server
            .mock("GET", "/collinfo.json")
            .with_status(200)
            .with_body(r#"[{"id": "CC-MAIN-2099-01"}]"#)
            .create_async()
            .await;

        let _idx = server
            .mock("GET", "/CC-MAIN-2099-01-index")
            .match_query(mockito::Matcher::Any)
            .with_status(200)
            .with_body("")
            .create_async()
            .await;

        let mut provider = CommonCrawlProvider::with_index("LATEST".to_string());
        provider.base_url = server.url();

        let result = provider.fetch_urls("example.com").await;
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_latest_alias_errors_on_empty_collinfo() {
        let mut server = mockito::Server::new_async().await;

        let _collinfo = server
            .mock("GET", "/collinfo.json")
            .with_status(200)
            .with_body("[]")
            .create_async()
            .await;

        let mut provider = CommonCrawlProvider::with_index(LATEST_INDEX_ALIAS.to_string());
        provider.base_url = server.url();
        provider.with_retries(0);

        let err = provider.fetch_urls("example.com").await.unwrap_err();
        assert!(err
            .to_string()
            .contains("collinfo.json returned no entries"));
    }

    #[tokio::test]
    async fn test_latest_alias_rejects_malformed_index_id() {
        let mut server = mockito::Server::new_async().await;

        let _collinfo = server
            .mock("GET", "/collinfo.json")
            .with_status(200)
            .with_body(r#"[{"id": "../../etc/passwd"}]"#)
            .create_async()
            .await;

        let mut provider = CommonCrawlProvider::with_index(LATEST_INDEX_ALIAS.to_string());
        provider.base_url = server.url();
        provider.with_retries(0);

        let err = provider.fetch_urls("example.com").await.unwrap_err();
        assert!(err.to_string().contains("unexpected index id"));
    }

    #[test]
    fn test_is_valid_cc_index_id() {
        assert!(is_valid_cc_index_id("CC-MAIN-2026-17"));
        assert!(is_valid_cc_index_id("CC-MAIN-2099-01"));
        assert!(!is_valid_cc_index_id("CC-MAIN-2026-1"));
        assert!(!is_valid_cc_index_id("CC-MAIN-2026"));
        assert!(!is_valid_cc_index_id("CC-MAIN-2026-17-extra"));
        assert!(!is_valid_cc_index_id("CC-MAIN-202X-17"));
        assert!(!is_valid_cc_index_id("../../etc/passwd"));
        assert!(!is_valid_cc_index_id(""));
    }
}
