SEC EDGAR Data Extraction Automation Blueprint

```python
import requests
import time
import os
from bs4 import BeautifulSoup

# --- Configuration ---
SEC_BASE_URL = "https://www.sec.gov"
USER_AGENT = "YourCompanyNameOrEmail"
# For higher rate limits, consider an official SEC API key if available, or manage IPs.
# For this script, we'll rely on standard rate limits and exponential backoff.

# --- Helper Functions ---
def rate_limited_request(url, method='GET', **kwargs):
    """Performs a request with rate limiting and exponential backoff."""
    max_retries = 5
    for attempt in range(max_retries):
        try:
            response = requests.request(method, url, headers={'User-Agent': USER_AGENT}, **kwargs)
            response.raise_for_status() # Raise an exception for bad status codes
            
            # SEC rate limit is 10 requests/sec. Sleep for 0.11 seconds minimum.
            # Add a small buffer for safety.
            time.sleep(0.15)
            return response
        except requests.exceptions.RequestException as e:
            print(f"Attempt {attempt + 1} failed: {e}")
            if attempt < max_retries - 1:
                sleep_time = (2 ** attempt) * 0.5 # Exponential backoff
                print(f"Retrying in {sleep_time:.2f} seconds...")
                time.sleep(sleep_time)
            else:
                print("Max retries reached. Request failed.")
                raise
    return None

def search_sec_filings(cik, form_type='10-K', period_end_date=None):
    """Searches for filings for a given CIK and form type."""
    search_url = f"{SEC_BASE_URL}/edgar/searchengine/companysearch/" 
    params = {
        'company': cik,
        'formType': form_type,
        'date_range': 'custom',
        'start_date': '2000-01-01', # Example start date
        'end_date': period_end_date if period_end_date else '2023-12-31' # Example end date
    }
    
    try:
        # The search engine might not be a direct API, often it's a web page.
        # For programmatic access, direct indexing or specific EDGAR APIs are better.
        # This example uses a conceptual search endpoint.
        # A more robust solution would use https://data.sec.gov/submissions/CIKXXXXX.json
        print(f"Note: Direct search engine API is not officially documented. Using conceptual approach.")
        print(f"Consider using https://data.sec.gov/submissions/{cik}.json for structured metadata.")
        
        # Placeholder for actual search logic or data.sec.gov access
        # Example: Fetching submission data from data.sec.gov
        submissions_url = f"https://data.sec.gov/submissions/{cik}.json"
        response = rate_limited_request(submissions_url)
        if response and response.status_code == 200:
            data = response.json()
            filings = []
            for filing in data.get('filings', {}).get('recent', []):
                if filing['form'] == form_type:
                    filings.append({
                        'accessionNumber': filing['accessionNumber'],
                        'form': filing['form'],
                        'filingDate': filing['filingDate'],
                        'reportDate': filing.get('reportDate'),
                        'primaryDocument': filing['primaryDocument'],
                        'primaryDocLink': f"{SEC_BASE_URL}/Archives/edgar/data/{cik}/{filing['accessionNumber'].replace('-', '')}/{filing['primaryDocument']}"
                    })
            return filings
        else:
            print(f"Failed to fetch submissions for CIK {cik}: {response.status_code}")
            return []

    except Exception as e:
        print(f"Error searching SEC filings for CIK {cik}: {e}")
        return []

def download_filing(cik, accession_number, filing_type='htm'):
    """Downloads a specific filing document."""
    # Accession numbers often need hyphens removed for URL construction
    acc_no_clean = accession_number.replace('-', '')
    
    # Constructing the URL can be tricky. Let's assume HTML for now.
    # For XBRL, the path is different.
    # Example: https://www.sec.gov/Archives/edgar/data/1318647/0001318647-23-000083/0001318647-23-000083-index.htm
    # Or for primary document: https://www.sec.gov/Archives/edgar/data/1318647/0001318647-23-000083/a10k-20230331.htm
    
    # Let's try to infer the primary document link from search results if possible, or use a common pattern.
    # For simplicity, we'll use a placeholder. In a real scenario, you'd get this link from search_sec_filings.
    # The function signature should ideally take the direct URL to the filing.
    
    # This function is a placeholder. A real implementation would need the specific filing URL.
    # Let's simulate downloading the primary document if we had the accession number and CIK.
    # Example URL construction for primary document:
    # https://www.sec.gov/Archives/edgar/data/{CIK}/{ACCESSION_NUMBER_CLEAN}/{PRIMARY_DOCUMENT_NAME}
    
    # For now, let's return a placeholder and assume the caller has the URL.
    print(f"Placeholder: Downloading filing for CIK {cik}, Acc. No. {accession_number}")
    return "/path/to/downloaded/filing.htm"

def parse_html_filing(file_path):
    """Parses an HTML filing to extract specific data."""
    data = {}
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            soup = BeautifulSoup(f, 'lxml') # Use lxml for better performance
            
            # --- Example Extraction Logic (Highly Dependent on Filing Structure) ---
            # This is a very basic example targeting a hypothetical table.
            # You'll need to inspect actual filings to define precise selectors.
            
            # Extract company name (often in header/title)
            title_tag = soup.find('title')
            if title_tag:
                data['company_name'] = title_tag.get_text().split(' - ')[0].strip()

            # Example: Find a table that might contain 'Total Revenues' or similar
            # This is fragile and will break if table structures change.
            tables = soup.find_all('table')
            for table in tables:
                rows = table.find_all('tr')
                for row in rows:
                    cells = row.find_all(['td', 'th'])
                    if len(cells) > 1:
                        cell_text = [cell.get_text(strip=True) for cell in cells]
                        # Look for keywords in the first column
                        if 'Total Revenues' in cell_text[0]:
                            # Assuming value is in the next column
                            if len(cell_text) > 1:
                                data['total_revenues'] = cell_text[1]
                                break # Found it, exit inner loop
                if 'total_revenues' in data: break # Found it, exit outer loop
            
            # Add more extraction logic here for other fields (e.g., Net Income, Assets)
            # This requires deep inspection of filing HTML.
            
            print(f"Successfully parsed {file_path}. Extracted: {data}")
            return data
            
    except FileNotFoundError:
        print(f"Error: File not found at {file_path}")
        return None
    except Exception as e:
        print(f"Error parsing HTML file {file_path}: {e}")
        return None

# --- Main Execution Logic ---
def main():
    # Example Usage:
    # Replace with a real CIK (e.g., Apple Inc. is 320193)
    example_cik = "320193"
    
    print(f"Searching for 10-K filings for CIK: {example_cik}...")
    filings = search_sec_filings(example_cik, form_type='10-K')
    
    if not filings:
        print("No filings found or error occurred.")
        return

    print(f"Found {len(filings)} 10-K filings.")
    
    # Process the most recent filing for demonstration
    if filings:
        latest_filing = filings[0]
        print(f"Processing latest filing: {latest_filing['primaryDocument']} ({latest_filing['filingDate']})")
        
        # In a real scenario, download_filing would take the actual URL.
        # For this example, we'll simulate having a file path.
        # You'd typically download it first:
        # file_url = latest_filing['primaryDocLink']
        # downloaded_path = download_filing(example_cik, latest_filing['accessionNumber'], file_url)
        
        # --- Simulate downloading a file for parsing ---
        # Create a dummy HTML file for demonstration
        dummy_html_content = """
        <html><body>
            <title>XYZ Corp. - 10-K Filing</title>
            <h1>XYZ Corporation</h1>
            <p>For the fiscal year ended March 31, 2023</p>
            <table>
                <tr><th>Item</th><th>Value</th></tr>
                <tr><td>Total Revenues</td><td>$1,234,567,890</td></tr>
                <tr><td>Net Income</td><td>$234,567,890</td></tr>
            </table>
            <p>Other details...</p>
            <p>XBRL data would be in a separate file.</p>
        </body></html>
        """
        dummy_file_name = f"{latest_filing['accessionNumber'].replace('-', '')}_10-K.htm"
        dummy_file_path = os.path.join(os.getcwd(), dummy_file_name)
        with open(dummy_file_path, "w", encoding="utf-8") as f:
            f.write(dummy_html_content)
        print(f"Created dummy file: {dummy_file_path}")
        # --- End Simulation ---

        extracted_data = parse_html_filing(dummy_file_path)
        
        if extracted_data:
            print("\n--- Extracted Data ---")
            for key, value in extracted_data.items():
                print(f"{key}: {value}")
        else:
            print("Failed to extract data from the filing.")
        
        # Clean up dummy file
        os.remove(dummy_file_path)
        print(f"Removed dummy file: {dummy_file_path}")

if __name__ == "__main__":
    main()
```