CRE SEC Edgar Data Automation Blueprint 2026

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import time

# --- Configuration ---
BASE_URL = "https://www.sec.gov/Archives/edgar/data/"
FILINGS_TO_DOWNLOAD = [
    {"cik": "320193", "accession_number": "0001193125-23-175075"}, # Example: Prologis, Inc. (PLD) 10-K
    # Add more filings here, e.g., from your target companies and periods
]
KEYWORD_SECTIONS = {
    "development_costs": ["development costs", "construction expenses", "project costs"],
    "rental_income": ["rental income", "lease revenue", "operating income from rentals"],
    "property_value": ["fair value of real estate", "appraised value", "asset value"]
}

# --- Helper Functions ---
def get_filing_url(cik, accession_number):
    # Construct the URL for the filing's index page
    # This is a simplified example; actual SEC filing structures can be more complex
    # For 10-K/10-Q, the main document is often in an HTML file named like 'cik-YYYYMMDD-accession_number.htm'
    # We'll assume a common structure for demonstration
    # A more robust solution would parse the 'index.html' to find the correct document
    # For simplicity, let's assume we can directly get the HTML of the main document
    # In reality, you'd need to fetch index.html and parse it.
    return f"{BASE_URL}{cik}/{accession_number.replace('-', '')}/index.html"

def download_filing_html(url):
    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status() # Raise an exception for bad status codes
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"Error downloading {url}: {e}")
        return None

def parse_filing_content(html_content):
    if not html_content:
        return None
    soup = BeautifulSoup(html_content, 'html.parser')
    text_content = soup.get_text(separator=' ', strip=True)
    return text_content

def extract_data_by_keywords(text, keywords):
    extracted_snippets = []
    for keyword in keywords:
        # Use regex to find sentences or paragraphs containing the keyword
        # This is a basic approach; more advanced NLP would be better
        pattern = re.compile(f".*?({re.escape(keyword)}).*?", re.IGNORECASE | re.DOTALL)
        matches = pattern.finditer(text)
        for match in matches:
            # Extract a snippet around the keyword for context
            start = max(0, match.start() - 200) # 200 chars before
            end = min(len(text), match.end() + 200) # 200 chars after
            snippet = text[start:end].replace('\n', ' ').strip()
            extracted_snippets.append({"keyword": keyword, "snippet": snippet})
    return extracted_snippets

# --- Main Execution ---
all_extracted_data = []

for filing_info in FILINGS_TO_DOWNLOAD:
    cik = filing_info['cik']
    accession_number = filing_info['accession_number']
    filing_url = get_filing_url(cik, accession_number)
    print(f"Processing filing: CIK={cik}, Accession={accession_number}")

    html_content = download_filing_html(filing_url)
    if not html_content:
        continue

    text_content = parse_filing_content(html_content)
    if not text_content:
        continue

    filing_data = {"cik": cik, "accession_number": accession_number}
    for section, keywords in KEYWORD_SECTIONS.items():
        filing_data[section] = extract_data_by_keywords(text_content, keywords)

    all_extracted_data.append(filing_data)
    time.sleep(1) # Be polite to SEC servers

# --- Data Structuring and Output ---

# Convert to Pandas DataFrame for easier handling
# This part is simplified as nested data needs careful flattening
df_data = []
for item in all_extracted_data:
    base_info = {"cik": item["cik"], "accession_number": item["accession_number"]}
    for section, snippets in item.items():
        if section == "cik" or section == "accession_number":
            continue
        for snippet_info in snippets:
            row = base_info.copy()
            row["section"] = section
            row["keyword"] = snippet_info["keyword"]
            row["snippet"] = snippet_info["snippet"]
            df_data.append(row)

df = pd.DataFrame(df_data)

# Output to CSV
output_filename = "cre_sec_filings_extracted_data.csv"
df.to_csv(output_filename, index=False)

print(f"\nExtraction complete. Data saved to {output_filename}")
print("\n--- Sample of Extracted Data ---")
print(df.head())

# --- Manual Review Reminder ---
print("\nIMPORTANT: Please manually review the extracted data in the CSV file for accuracy.")
print("This script provides a starting point; refine keywords and parsing logic as needed.")