Implementation Guide: Automate FOIA Request Triage — Classify Requests, Retrieve Responsive Records, Apply Redactions & Generate Acknowledgment Letters

# Classifies incoming FOIA requests and routes to appropriate processing
# queue

# foia_intake_classifier.py
# Classifies incoming FOIA requests and routes to appropriate processing queue

from openai import AzureOpenAI
import os, json, datetime

client = AzureOpenAI(
    azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],
    api_key=os.environ["AZURE_OPENAI_KEY"],
    api_version="2024-08-01-preview"
)

# FOIA exemption reference (customize per agency regulations)
FOIA_EXEMPTIONS = {
    "Exemption 1": "Classified national defense or foreign policy information",
    "Exemption 2": "Internal agency personnel rules and practices",
    "Exemption 3": "Information prohibited from disclosure by another statute",
    "Exemption 4": "Trade secrets and confidential commercial/financial information",
    "Exemption 5": "Inter/intra-agency communications protected by deliberative process, attorney-client, or attorney work product privilege",
    "Exemption 6": "Personal privacy information — clearly unwarranted invasion of privacy",
    "Exemption 7A": "Law enforcement — interfere with ongoing investigation",
    "Exemption 7B": "Law enforcement — deprive of fair trial",
    "Exemption 7C": "Law enforcement — personal privacy of individuals",
    "Exemption 7D": "Law enforcement — confidential sources",
    "Exemption 7E": "Law enforcement — investigative techniques and procedures",
    "Exemption 7F": "Law enforcement — endanger life or physical safety",
    "Exemption 8": "Financial institution examination information",
    "Exemption 9": "Geological and geophysical information"
}

# Request complexity tiers
COMPLEXITY_TIERS = {
    "Simple": "Single record or narrow date range; likely <50 pages; no likely exemptions",
    "Standard": "Moderate scope; multiple systems may need searching; 1-2 likely exemptions",
    "Complex": "Broad scope; cross-system search required; multiple exemptions likely; >500 pages expected",
    "Voluminous": "Exceptionally broad scope; likely thousands of pages; may require rolling production"
}

def classify_foia_request(request: dict) -> dict:
    """Classify a FOIA request by topic, complexity, and likely exemptions."""

    classification_prompt = f"""You are a FOIA processing specialist at a federal agency.
Classify the following FOIA request for routing and processing planning.

AGENCY: {request.get('agency', 'Federal Agency')}
REQUEST DATE: {request.get('date_received', datetime.date.today().isoformat())}
REQUESTER CATEGORY: {request.get('requester_category', 'Unknown')}
REQUEST TEXT: {request.get('request_text', '')}
ATTACHMENTS: {request.get('attachments_description', 'None')}

Provide classification as JSON:
{{
  "topic_classification": "[primary topic — e.g., Contracts/Procurement, Personnel, Enforcement, Environmental, Financial, Policy, Other]",
  "topic_subtopics": ["list of specific subjects mentioned"],
  "complexity_tier": "[Simple/Standard/Complex/Voluminous]",
  "complexity_rationale": "[why this tier was assigned]",
  "estimated_responsive_pages": "[range estimate, e.g., '10-50' or '500+']",
  "likely_exemptions": [
    {{
      "exemption": "[Exemption number and name]",
      "rationale": "[why this exemption may apply — based on request text]",
      "confidence": "[High/Medium/Low]"
    }}
  ],
  "privacy_act_implicated": true/false,
  "privacy_act_rationale": "[why Privacy Act may apply, or 'Not applicable']",
  "fee_category": "[Commercial/News Media/Educational/Other — based on requester info]",
  "fee_waiver_requested": true/false,
  "expedited_processing_requested": true/false,
  "expedited_processing_basis": "[stated basis or 'Not requested']",
  "systems_to_search": ["suggested systems/locations to search based on request topic"],
  "recommended_search_terms": ["5-10 specific search terms to use in records retrieval"],
  "special_handling_flags": ["any flags — e.g., SENSITIVE TOPIC, LITIGATION RISK, MEDIA REQUESTER, CONGRESSIONAL INTEREST"],
  "routing_queue": "[Simple/Standard/Complex/Voluminous/Expedited/Contested]",
  "estimated_processing_days": "[statutory estimate: Simple=20 days, Complex=may require extension]"
}}

IMPORTANT:
- Do not make final exemption determinations — identify candidates for FOIA officer review
- Flag [FOIA OFFICER JUDGMENT REQUIRED] for any complex exemption analysis
- Flag [LEGAL REVIEW] if litigation, congressional, or sensitive law enforcement topics appear
- All output is for internal FOIA office use only — not for release to requester"""

    response = client.chat.completions.create(
        model=os.environ["AZURE_OPENAI_DEPLOYMENT"],
        messages=[
            {"role": "system", "content": "You are a federal FOIA processing specialist. Classify requests accurately and conservatively — when in doubt, assign higher complexity and flag for human review."},
            {"role": "user", "content": classification_prompt}
        ],
        temperature=0.0,
        max_tokens=2000,
        response_format={"type": "json_object"}
    )

    classification = json.loads(response.choices[0].message.content)
    classification["request_id"] = request.get("request_id", "UNKNOWN")
    classification["classification_timestamp"] = datetime.datetime.utcnow().isoformat()
    classification["classification_status"] = "AI-GENERATED — REQUIRES FOIA OFFICER REVIEW"

    return classification


def assign_tracking_number(agency_prefix: str) -> str:
    """Generate a unique FOIA tracking number."""
    year = datetime.date.today().year
    import random
    seq = random.randint(10000, 99999)  # In production, use a database sequence
    return f"{agency_prefix}-{year}-{seq:05d}"

# Searches M365 GCC High for records responsive to FOIA requests

# records_retrieval.py
# Searches M365 GCC High for records responsive to FOIA requests

import requests
import os, json

# Microsoft Graph API (Government endpoint)
GRAPH_ENDPOINT = "https://graph.microsoft.us/v1.0"

def get_graph_token() -> str:
    """Get access token for Microsoft Graph (Government)."""
    token_url = f"https://login.microsoftonline.us/{os.environ['AZURE_TENANT_ID']}/oauth2/v2.0/token"
    resp = requests.post(token_url, data={
        "client_id": os.environ["AZURE_CLIENT_ID"],
        "client_secret": os.environ["AZURE_CLIENT_SECRET"],
        "scope": "https://graph.microsoft.us/.default",
        "grant_type": "client_credentials"
    })
    resp.raise_for_status()
    return resp.json()["access_token"]


def create_compliance_search(
    request_id: str,
    search_terms: list,
    date_range: dict,
    custodians: list = None
) -> dict:
    """Create a Purview Content Search for FOIA records retrieval."""

    token = get_graph_token()
    headers = {
        "Authorization": f"Bearer {token}",
        "Content-Type": "application/json"
    }

    # Build KQL search query from AI-generated terms
    kql_terms = " OR ".join([f'"{term}"' for term in search_terms])
    date_filter = ""
    if date_range.get("start") and date_range.get("end"):
        date_filter = f" AND (date>={date_range['start']} AND date<={date_range['end']})"

    kql_query = f"({kql_terms}){date_filter}"

    # Create compliance search via Graph API
    # Endpoint: /compliance/ediscovery/cases/{caseId}/searches
    search_payload = {
        "displayName": f"FOIA-{request_id}",
        "description": f"Records retrieval for FOIA request {request_id}",
        "contentQuery": kql_query,
        "dataSourceScopes": "allTenantMailboxes,allTenantSites",  # Search all mailboxes and SharePoint
        "includeAttachments": True,
        "runStatisticsQuery": True
    }

    # Note: Full eDiscovery case creation requires additional API calls
    # This is a simplified representation — full implementation uses
    # the Microsoft Purview Compliance Portal API or PowerShell

    print(f"KQL Query for FOIA {request_id}:")
    print(kql_query)
    print(f"Estimated search scope: All M365 GCC High mailboxes and SharePoint sites")

    return {
        "request_id": request_id,
        "kql_query": kql_query,
        "search_terms": search_terms,
        "date_range": date_range,
        "status": "SEARCH_CREATED — Run in Purview Compliance Portal to retrieve results"
    }


def generate_search_terms_from_request(
    request_text: str,
    classification: dict
) -> list:
    """Refine and expand AI-generated search terms for Purview search."""

    refine_prompt = f"""Generate optimized Microsoft Purview / eDiscovery search terms for this FOIA request.

REQUEST TEXT: {request_text[:2000]}
INITIAL SEARCH TERMS: {classification.get('recommended_search_terms', [])}
TOPIC: {classification.get('topic_classification', '')}
DATE RANGE (if specified in request): Extract from request text or 'Not specified'

Generate:
1. EXACT PHRASE TERMS: Specific phrases that should appear in responsive documents (in quotes)
2. KEYWORD TERMS: Individual keywords likely in responsive documents
3. CUSTODIAN SUGGESTIONS: Job titles or roles likely to hold responsive records
4. DATE RANGE: Extract any date range specified in the request
5. FILE TYPE FILTERS: Any specific document types requested (.pdf, .xlsx, emails, etc.)
6. EXCLUSIONS: Terms to exclude to reduce non-responsive results (use NOT operator)

Format as JSON:
{{
  "exact_phrases": ["phrase 1", "phrase 2"],
  "keywords": ["keyword 1", "keyword 2"],
  "boolean_query": "complete KQL/boolean query combining all terms",
  "custodians": ["job title 1", "job title 2"],
  "date_start": "YYYY-MM-DD or null",
  "date_end": "YYYY-MM-DD or null",
  "file_types": [".pdf", ".docx", "email"],
  "exclusions": ["term to exclude"]
}}"""

    response = client.chat.completions.create(
        model=os.environ["AZURE_OPENAI_DEPLOYMENT"],
        messages=[{"role": "user", "content": refine_prompt}],
        temperature=0.0,
        max_tokens=1000,
        response_format={"type": "json_object"}
    )

    return json.loads(response.choices[0].message.content)

# Identifies candidate FOIA exemptions in responsive documents for FOIA
# officer review

# exemption_analyzer.py
# Identifies candidate FOIA exemptions in responsive documents for FOIA officer review

EXEMPTION_ANALYSIS_PROMPT = """You are a FOIA processing specialist providing decision support
to a FOIA officer. Review the following document excerpt and identify portions that MAY be
exempt from disclosure under FOIA.

IMPORTANT LEGAL CAVEAT:
- This is decision support ONLY — the FOIA officer makes all final exemption determinations
- Apply the DOJ's "foreseeable harm" standard: withhold only if foreseeable harm would result
- Apply the presumption of openness: when in doubt, lean toward disclosure
- Do not recommend withholding based on speculation — require articulable harm

AGENCY: {agency}
DOCUMENT TYPE: {doc_type}
DOCUMENT DATE: {doc_date}

FOIA EXEMPTIONS TO ANALYZE AGAINST:
{exemptions_text}

FOR EACH IDENTIFIED CANDIDATE EXEMPTION, PROVIDE:
1. LOCATION: Paragraph/sentence location in document
2. QUOTED TEXT: The exact text that may be exempt (max 100 words)
3. CANDIDATE EXEMPTION: Which exemption(s) may apply
4. RATIONALE: Why this exemption may apply — specific articulable harm
5. FORESEEABLE HARM: What harm would result from disclosure?
6. SEGREGABILITY: Can the exempt portion be redacted while releasing the rest?
7. CONFIDENCE: [High/Medium/Low] that a FOIA officer would apply this exemption
8. FOIA OFFICER ACTION: [REDACT CANDIDATE — Review / RELEASE — No exemption applicable / LEGAL REVIEW REQUIRED]

OUTPUT FORMAT: Structured list of candidate exemptions, then a segregability summary.

DOCUMENT EXCERPT:
{document_text}

REMINDER: Output flagged portions for FOIA officer review — do not make final determinations.
All output marked: [AI-ASSISTED ANALYSIS — FOIA OFFICER DETERMINATION REQUIRED]"""

def analyze_document_for_exemptions(
    document_text: str,
    document_metadata: dict,
    agency: str
) -> dict:
    """Analyze a document for candidate FOIA exemptions."""

    exemptions_formatted = "\n".join([
        f"{k}: {v}" for k, v in FOIA_EXEMPTIONS.items()
    ])

    response = client.chat.completions.create(
        model=os.environ["AZURE_OPENAI_DEPLOYMENT"],
        messages=[
            {"role": "system", "content": "You are a federal FOIA compliance specialist. Identify candidate exemptions accurately and conservatively. The presumption is always toward disclosure."},
            {"role": "user", "content": EXEMPTION_ANALYSIS_PROMPT.format(
                agency=agency,
                doc_type=document_metadata.get("type", "Unknown"),
                doc_date=document_metadata.get("date", "Unknown"),
                exemptions_text=exemptions_formatted,
                document_text=document_text[:5000]
            )}
        ],
        temperature=0.0,
        max_tokens=3000
    )

    return {
        "document_id": document_metadata.get("id", "Unknown"),
        "exemption_analysis": response.choices[0].message.content,
        "analysis_timestamp": datetime.datetime.utcnow().isoformat(),
        "status": "PENDING FOIA OFFICER REVIEW",
        "legal_disclaimer": "AI-generated candidate analysis only. FOIA officer must make all final withholding determinations."
    }

# Generates FOIA acknowledgment letters, fee notices, and response letters

# foia_letter_generator.py
# Generates FOIA acknowledgment letters, fee notices, and response letters

ACKNOWLEDGMENT_LETTER_PROMPT = """Generate a FOIA acknowledgment letter for the following request.
The letter must comply with the agency's FOIA regulations and DOJ FOIA guidelines.

AGENCY: {agency}
AGENCY ADDRESS: {agency_address}
REQUEST DATE: {request_date}
TRACKING NUMBER: {tracking_number}
REQUESTER NAME: {requester_name}
REQUESTER ADDRESS: {requester_address}
COMPLEXITY TIER: {complexity_tier}
FEE CATEGORY: {fee_category}
EXPEDITED PROCESSING REQUESTED: {expedited}
ESTIMATED PROCESSING DAYS: {est_days}

Generate a formal acknowledgment letter that includes:
1. Confirmation of receipt with tracking number
2. Summary of the request as understood (paraphrase, do not reproduce verbatim)
3. Complexity tier designation (Simple/Complex/Voluminous) if applicable
4. Estimated response timeframe (statutory 20 working days + extension notice if Complex)
5. Fee category determination and fee estimate if applicable
   - Notify if fees are expected to exceed $25 (requires requester notification per OMB guidelines)
   - Notify of fee waiver determination if waiver was requested
6. Expedited processing determination (if requested)
7. Contact information for questions
8. FOIA officer name and title (use [FOIA OFFICER NAME] placeholder)
9. Appeal rights notice

FORMAT: Formal government letter format. Professional, clear, and courteous.
Do not include any information about the substance of the records to be searched.

LETTER DATE: {letter_date}"""

def generate_acknowledgment_letter(
    request: dict,
    classification: dict,
    tracking_number: str,
    agency_info: dict
) -> str:
    """Generate a FOIA acknowledgment letter."""

    response = client.chat.completions.create(
        model=os.environ["AZURE_OPENAI_DEPLOYMENT"],
        messages=[
            {"role": "system", "content": "You are a federal FOIA officer generating compliance letters. Letters must be legally accurate, clearly written, and meet all statutory requirements."},
            {"role": "user", "content": ACKNOWLEDGMENT_LETTER_PROMPT.format(
                agency=agency_info.get("name"),
                agency_address=agency_info.get("address"),
                request_date=request.get("date_received"),
                tracking_number=tracking_number,
                requester_name=request.get("requester_name"),
                requester_address=request.get("requester_address"),
                complexity_tier=classification.get("complexity_tier"),
                fee_category=classification.get("fee_category"),
                expedited=classification.get("expedited_processing_requested"),
                est_days=classification.get("estimated_processing_days"),
                letter_date=datetime.date.today().strftime("%B %d, %Y")
            )}
        ],
        temperature=0.1,
        max_tokens=2000
    )

    return f"[DRAFT — REQUIRES FOIA OFFICER REVIEW AND SIGNATURE BEFORE MAILING]\n\n{response.choices[0].message.content}"


def calculate_foia_fees(
    classification: dict,
    fee_schedule: dict,
    estimated_pages: int
) -> dict:
    """Calculate FOIA processing fees per agency fee schedule."""

    requester_category = classification.get("fee_category", "Other")
    fee_waiver_requested = classification.get("fee_waiver_requested", False)

    # Standard fee categories per OMB FOIA fee guidelines
    fees = {}

    if requester_category == "Commercial":
        # Commercial requesters pay all fees
        fees["search_hours"] = 2  # Estimated
        fees["search_rate"] = fee_schedule.get("search_rate_clerical", 28.00)
        fees["review_hours"] = 1
        fees["review_rate"] = fee_schedule.get("review_rate", 65.00)
        fees["duplication_pages"] = max(0, estimated_pages - 0)  # No free pages
        fees["duplication_rate"] = fee_schedule.get("duplication_rate", 0.10)

    elif requester_category in ["News Media", "Educational", "Scientific"]:
        # News media / educational / scientific — only duplication fees (after 100 free pages)
        fees["search_hours"] = 0  # No search fee
        fees["search_rate"] = 0
        fees["review_hours"] = 0
        fees["review_rate"] = 0
        fees["duplication_pages"] = max(0, estimated_pages - 100)  # 100 free pages
        fees["duplication_rate"] = fee_schedule.get("duplication_rate", 0.10)

    else:
        # Other requesters (individuals, nonprofits) — search and duplication (after 2hrs/100pgs free)
        fees["search_hours"] = max(0, 2 - 2)  # 2 free hours
        fees["search_rate"] = fee_schedule.get("search_rate_clerical", 28.00)
        fees["duplication_pages"] = max(0, estimated_pages - 100)  # 100 free pages
        fees["duplication_rate"] = fee_schedule.get("duplication_rate", 0.10)

    total = (fees.get("search_hours", 0) * fees.get("search_rate", 0) +
             fees.get("review_hours", 0) * fees.get("review_rate", 0) +
             fees.get("duplication_pages", 0) * fees.get("duplication_rate", 0))

    fees["total_estimated"] = round(total, 2)
    fees["fee_waiver_requested"] = fee_waiver_requested
    fees["requires_advance_payment"] = total > 250  # >$250 requires advance payment per OMB
    fees["requires_notification"] = total > 25      # >$25 requires requester notification

    return fees

HTTP POST: Azure Function classify_foia_request

HTTP POST: Azure Function generate_acknowledgment_letter

HTTP POST: Azure Function generate_response_letter

# required under 5 U.S.C. § 552(e)

# Annual FOIA report statistics — required under 5 U.S.C. § 552(e)
# DOJ requires specific statistics in annual reports

ANNUAL_REPORT_STATS_QUERY = """
SELECT
  COUNT(*) as total_requests_received,
  SUM(CASE WHEN status = 'Full Release' THEN 1 ELSE 0 END) as full_releases,
  SUM(CASE WHEN status = 'Partial Release' THEN 1 ELSE 0 END) as partial_releases,
  SUM(CASE WHEN status = 'Full Denial' THEN 1 ELSE 0 END) as full_denials,
  SUM(CASE WHEN status = 'No Records' THEN 1 ELSE 0 END) as no_records,
  AVG(processing_days) as avg_processing_days,
  SUM(fees_collected) as total_fees_collected,
  SUM(fees_waived) as total_fees_waived,
  SUM(CASE WHEN exemption_1_applied = 1 THEN 1 ELSE 0 END) as exemption_1_count,
  SUM(CASE WHEN exemption_5_applied = 1 THEN 1 ELSE 0 END) as exemption_5_count,
  SUM(CASE WHEN exemption_6_applied = 1 THEN 1 ELSE 0 END) as exemption_6_count,
  SUM(CASE WHEN expedited_granted = 1 THEN 1 ELSE 0 END) as expedited_granted_count,
  SUM(CASE WHEN appeal_filed = 1 THEN 1 ELSE 0 END) as appeals_filed
FROM foia_cases
WHERE YEAR(date_received) = [REPORT_YEAR]
"""

SYSTEM PROMPT:
You are a federal FOIA appeals specialist drafting an administrative appeal response.
The requester has appealed the agency's initial determination.

APPEAL GROUNDS: {appeal_grounds}
ORIGINAL DETERMINATION: {original_determination}
EXEMPTIONS APPLIED: {exemptions_applied}
RESPONSIVE RECORDS: {records_summary}

Draft an administrative appeal response that:
1. Acknowledges the appeal and identifies the tracking numbers
2. States the agency's final determination on appeal
3. For each exemption upheld on appeal: provide a more detailed legal justification
   citing DOJ guidance and relevant case law
4. For any exemption NOT upheld on appeal: explain what records are being released
5. Notifies the requester of their right to seek judicial review in U.S. District Court
6. Provides contact information for the agency's FOIA Public Liaison

IMPORTANT:
- All legal citations must be verified by agency counsel before release
- Mark all case law citations with [VERIFY CITATION]
- This is a DRAFT — requires FOIA Appeals Officer and agency counsel review
- Final determination has legal consequence — must not be sent without attorney approval