Implementation Guide: Generate rfp responses and bid proposals from project specs

# Connect to SharePoint Online via PowerShell
Install-Module -Name PnP.PowerShell -Scope CurrentUser
Connect-PnPOnline -Url https://[tenant].sharepoint.com/sites/proposals -Interactive
# Create document library structure
New-PnPList -Title 'Won Proposals' -Template DocumentLibrary
New-PnPList -Title 'Lost Proposals' -Template DocumentLibrary
New-PnPList -Title 'Templates' -Template DocumentLibrary
New-PnPList -Title 'Certifications' -Template DocumentLibrary
New-PnPList -Title 'Crew Resumes' -Template DocumentLibrary
New-PnPList -Title 'Boilerplate Sections' -Template DocumentLibrary
New-PnPList -Title 'RFP Inbox' -Template DocumentLibrary
# Add metadata columns for proposal categorization
Add-PnPField -List 'Won Proposals' -DisplayName 'Project Type' -InternalName 'ProjectType' -Type Choice -Choices 'Commercial','Residential','Industrial','Infrastructure','Government'
Add-PnPField -List 'Won Proposals' -DisplayName 'Bid Type' -InternalName 'BidType' -Type Choice -Choices 'GC','Subcontractor','Design-Build','CM at Risk','IDIQ'
Add-PnPField -List 'Won Proposals' -DisplayName 'Contract Value' -InternalName 'ContractValue' -Type Currency
Add-PnPField -List 'Won Proposals' -DisplayName 'Win Date' -InternalName 'WinDate' -Type DateTime

# Assign Copilot licenses via Microsoft 365 Admin Center PowerShell
Connect-MgGraph -Scopes 'User.ReadWrite.All','Organization.Read.All'
# Get the Copilot SKU ID
Get-MgSubscribedSku | Where-Object {$_.SkuPartNumber -like '*Copilot*'} | Select-Object SkuId, SkuPartNumber
# Assign to specific users (repeat for each user)
$userId = (Get-MgUser -Filter "userPrincipalName eq 'estimator@contoso.com'").Id
$copilotSkuId = '<SKU-ID-FROM-ABOVE>'
Set-MgUserLicense -UserId $userId -AddLicenses @(@{SkuId=$copilotSkuId}) -RemoveLicenses @()
# Configure Sensitivity Labels for bid data
Connect-IPPSSession
New-Label -DisplayName 'Confidential - Bid Pricing' -Name 'ConfidentialBidPricing' -Tooltip 'Apply to all documents containing bid pricing, cost estimates, or margin calculations' -ContentType 'File,Email'
# Configure SharePoint search for Copilot indexing
# Ensure the Proposals site is included in Copilot's content sources
Set-PnPSearchConfiguration -Scope Site -Path 'searchconfig.xml'

# Login to Azure CLI
az login
# Create resource group for the AI proposal system
az group create --name rg-proposal-ai --location eastus2
# Create Azure OpenAI resource
az cognitiveservices account create \
  --name proposal-ai-openai \
  --resource-group rg-proposal-ai \
  --kind OpenAI \
  --sku S0 \
  --location eastus2 \
  --custom-domain proposal-ai-openai
# Deploy GPT-4.1 model for proposal generation
az cognitiveservices account deployment create \
  --name proposal-ai-openai \
  --resource-group rg-proposal-ai \
  --deployment-name gpt-41-proposal \
  --model-name gpt-4.1 \
  --model-version 2025-04-14 \
  --model-format OpenAI \
  --sku-capacity 80 \
  --sku-name Standard
# Deploy embedding model for vector generation
az cognitiveservices account deployment create \
  --name proposal-ai-openai \
  --resource-group rg-proposal-ai \
  --deployment-name text-embedding-ada-002 \
  --model-name text-embedding-ada-002 \
  --model-version 2 \
  --model-format OpenAI \
  --sku-capacity 120 \
  --sku-name Standard
# Get API keys
az cognitiveservices account keys list \
  --name proposal-ai-openai \
  --resource-group rg-proposal-ai
# Set up cost alert at $300/month
az consumption budget create \
  --budget-name proposal-ai-budget \
  --amount 300 \
  --category Cost \
  --resource-group rg-proposal-ai \
  --time-grain Monthly \
  --start-date 2025-08-01 \
  --end-date 2026-07-31 \
  --notification-key alert1 \
  --notification-threshold 80 \
  --notification-enabled true \
  --notification-contact-emails admin@msp.com finance@client.com

# Install Pinecone CLI / Python SDK
pip install pinecone-client
# Python script to initialize Pinecone index
cat > init_pinecone.py << 'EOF'
from pinecone import Pinecone, ServerlessSpec
import os

pc = Pinecone(api_key=os.environ['PINECONE_API_KEY'])

# Create the proposal knowledge base index
pc.create_index(
    name='proposal-knowledge-base',
    dimension=1536,  # Matches text-embedding-ada-002
    metric='cosine',
    spec=ServerlessSpec(
        cloud='azure',
        region='eastus2'
    )
)

 print('Index created successfully')

# Verify index is ready
index = pc.Index('proposal-knowledge-base')
print(index.describe_index_stats())
EOF

# Set environment variable and run
export PINECONE_API_KEY='your-api-key-here'
python init_pinecone.py

# Azure AD App Registration for SSO
az ad app create \
  --display-name 'AutoRFP.ai Production' \
  --web-redirect-uris 'https://app.autorfp.ai/saml/acs' \
  --sign-in-audience AzureADMyOrg

# Create Azure App Service for the RAG pipeline
az appservice plan create \
  --name proposal-rag-plan \
  --resource-group rg-proposal-ai \
  --sku B2 \
  --is-linux

az webapp create \
  --name proposal-rag-api \
  --resource-group rg-proposal-ai \
  --plan proposal-rag-plan \
  --runtime 'PYTHON:3.11'

# Configure app settings (environment variables)
az webapp config appsettings set \
  --name proposal-rag-api \
  --resource-group rg-proposal-ai \
  --settings \
    AZURE_OPENAI_ENDPOINT=https://proposal-ai-openai.openai.azure.com/ \
    AZURE_OPENAI_API_KEY=@Microsoft.KeyVault(VaultName=proposal-ai-kv;SecretName=openai-key) \
    AZURE_OPENAI_DEPLOYMENT=gpt-41-proposal \
    AZURE_OPENAI_EMBEDDING_DEPLOYMENT=text-embedding-ada-002 \
    PINECONE_API_KEY=@Microsoft.KeyVault(VaultName=proposal-ai-kv;SecretName=pinecone-key) \
    PINECONE_INDEX=proposal-knowledge-base \
    PINECONE_ENVIRONMENT=eastus2

# Create Key Vault for secrets
az keyvault create \
  --name proposal-ai-kv \
  --resource-group rg-proposal-ai \
  --location eastus2

# Store secrets
az keyvault secret set --vault-name proposal-ai-kv --name openai-key --value '<OPENAI_API_KEY>'
az keyvault secret set --vault-name proposal-ai-kv --name pinecone-key --value '<PINECONE_API_KEY>'

# Grant App Service access to Key Vault
az webapp identity assign --name proposal-rag-api --resource-group rg-proposal-ai
$principalId=$(az webapp identity show --name proposal-rag-api --resource-group rg-proposal-ai --query principalId -o tsv)
az keyvault set-policy --name proposal-ai-kv --object-id $principalId --secret-permissions get list

# Deploy application code (see custom_ai_components for full source)
# Clone repo and deploy
git clone https://github.com/[msp-org]/proposal-rag-pipeline.git
cd proposal-rag-pipeline
az webapp deployment source config-zip \
  --name proposal-rag-api \
  --resource-group rg-proposal-ai \
  --src deploy.zip

# First, ensure all proposals are in the SharePoint 'Won Proposals' library
# with metadata fields populated.

# Install dependencies
pip install langchain langchain-community langchain-openai pinecone-client \
  python-docx PyPDF2 openpyxl beautifulsoup4 tiktoken

# Run ingestion
python ingest_proposals.py \
  --sharepoint-site 'https://[tenant].sharepoint.com/sites/proposals' \
  --library 'Won Proposals' \
  --namespace 'won_proposals' \
  --chunk-size 1500 \
  --chunk-overlap 200

# Ingest boilerplate sections
python ingest_proposals.py \
  --sharepoint-site 'https://[tenant].sharepoint.com/sites/proposals' \
  --library 'Boilerplate Sections' \
  --namespace 'boilerplate' \
  --chunk-size 1000 \
  --chunk-overlap 150

# Ingest certifications and qualifications
python ingest_proposals.py \
  --sharepoint-site 'https://[tenant].sharepoint.com/sites/proposals' \
  --library 'Certifications' \
  --namespace 'certifications' \
  --chunk-size 800 \
  --chunk-overlap 100

# Ingest crew resumes and qualifications
python ingest_proposals.py \
  --sharepoint-site 'https://[tenant].sharepoint.com/sites/proposals' \
  --library 'Crew Resumes' \
  --namespace 'resumes' \
  --chunk-size 600 \
  --chunk-overlap 100

# Verify ingestion
python verify_knowledge_base.py --index proposal-knowledge-base

# Register Procore API application
# Navigate to https://developers.procore.com
# Create new App > Data Connection App
# Set redirect URI: https://proposal-rag-api.azurewebsites.net/auth/procore/callback
# Request scopes: read (projects, bids, documents, directory)

# Store Procore credentials in Key Vault
az keyvault secret set --vault-name proposal-ai-kv --name procore-client-id --value '<CLIENT_ID>'
az keyvault secret set --vault-name proposal-ai-kv --name procore-client-secret --value '<CLIENT_SECRET>'

# Add Procore settings to App Service
az webapp config appsettings set \
  --name proposal-rag-api \
  --resource-group rg-proposal-ai \
  --settings \
    PROCORE_CLIENT_ID=@Microsoft.KeyVault(VaultName=proposal-ai-kv;SecretName=procore-client-id) \
    PROCORE_CLIENT_SECRET=@Microsoft.KeyVault(VaultName=proposal-ai-kv;SecretName=procore-client-secret) \
    PROCORE_COMPANY_ID='<COMPANY_ID>'

# Test the Procore connection
curl -X GET 'https://proposal-rag-api.azurewebsites.net/api/procore/test' \
  -H 'Authorization: Bearer <APP_TOKEN>'

# Run initial project history sync
python sync_procore_projects.py --full-sync --company-id <COMPANY_ID>

# Store PandaDoc API key
az keyvault secret set --vault-name proposal-ai-kv --name pandadoc-api-key --value '<PANDADOC_API_KEY>'

# Add to App Service settings
az webapp config appsettings set \
  --name proposal-rag-api \
  --resource-group rg-proposal-ai \
  --settings \
    PANDADOC_API_KEY=@Microsoft.KeyVault(VaultName=proposal-ai-kv;SecretName=pandadoc-api-key)

# Test PandaDoc API connection
curl -X GET 'https://api.pandadoc.com/public/v1/templates' \
  -H 'Authorization: API-Key <PANDADOC_API_KEY>'

# Enable Azure AD Conditional Access for AI apps
# Policy 1: Require MFA for all AI platform access
az ad policy create --display-name 'Require MFA for AI Platforms' \
  --definition '{"conditions":{"applications":{"includeApplications":["AutoRFP-App-ID","PandaDoc-App-ID","proposal-rag-api-App-ID"]}},"grantControls":{"builtInControls":["mfa"]}}'

# Policy 2: Block access from non-compliant devices
# Configure in Azure AD Portal > Security > Conditional Access

# Enable Azure AD audit logging
az monitor diagnostic-settings create \
  --name 'ai-platform-audit' \
  --resource /subscriptions/<SUB_ID>/resourceGroups/rg-proposal-ai \
  --logs '[{"category":"AuditEvent","enabled":true,"retentionPolicy":{"enabled":true,"days":365}}]' \
  --workspace /subscriptions/<SUB_ID>/resourceGroups/rg-proposal-ai/providers/Microsoft.OperationalInsights/workspaces/proposal-ai-logs

# Create Log Analytics workspace
az monitor log-analytics workspace create \
  --resource-group rg-proposal-ai \
  --workspace-name proposal-ai-logs

# Enable Application Insights for the RAG API
az monitor app-insights component create \
  --app proposal-rag-insights \
  --location eastus2 \
  --resource-group rg-proposal-ai \
  --application-type web \
  --workspace proposal-ai-logs

# Configure data loss prevention - prevent bid data from leaving tenant
# In Microsoft Purview Compliance Center:
# 1. Create DLP policy for 'Financial Data - Bid Pricing'
# 2. Apply to SharePoint, OneDrive, Teams, Exchange
# 3. Detect patterns: dollar amounts + keywords (bid, proposal, estimate, unit price)
# 4. Action: Block external sharing, notify compliance admin

# rfp_parser.py
import os
import json
from datetime import datetime
from langchain_openai import AzureChatOpenAI
from langchain.prompts import ChatPromptTemplate
from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

class RFPParser:
    def __init__(self):
        self.llm = AzureChatOpenAI(
            azure_deployment=os.environ['AZURE_OPENAI_DEPLOYMENT'],
            azure_endpoint=os.environ['AZURE_OPENAI_ENDPOINT'],
            api_key=os.environ['AZURE_OPENAI_API_KEY'],
            api_version='2025-03-01-preview',
            temperature=0.1,
            max_tokens=4096
        )
        self.text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=8000,
            chunk_overlap=500,
            separators=['\n\n', '\n', '. ', ' ']
        )
        self.extraction_prompt = ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction RFP analyst. Extract ALL requirements from this RFP section.
Return a JSON object with the following structure:
{
  "project_name": "string",
  "project_type": "commercial|residential|industrial|infrastructure|government",
  "owner_entity": "string",
  "bid_type": "GC|subcontractor|design-build|CM_at_risk|IDIQ|JOC",
  "submission_deadline": "ISO 8601 datetime or null",
  "submission_method": "email|portal|physical|electronic_only",
  "submission_address_or_portal": "string",
  "pre_bid_conference": {"date": "string", "mandatory": true/false, "location": "string"},
  "project_location": {"address": "string", "city": "string", "state": "string"},
  "estimated_budget_range": "string or null",
  "project_duration": "string or null",
  "required_sections": [
    {"section_name": "string", "page_limit": "int or null", "required": true/false, "description": "string"}
  ],
  "qualification_requirements": [
    {"requirement": "string", "details": "string", "mandatory": true/false}
  ],
  "bonding_requirements": {"bid_bond": "string", "performance_bond": "string", "payment_bond": "string"},
  "insurance_minimums": [
    {"type": "string", "minimum_amount": "string"}
  ],
  "dbe_mbe_goals": {"percentage": "float or null", "type": "DBE|MBE|WBE|SDVOSB|HUBZone", "details": "string"},
  "prevailing_wage": true/false,
  "evaluation_criteria": [
    {"criterion": "string", "weight_percentage": "float or null", "description": "string"}
  ],
  "page_limit_total": "int or null",
  "formatting_requirements": {"font": "string", "font_size": "string", "margins": "string", "other": "string"},
  "questions_deadline": "ISO 8601 datetime or null",
  "questions_contact": {"name": "string", "email": "string", "phone": "string"},
  "key_personnel_required": ["string"],
  "similar_project_experience_required": {"count": "int", "years": "int", "min_value": "string"},
  "special_requirements": ["string"],
  "addenda": [{"number": "int", "date": "string", "summary": "string"}]
}

If information is not found in this section, use null. Do NOT invent or assume information.'''),
            ('human', 'Extract requirements from this RFP section:\n\n{rfp_text}')
        ])

    def load_document(self, file_path: str) -> str:
        ext = os.path.splitext(file_path)[1].lower()
        if ext == '.pdf':
            loader = PyPDFLoader(file_path)
        elif ext in ('.docx', '.doc'):
            loader = Docx2txtLoader(file_path)
        else:
            raise ValueError(f'Unsupported file type: {ext}')
        pages = loader.load()
        return '\n\n'.join([p.page_content for p in pages])

    def parse_rfp(self, file_path: str) -> dict:
        full_text = self.load_document(file_path)
        chunks = self.text_splitter.split_text(full_text)
        
        # Process each chunk and merge results
        all_extractions = []
        for chunk in chunks:
            chain = self.extraction_prompt | self.llm
            result = chain.invoke({'rfp_text': chunk})
            try:
                parsed = json.loads(result.content)
                all_extractions.append(parsed)
            except json.JSONDecodeError:
                # Try to extract JSON from markdown code blocks
                content = result.content
                if '                    content = content.split('                    parsed = json.loads(content)
                    all_extractions.append(parsed)
        
        # Merge extractions (later chunks fill in nulls from earlier)
        merged = self._merge_extractions(all_extractions)
        merged['_metadata'] = {
            'source_file': os.path.basename(file_path),
            'parsed_at': datetime.utcnow().isoformat(),
            'chunk_count': len(chunks),
            'full_text_length': len(full_text)
        }
        return merged

    def _merge_extractions(self, extractions: list) -> dict:
        if not extractions:
            return {}
        merged = extractions[0].copy()
        for ext in extractions[1:]:
            for key, value in ext.items():
                if value is not None and (key not in merged or merged[key] is None):
                    merged[key] = value
                elif isinstance(value, list) and isinstance(merged.get(key), list):
                    # Merge lists, avoiding duplicates
                    existing = [json.dumps(item, sort_keys=True) for item in merged[key]]
                    for item in value:
                        if json.dumps(item, sort_keys=True) not in existing:
                            merged[key].append(item)
        return merged

    def generate_compliance_matrix(self, requirements: dict) -> list:
        matrix = []
        for section in requirements.get('required_sections', []):
            matrix.append({
                'requirement': section['section_name'],
                'type': 'section',
                'mandatory': section.get('required', True),
                'page_limit': section.get('page_limit'),
                'status': 'pending',
                'assigned_to': None,
                'notes': section.get('description', '')
            })
        for qual in requirements.get('qualification_requirements', []):
            matrix.append({
                'requirement': qual['requirement'],
                'type': 'qualification',
                'mandatory': qual.get('mandatory', True),
                'status': 'pending',
                'assigned_to': None,
                'notes': qual.get('details', '')
            })
        return matrix

# ingest_proposals.py
import os
import sys
import argparse
import hashlib
import json
from datetime import datetime
from typing import List, Dict, Optional

from langchain_openai import AzureOpenAIEmbeddings
from langchain_community.document_loaders import PyPDFLoader, Docx2txtLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from pinecone import Pinecone
import tiktoken

class ProposalIngestionPipeline:
    def __init__(self):
        self.embeddings = AzureOpenAIEmbeddings(
            azure_deployment=os.environ.get('AZURE_OPENAI_EMBEDDING_DEPLOYMENT', 'text-embedding-ada-002'),
            azure_endpoint=os.environ['AZURE_OPENAI_ENDPOINT'],
            api_key=os.environ['AZURE_OPENAI_API_KEY'],
            api_version='2025-03-01-preview'
        )
        self.pc = Pinecone(api_key=os.environ['PINECONE_API_KEY'])
        self.index = self.pc.Index(os.environ.get('PINECONE_INDEX', 'proposal-knowledge-base'))
        self.tokenizer = tiktoken.encoding_for_model('text-embedding-ada-002')
        
        # Construction proposal section headers for intelligent chunking
        self.section_markers = [
            'EXECUTIVE SUMMARY', 'COVER LETTER', 'TABLE OF CONTENTS',
            'PROJECT UNDERSTANDING', 'SCOPE OF WORK', 'APPROACH AND METHODOLOGY',
            'PROJECT SCHEDULE', 'SCHEDULE', 'TIMELINE',
            'QUALIFICATIONS', 'EXPERIENCE', 'FIRM QUALIFICATIONS',
            'KEY PERSONNEL', 'PROJECT TEAM', 'ORGANIZATIONAL CHART',
            'SIMILAR PROJECTS', 'RELEVANT EXPERIENCE', 'PROJECT REFERENCES',
            'SAFETY', 'SAFETY PLAN', 'SAFETY PROGRAM',
            'QUALITY CONTROL', 'QUALITY ASSURANCE', 'QA/QC PLAN',
            'PRICING', 'COST PROPOSAL', 'BID FORM', 'FEE PROPOSAL',
            'APPENDIX', 'ATTACHMENTS', 'EXHIBITS',
            'DBE PARTICIPATION', 'SUBCONTRACTING PLAN',
            'VALUE ENGINEERING', 'ALTERNATES',
            'WARRANTY', 'MAINTENANCE', 'CLOSEOUT'
        ]

    def load_document(self, file_path: str) -> str:
        ext = os.path.splitext(file_path)[1].lower()
        if ext == '.pdf':
            loader = PyPDFLoader(file_path)
        elif ext in ('.docx', '.doc'):
            loader = Docx2txtLoader(file_path)
        else:
            print(f'Skipping unsupported file: {file_path}')
            return ''
        pages = loader.load()
        return '\n\n'.join([p.page_content for p in pages])

    def intelligent_chunk(self, text: str, chunk_size: int = 1500, chunk_overlap: int = 200) -> List[Dict]:
        """Chunk by proposal section when possible, fall back to recursive splitting."""
        sections = []
        current_section = 'GENERAL'
        current_text = ''
        
        lines = text.split('\n')
        for line in lines:
            line_upper = line.strip().upper()
            matched_section = None
            for marker in self.section_markers:
                if marker in line_upper and len(line.strip()) < 100:
                    matched_section = marker
                    break
            
            if matched_section:
                if current_text.strip():
                    sections.append({'section': current_section, 'text': current_text.strip()})
                current_section = matched_section
                current_text = line + '\n'
            else:
                current_text += line + '\n'
        
        if current_text.strip():
            sections.append({'section': current_section, 'text': current_text.strip()})
        
        # Now chunk each section
        splitter = RecursiveCharacterTextSplitter(
            chunk_size=chunk_size,
            chunk_overlap=chunk_overlap,
            separators=['\n\n', '\n', '. ', ' ']
        )
        
        chunks = []
        for section in sections:
            sub_chunks = splitter.split_text(section['text'])
            for i, chunk in enumerate(sub_chunks):
                token_count = len(self.tokenizer.encode(chunk))
                chunks.append({
                    'text': chunk,
                    'section': section['section'],
                    'chunk_index': i,
                    'total_chunks_in_section': len(sub_chunks),
                    'token_count': token_count
                })
        
        return chunks

    def generate_doc_id(self, file_path: str, chunk_index: int) -> str:
        file_hash = hashlib.md5(file_path.encode()).hexdigest()[:12]
        return f'{file_hash}-{chunk_index:04d}'

    def ingest_file(self, file_path: str, namespace: str, metadata: Optional[Dict] = None) -> int:
        print(f'Processing: {os.path.basename(file_path)}')
        text = self.load_document(file_path)
        if not text:
            return 0
        
        chunks = self.intelligent_chunk(text)
        if not chunks:
            print(f'  No chunks generated for {file_path}')
            return 0
        
        # Generate embeddings in batches of 16
        batch_size = 16
        total_upserted = 0
        
        for i in range(0, len(chunks), batch_size):
            batch = chunks[i:i + batch_size]
            texts = [c['text'] for c in batch]
            embeddings = self.embeddings.embed_documents(texts)
            
            vectors = []
            for j, (chunk, embedding) in enumerate(zip(batch, embeddings)):
                doc_id = self.generate_doc_id(file_path, i + j)
                vec_metadata = {
                    'text': chunk['text'],
                    'section': chunk['section'],
                    'chunk_index': chunk['chunk_index'],
                    'source_file': os.path.basename(file_path),
                    'token_count': chunk['token_count'],
                    'ingested_at': datetime.utcnow().isoformat()
                }
                if metadata:
                    vec_metadata.update(metadata)
                
                vectors.append({
                    'id': doc_id,
                    'values': embedding,
                    'metadata': vec_metadata
                })
            
            self.index.upsert(vectors=vectors, namespace=namespace)
            total_upserted += len(vectors)
        
        print(f'  Ingested {total_upserted} chunks from {os.path.basename(file_path)}')
        return total_upserted

    def ingest_directory(self, directory: str, namespace: str, metadata: Optional[Dict] = None) -> int:
        total = 0
        for root, dirs, files in os.walk(directory):
            for file in files:
                if file.lower().endswith(('.pdf', '.docx', '.doc')):
                    file_path = os.path.join(root, file)
                    total += self.ingest_file(file_path, namespace, metadata)
        print(f'\nTotal chunks ingested: {total}')
        return total


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Ingest proposals into vector database')
    parser.add_argument('--directory', required=True, help='Directory containing proposal documents')
    parser.add_argument('--namespace', required=True, help='Pinecone namespace (e.g., won_proposals, boilerplate)')
    parser.add_argument('--project-type', help='Project type metadata (commercial, residential, etc.)')
    parser.add_argument('--bid-type', help='Bid type metadata (GC, subcontractor, design-build, etc.)')
    parser.add_argument('--outcome', default='won', help='Bid outcome (won, lost)')
    parser.add_argument('--chunk-size', type=int, default=1500)
    parser.add_argument('--chunk-overlap', type=int, default=200)
    args = parser.parse_args()
    
    metadata = {'outcome': args.outcome}
    if args.project_type:
        metadata['project_type'] = args.project_type
    if args.bid_type:
        metadata['bid_type'] = args.bid_type
    
    pipeline = ProposalIngestionPipeline()
    pipeline.ingest_directory(args.directory, args.namespace, metadata)

# proposal_generator.py
import os
import json
from typing import List, Dict, Optional
from langchain_openai import AzureChatOpenAI, AzureOpenAIEmbeddings
from langchain.prompts import ChatPromptTemplate
from pinecone import Pinecone

class ProposalSectionGenerator:
    def __init__(self):
        self.llm = AzureChatOpenAI(
            azure_deployment=os.environ['AZURE_OPENAI_DEPLOYMENT'],
            azure_endpoint=os.environ['AZURE_OPENAI_ENDPOINT'],
            api_key=os.environ['AZURE_OPENAI_API_KEY'],
            api_version='2025-03-01-preview',
            temperature=0.4,
            max_tokens=4096
        )
        self.embeddings = AzureOpenAIEmbeddings(
            azure_deployment=os.environ.get('AZURE_OPENAI_EMBEDDING_DEPLOYMENT', 'text-embedding-ada-002'),
            azure_endpoint=os.environ['AZURE_OPENAI_ENDPOINT'],
            api_key=os.environ['AZURE_OPENAI_API_KEY'],
            api_version='2025-03-01-preview'
        )
        self.pc = Pinecone(api_key=os.environ['PINECONE_API_KEY'])
        self.index = self.pc.Index(os.environ.get('PINECONE_INDEX', 'proposal-knowledge-base'))

        # Section-specific prompt templates
        self.section_prompts = {
            'EXECUTIVE_SUMMARY': self._executive_summary_prompt(),
            'PROJECT_UNDERSTANDING': self._project_understanding_prompt(),
            'SCOPE_AND_APPROACH': self._scope_approach_prompt(),
            'QUALIFICATIONS': self._qualifications_prompt(),
            'SIMILAR_PROJECTS': self._similar_projects_prompt(),
            'SCHEDULE': self._schedule_prompt(),
            'SAFETY_PLAN': self._safety_plan_prompt(),
            'KEY_PERSONNEL': self._key_personnel_prompt(),
            'COVER_LETTER': self._cover_letter_prompt(),
            'GENERAL': self._general_section_prompt()
        }

    def retrieve_context(self, query: str, namespaces: List[str], 
                         top_k: int = 8, filter_dict: Optional[Dict] = None) -> List[Dict]:
        query_embedding = self.embeddings.embed_query(query)
        all_results = []
        for namespace in namespaces:
            results = self.index.query(
                vector=query_embedding,
                top_k=top_k,
                namespace=namespace,
                include_metadata=True,
                filter=filter_dict
            )
            all_results.extend([
                {
                    'text': match.metadata.get('text', ''),
                    'section': match.metadata.get('section', 'UNKNOWN'),
                    'source': match.metadata.get('source_file', 'unknown'),
                    'score': match.score,
                    'namespace': namespace
                }
                for match in results.matches
            ])
        # Sort by relevance score and deduplicate
        all_results.sort(key=lambda x: x['score'], reverse=True)
        return all_results[:top_k * 2]

    def generate_section(self, section_type: str, rfp_requirements: Dict,
                         project_context: Optional[str] = None,
                         additional_instructions: Optional[str] = None) -> Dict:
        # Build retrieval query from RFP requirements
        query_parts = [
            rfp_requirements.get('project_name', ''),
            rfp_requirements.get('project_type', ''),
            section_type.replace('_', ' ')
        ]
        if project_context:
            query_parts.append(project_context[:500])
        retrieval_query = ' '.join(filter(None, query_parts))

        # Determine which namespaces to search
        namespace_map = {
            'EXECUTIVE_SUMMARY': ['won_proposals', 'boilerplate'],
            'PROJECT_UNDERSTANDING': ['won_proposals'],
            'SCOPE_AND_APPROACH': ['won_proposals', 'boilerplate'],
            'QUALIFICATIONS': ['won_proposals', 'certifications', 'boilerplate'],
            'SIMILAR_PROJECTS': ['won_proposals'],
            'SCHEDULE': ['won_proposals'],
            'SAFETY_PLAN': ['boilerplate', 'won_proposals'],
            'KEY_PERSONNEL': ['resumes', 'won_proposals'],
            'COVER_LETTER': ['won_proposals', 'boilerplate'],
            'GENERAL': ['won_proposals', 'boilerplate']
        }
        namespaces = namespace_map.get(section_type, ['won_proposals', 'boilerplate'])

        # Retrieve relevant context
        context_docs = self.retrieve_context(retrieval_query, namespaces)
        context_text = '\n\n---\n\n'.join([
            f"[Source: {doc['source']} | Section: {doc['section']} | Relevance: {doc['score']:.2f}]\n{doc['text']}"
            for doc in context_docs
        ])

        # Select prompt template
        prompt_template = self.section_prompts.get(section_type, self.section_prompts['GENERAL'])

        # Build the full prompt
        chain = prompt_template | self.llm
        result = chain.invoke({
            'rfp_requirements': json.dumps(rfp_requirements, indent=2),
            'context': context_text,
            'project_context': project_context or 'No additional project context provided.',
            'additional_instructions': additional_instructions or 'None.',
            'section_type': section_type.replace('_', ' ').title()
        })

        return {
            'section_type': section_type,
            'content': result.content,
            'sources_used': [doc['source'] for doc in context_docs[:5]],
            'retrieval_scores': [doc['score'] for doc in context_docs[:5]],
            'token_count': len(result.content.split()) * 1.3  # rough estimate
        }

    def generate_full_proposal(self, rfp_requirements: Dict,
                               project_context: Optional[str] = None) -> Dict:
        sections_to_generate = []
        for req_section in rfp_requirements.get('required_sections', []):
            section_name = req_section.get('section_name', '').upper().replace(' ', '_')
            # Map RFP section names to our section types
            mapped = self._map_section_name(section_name)
            sections_to_generate.append({
                'type': mapped,
                'original_name': req_section.get('section_name'),
                'page_limit': req_section.get('page_limit'),
                'description': req_section.get('description', '')
            })
        
        if not sections_to_generate:
            # Default proposal structure
            sections_to_generate = [
                {'type': 'COVER_LETTER', 'original_name': 'Cover Letter', 'page_limit': 1},
                {'type': 'EXECUTIVE_SUMMARY', 'original_name': 'Executive Summary', 'page_limit': 2},
                {'type': 'PROJECT_UNDERSTANDING', 'original_name': 'Project Understanding', 'page_limit': 3},
                {'type': 'SCOPE_AND_APPROACH', 'original_name': 'Scope and Approach', 'page_limit': 5},
                {'type': 'QUALIFICATIONS', 'original_name': 'Firm Qualifications', 'page_limit': 3},
                {'type': 'SIMILAR_PROJECTS', 'original_name': 'Similar Project Experience', 'page_limit': 4},
                {'type': 'KEY_PERSONNEL', 'original_name': 'Key Personnel', 'page_limit': 4},
                {'type': 'SCHEDULE', 'original_name': 'Project Schedule', 'page_limit': 2},
                {'type': 'SAFETY_PLAN', 'original_name': 'Safety Plan', 'page_limit': 2}
            ]

        proposal = {
            'project_name': rfp_requirements.get('project_name', 'Untitled Project'),
            'generated_at': __import__('datetime').datetime.utcnow().isoformat(),
            'sections': []
        }

        for section_info in sections_to_generate:
            page_instruction = ''
            if section_info.get('page_limit'):
                page_instruction = f'Target approximately {section_info["page_limit"]} pages (roughly {section_info["page_limit"] * 400} words).'
            
            result = self.generate_section(
                section_type=section_info['type'],
                rfp_requirements=rfp_requirements,
                project_context=project_context,
                additional_instructions=f"{page_instruction} {section_info.get('description', '')}"
            )
            result['original_section_name'] = section_info['original_name']
            result['page_limit'] = section_info.get('page_limit')
            proposal['sections'].append(result)

        return proposal

    def _map_section_name(self, name: str) -> str:
        mapping = {
            'COVER_LETTER': 'COVER_LETTER', 'LETTER_OF_TRANSMITTAL': 'COVER_LETTER',
            'EXECUTIVE_SUMMARY': 'EXECUTIVE_SUMMARY', 'SUMMARY': 'EXECUTIVE_SUMMARY',
            'PROJECT_UNDERSTANDING': 'PROJECT_UNDERSTANDING', 'UNDERSTANDING': 'PROJECT_UNDERSTANDING',
            'SCOPE': 'SCOPE_AND_APPROACH', 'APPROACH': 'SCOPE_AND_APPROACH',
            'SCOPE_OF_WORK': 'SCOPE_AND_APPROACH', 'METHODOLOGY': 'SCOPE_AND_APPROACH',
            'SCOPE_AND_APPROACH': 'SCOPE_AND_APPROACH',
            'QUALIFICATIONS': 'QUALIFICATIONS', 'FIRM_QUALIFICATIONS': 'QUALIFICATIONS',
            'EXPERIENCE': 'QUALIFICATIONS',
            'SIMILAR_PROJECTS': 'SIMILAR_PROJECTS', 'RELEVANT_EXPERIENCE': 'SIMILAR_PROJECTS',
            'PROJECT_REFERENCES': 'SIMILAR_PROJECTS', 'REFERENCES': 'SIMILAR_PROJECTS',
            'SCHEDULE': 'SCHEDULE', 'PROJECT_SCHEDULE': 'SCHEDULE', 'TIMELINE': 'SCHEDULE',
            'SAFETY': 'SAFETY_PLAN', 'SAFETY_PLAN': 'SAFETY_PLAN', 'SAFETY_PROGRAM': 'SAFETY_PLAN',
            'KEY_PERSONNEL': 'KEY_PERSONNEL', 'PROJECT_TEAM': 'KEY_PERSONNEL',
            'ORGANIZATIONAL_CHART': 'KEY_PERSONNEL', 'TEAM': 'KEY_PERSONNEL'
        }
        return mapping.get(name, 'GENERAL')

    def _executive_summary_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting an Executive Summary for a bid proposal.

RULES:
- Write in first person plural ("we", "our firm", "our team")
- Lead with the most compelling differentiator (safety record, local presence, similar project experience)
- Reference specific project details from the RFP (project name, owner, location)
- Include 1-2 sentences about relevant past project successes with specific metrics (delivered $X project Y% under budget, Z days ahead of schedule)
- Close with a confident but professional commitment statement
- Match the tone and style of the reference materials provided
- DO NOT fabricate specific numbers, project names, or personnel names — use placeholders like [INSERT PROJECT NAME] if not available in context
- DO NOT include pricing or cost information in the executive summary

Here are excerpts from our past winning proposals for reference on tone, style, and content:\n{context}'''),
            ('human', '''Generate an Executive Summary for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _project_understanding_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Project Understanding section.

RULES:
- Demonstrate deep understanding of the owner\'s goals, not just repeating the scope
- Identify potential challenges specific to the project (site access, phasing, weather, utilities, adjacent operations)
- Show how your firm\'s experience with similar projects informs your understanding
- Reference specific RFP requirements to show thorough reading
- Use construction industry terminology accurately
- DO NOT fabricate details — use placeholders if information is not in the context

Reference materials from past proposals:\n{context}'''),
            ('human', '''Generate a Project Understanding section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _scope_approach_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Scope of Work and Approach/Methodology section.

RULES:
- Organize by construction phases (Pre-Construction, Mobilization, Construction, Closeout)
- Address each scope item from the RFP requirements explicitly
- Include specific methodologies for key activities (excavation, concrete, structural steel, MEP coordination, etc.)
- Describe your QA/QC approach for each major trade
- Address phasing and sequencing if multiple buildings or areas
- Reference BIM coordination, prefabrication, or lean construction practices where applicable
- DO NOT include pricing — focus on HOW the work will be done
- Use placeholders for specific details not in context

Reference materials from past proposals:\n{context}'''),
            ('human', '''Generate a Scope and Approach section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _qualifications_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Firm Qualifications section.

RULES:
- Lead with years in business, bonding capacity, and current insurance coverage
- Highlight relevant licenses and certifications
- Include EMR (Experience Modification Rate) and safety statistics
- Reference number and value of similar projects completed
- Mention any relevant DBE/MBE/WBE/SDVOSB certifications
- Include notable awards or recognitions
- Reference financial stability indicators (bonding capacity, banking relationships)
- Use actual data from the context; use [INSERT] placeholders for missing info

Reference materials from past proposals and certifications:\n{context}'''),
            ('human', '''Generate a Firm Qualifications section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _similar_projects_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Similar Projects / Relevant Experience section.

RULES:
- Present each project in a consistent format: Project Name, Owner, Location, Contract Value, Completion Date, Scope Description, Key Challenges, Outcomes
- Select projects most similar to the RFP\'s project type, size, and complexity
- Include quantifiable results (on-time delivery, budget adherence, safety record, change order percentage)
- Include owner reference contact information where available
- Aim for 3-5 similar projects unless the RFP specifies a different number
- Match the required experience (years, value, type) from the RFP evaluation criteria
- Use actual project data from context; use [INSERT] for missing details

Reference materials from past proposals:\n{context}'''),
            ('human', '''Generate a Similar Projects section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _schedule_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Project Schedule section.

RULES:
- Describe your scheduling methodology (CPM, Last Planner System, pull planning)
- Break the project into major milestones and phases
- Address the RFP\'s required completion date explicitly
- Discuss schedule risk mitigation (weather days, long-lead items, permit timelines)
- Reference your use of scheduling software (Primavera P6, Microsoft Project, Procore)
- Mention look-ahead scheduling and progress tracking approach
- Note: Do NOT create an actual Gantt chart — describe the approach and key milestones in narrative form

Reference materials from past proposals:\n{context}'''),
            ('human', '''Generate a Project Schedule section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _safety_plan_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Safety Plan section.

RULES:
- Lead with your firm\'s EMR (Experience Modification Rate) and OSHA recordable rates
- Reference your OSHA compliance program and any VPP Star status
- Describe site-specific safety planning process
- Address key hazards for this project type (fall protection, excavation, confined space, electrical, crane operations)
- Describe your safety training and orientation programs
- Mention drug testing policies and requirements
- Address COVID-19 / infectious disease protocols if relevant
- Reference safety technology (wearables, drone inspections, AI monitoring)
- Use actual safety statistics from context; use [INSERT] for missing data

Reference materials from past proposals and safety programs:\n{context}'''),
            ('human', '''Generate a Safety Plan section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _key_personnel_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the Key Personnel section.

RULES:
- Present each key team member in a consistent format: Name, Title, Years of Experience, Relevant Certifications, Education, Similar Project Experience
- Include: Project Executive, Project Manager, Superintendent, Safety Manager, and any other roles required by the RFP
- Highlight relevant certifications (PE, PMP, LEED AP, OSHA 30, CPR/First Aid)
- Reference specific similar projects each person has worked on
- Describe the reporting structure and communication plan
- Use actual personnel data from context; use [INSERT NAME/TITLE] for missing info

Reference materials from resumes and past proposals:\n{context}'''),
            ('human', '''Generate a Key Personnel section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _cover_letter_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting a Cover Letter / Letter of Transmittal.

RULES:
- Address to the specific person/entity named in the RFP
- Reference the RFP number, project name, and submission date
- Express enthusiasm for the project in 1-2 sentences
- Briefly state your top 3 differentiators (2-3 sentences total)
- Acknowledge all addenda received
- Include required statements (non-collusion, no conflict of interest) if required by the RFP
- Sign off with the company principal\'s name and title
- Keep to 1 page maximum
- Use actual company details from context

Reference materials from past proposals:\n{context}'''),
            ('human', '''Generate a Cover Letter for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

    def _general_section_prompt(self) -> ChatPromptTemplate:
        return ChatPromptTemplate.from_messages([
            ('system', '''You are an expert construction proposal writer drafting the {section_type} section of a bid proposal.

RULES:
- Write in first person plural ("we", "our firm", "our team")
- Use construction industry terminology accurately
- Reference specific RFP requirements to demonstrate compliance
- Draw from past proposal language and style in the reference materials
- DO NOT fabricate specific details — use [INSERT] placeholders for missing information
- Match the professional tone of past winning proposals

Reference materials from past proposals:\n{context}'''),
            ('human', '''Generate the {section_type} section for this RFP:\n\nRFP Requirements:\n{rfp_requirements}\n\nAdditional Project Context:\n{project_context}\n\nSpecial Instructions:\n{additional_instructions}''')
        ])

# app.py - Main FastAPI application
import os
import json
import tempfile
from datetime import datetime
from typing import Optional, List
from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel

from rfp_parser import RFPParser
from proposal_generator import ProposalSectionGenerator
from ingest_proposals import ProposalIngestionPipeline

app = FastAPI(
    title='Construction Proposal AI API',
    description='AI-powered RFP response and bid proposal generation for construction contractors',
    version='1.0.0'
)

app.add_middleware(
    CORSMiddleware,
    allow_origins=[os.environ.get('ALLOWED_ORIGINS', 'https://*.azurewebsites.net').split(',')],
    allow_credentials=True,
    allow_methods=['*'],
    allow_headers=['*']
)

rfp_parser = RFPParser()
proposal_generator = ProposalSectionGenerator()
ingestion_pipeline = ProposalIngestionPipeline()


class GenerateSectionRequest(BaseModel):
    section_type: str
    rfp_requirements: dict
    project_context: Optional[str] = None
    additional_instructions: Optional[str] = None


class GenerateFullProposalRequest(BaseModel):
    rfp_requirements: dict
    project_context: Optional[str] = None
    sections: Optional[List[str]] = None


class IngestRequest(BaseModel):
    file_url: str
    namespace: str
    metadata: Optional[dict] = None


@app.get('/health')
async def health_check():
    return {'status': 'healthy', 'timestamp': datetime.utcnow().isoformat()}


@app.post('/api/rfp/parse')
async def parse_rfp(file: UploadFile = File(...)):
    """Upload and parse an RFP document to extract requirements."""
    if not file.filename.lower().endswith(('.pdf', '.docx', '.doc')):
        raise HTTPException(status_code=400, detail='Unsupported file type. Upload PDF or DOCX.')
    
    with tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(file.filename)[1]) as tmp:
        content = await file.read()
        tmp.write(content)
        tmp_path = tmp.name
    
    try:
        requirements = rfp_parser.parse_rfp(tmp_path)
        compliance_matrix = rfp_parser.generate_compliance_matrix(requirements)
        return {
            'requirements': requirements,
            'compliance_matrix': compliance_matrix,
            'parsed_at': datetime.utcnow().isoformat()
        }
    finally:
        os.unlink(tmp_path)


@app.post('/api/proposal/section')
async def generate_section(request: GenerateSectionRequest):
    """Generate a single proposal section."""
    valid_sections = [
        'EXECUTIVE_SUMMARY', 'PROJECT_UNDERSTANDING', 'SCOPE_AND_APPROACH',
        'QUALIFICATIONS', 'SIMILAR_PROJECTS', 'SCHEDULE', 'SAFETY_PLAN',
        'KEY_PERSONNEL', 'COVER_LETTER', 'GENERAL'
    ]
    if request.section_type.upper() not in valid_sections:
        raise HTTPException(status_code=400, detail=f'Invalid section type. Valid types: {valid_sections}')
    
    result = proposal_generator.generate_section(
        section_type=request.section_type.upper(),
        rfp_requirements=request.rfp_requirements,
        project_context=request.project_context,
        additional_instructions=request.additional_instructions
    )
    return result


@app.post('/api/proposal/full')
async def generate_full_proposal(request: GenerateFullProposalRequest):
    """Generate a complete proposal from RFP requirements."""
    result = proposal_generator.generate_full_proposal(
        rfp_requirements=request.rfp_requirements,
        project_context=request.project_context
    )
    return result


@app.post('/api/knowledge-base/ingest')
async def ingest_document(file: UploadFile = File(...), namespace: str = 'won_proposals',
                          project_type: Optional[str] = None, bid_type: Optional[str] = None,
                          outcome: str = 'won'):
    """Ingest a document into the proposal knowledge base."""
    with tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(file.filename)[1]) as tmp:
        content = await file.read()
        tmp.write(content)
        tmp_path = tmp.name
    
    try:
        metadata = {'outcome': outcome}
        if project_type:
            metadata['project_type'] = project_type
        if bid_type:
            metadata['bid_type'] = bid_type
        
        count = ingestion_pipeline.ingest_file(tmp_path, namespace, metadata)
        return {
            'status': 'success',
            'chunks_ingested': count,
            'namespace': namespace,
            'file': file.filename
        }
    finally:
        os.unlink(tmp_path)


@app.get('/api/knowledge-base/stats')
async def knowledge_base_stats():
    """Get statistics about the knowledge base."""
    stats = ingestion_pipeline.index.describe_index_stats()
    return {
        'total_vectors': stats.total_vector_count,
        'namespaces': {ns: data.vector_count for ns, data in stats.namespaces.items()},
        'dimension': stats.dimension
    }

fastapi==0.115.0
uvicorn==0.30.0
python-multipart==0.0.9
langchain==0.3.0
langchain-openai==0.2.0
langchain-community==0.3.0
pinecone-client==5.0.0
python-docx==1.1.0
PyPDF2==3.0.1
tiktoken==0.7.0
pydantic==2.9.0

web: uvicorn app:app --host 0.0.0.0 --port 8000

# go_no_go_analyzer.py
import os
import json
from langchain_openai import AzureChatOpenAI
from langchain.prompts import ChatPromptTemplate
from typing import Dict, Optional

class GoNoGoAnalyzer:
    def __init__(self):
        self.llm = AzureChatOpenAI(
            azure_deployment=os.environ['AZURE_OPENAI_DEPLOYMENT'],
            azure_endpoint=os.environ['AZURE_OPENAI_ENDPOINT'],
            api_key=os.environ['AZURE_OPENAI_API_KEY'],
            api_version='2025-03-01-preview',
            temperature=0.2,
            max_tokens=3000
        )
        self.prompt = ChatPromptTemplate.from_messages([
            ('system', '''You are a construction business development advisor performing a Go/No-Go analysis on an RFP opportunity.

Analyze the RFP requirements against the contractor profile and produce a structured recommendation.

Score each dimension from 1-10 (10 = strongest fit) and provide a brief justification.

Dimensions to evaluate:
1. PROJECT TYPE FIT: How well does this project type match the firm's core competencies?
2. SIZE/VALUE FIT: Is the project value in the firm's sweet spot? Not too small (overhead ratio) or too large (bonding/capacity)?
3. GEOGRAPHIC FIT: Is the project location within the firm's operating radius? Are they licensed in that jurisdiction?
4. SCHEDULE FIT: Can the firm meet the timeline given current backlog and resource availability?
5. QUALIFICATION MATCH: Does the firm meet all mandatory qualification requirements (certifications, experience, similar projects)?
6. RELATIONSHIP: Does the firm have an existing relationship with the owner, architect, or CM?
7. COMPETITION LEVEL: How competitive is this bid likely to be? Are there incumbent advantages?
8. PROFITABILITY POTENTIAL: Based on project type and delivery method, what's the likely margin?
9. STRATEGIC VALUE: Does this project open new markets, clients, or project types?
10. RESOURCE AVAILABILITY: Are the key personnel and crews available for this project?

Return a JSON object with this structure:
{
  "recommendation": "GO" | "NO-GO" | "CONDITIONAL GO",
  "overall_score": float (average of all dimension scores),
  "confidence": "HIGH" | "MEDIUM" | "LOW",
  "dimensions": [
    {"name": "string", "score": int, "justification": "string"}
  ],
  "key_risks": ["string"],
  "key_strengths": ["string"],
  "conditions": ["string (only if CONDITIONAL GO)"],
  "estimated_proposal_effort_hours": int,
  "estimated_win_probability": "HIGH (>40%) | MEDIUM (20-40%) | LOW (<20%)",
  "summary": "2-3 sentence recommendation summary"
}

Guidelines:
- Recommend GO if overall score >= 6.5
- Recommend NO-GO if overall score < 4.5
- Recommend CONDITIONAL GO if 4.5-6.5 with identifiable mitigations
- Be realistic about win probability — most competitive bids have <25% win rates
- Factor in the opportunity cost of pursuing this bid vs. other opportunities'''),
            ('human', '''Analyze this opportunity:\n\nRFP Requirements:\n{rfp_requirements}\n\nContractor Profile:\n{contractor_profile}\n\nCurrent Backlog & Resources:\n{current_status}\n\nAdditional Context:\n{additional_context}''')
        ])

    def analyze(self, rfp_requirements: Dict, contractor_profile: Dict,
                current_status: Optional[str] = None,
                additional_context: Optional[str] = None) -> Dict:
        chain = self.prompt | self.llm
        result = chain.invoke({
            'rfp_requirements': json.dumps(rfp_requirements, indent=2),
            'contractor_profile': json.dumps(contractor_profile, indent=2),
            'current_status': current_status or 'No current backlog information provided.',
            'additional_context': additional_context or 'None.'
        })
        
        try:
            analysis = json.loads(result.content)
        except json.JSONDecodeError:
            content = result.content
            if '```json' in content:
                content = content.split('```json')[1].split('```')[0].strip()
                analysis = json.loads(content)
            else:
                analysis = {'error': 'Failed to parse analysis', 'raw': result.content}
        
        return analysis


# Example contractor_profile structure:
# {
#   "company_name": "ABC Construction Co.",
#   "years_in_business": 25,
#   "annual_revenue": "$35M",
#   "bonding_capacity": "$20M single / $50M aggregate",
#   "emr": 0.78,
#   "core_competencies": ["K-12 Education", "Healthcare", "Commercial Office", "Multifamily"],
#   "licenses": ["CA Class A General", "NV Class A General"],
#   "certifications": ["LEED AP on staff", "OSHA VPP Star"],
#   "operating_radius_miles": 150,
#   "home_office_city": "Sacramento, CA",
#   "avg_project_size": "$2M-$15M",
#   "current_backlog": "$45M",
#   "available_supers": 2,
#   "available_pms": 1
# }

# proposal_formatter.py
import os
import json
import requests
from datetime import datetime
from typing import Dict, List, Optional
from docx import Document
from docx.shared import Inches, Pt, Cm, RGBColor
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.enum.section import WD_ORIENT

class ProposalWordFormatter:
    def __init__(self, template_path: Optional[str] = None):
        if template_path and os.path.exists(template_path):
            self.doc = Document(template_path)
        else:
            self.doc = Document()
            self._setup_default_styles()

    def _setup_default_styles(self):
        style = self.doc.styles['Normal']
        font = style.font
        font.name = 'Calibri'
        font.size = Pt(11)
        font.color.rgb = RGBColor(0x33, 0x33, 0x33)
        
        paragraph_format = style.paragraph_format
        paragraph_format.space_after = Pt(6)
        paragraph_format.line_spacing = 1.15

        for level in range(1, 4):
            heading_style = self.doc.styles[f'Heading {level}']
            heading_style.font.name = 'Calibri'
            heading_style.font.bold = True
            if level == 1:
                heading_style.font.size = Pt(18)
                heading_style.font.color.rgb = RGBColor(0x1B, 0x3A, 0x5C)  # Dark blue
            elif level == 2:
                heading_style.font.size = Pt(14)
                heading_style.font.color.rgb = RGBColor(0x2C, 0x5F, 0x8A)
            elif level == 3:
                heading_style.font.size = Pt(12)
                heading_style.font.color.rgb = RGBColor(0x3D, 0x85, 0xC6)

    def add_cover_page(self, project_name: str, owner: str,
                       company_name: str, date: str,
                       logo_path: Optional[str] = None):
        if logo_path and os.path.exists(logo_path):
            paragraph = self.doc.add_paragraph()
            paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
            run = paragraph.add_run()
            run.add_picture(logo_path, width=Inches(3))
        
        self.doc.add_paragraph()  # Spacer
        self.doc.add_paragraph()  # Spacer
        
        title = self.doc.add_paragraph()
        title.alignment = WD_ALIGN_PARAGRAPH.CENTER
        run = title.add_run('PROPOSAL FOR')
        run.font.size = Pt(14)
        run.font.color.rgb = RGBColor(0x66, 0x66, 0x66)
        
        project = self.doc.add_paragraph()
        project.alignment = WD_ALIGN_PARAGRAPH.CENTER
        run = project.add_run(project_name.upper())
        run.font.size = Pt(24)
        run.font.bold = True
        run.font.color.rgb = RGBColor(0x1B, 0x3A, 0x5C)
        
        self.doc.add_paragraph()  # Spacer
        
        submitted = self.doc.add_paragraph()
        submitted.alignment = WD_ALIGN_PARAGRAPH.CENTER
        run = submitted.add_run(f'Submitted to: {owner}')
        run.font.size = Pt(14)
        
        self.doc.add_paragraph()  # Spacer
        
        by = self.doc.add_paragraph()
        by.alignment = WD_ALIGN_PARAGRAPH.CENTER
        run = by.add_run(f'Prepared by: {company_name}')
        run.font.size = Pt(14)
        
        date_p = self.doc.add_paragraph()
        date_p.alignment = WD_ALIGN_PARAGRAPH.CENTER
        run = date_p.add_run(date)
        run.font.size = Pt(12)
        
        self.doc.add_page_break()

    def add_table_of_contents(self, sections: List[Dict]):
        self.doc.add_heading('Table of Contents', level=1)
        for i, section in enumerate(sections, 1):
            toc_entry = self.doc.add_paragraph()
            toc_entry.style = self.doc.styles['Normal']
            run = toc_entry.add_run(f'{i}. {section.get("original_section_name", section["section_type"])}')
            run.font.size = Pt(12)
        self.doc.add_page_break()

    def add_section(self, title: str, content: str, heading_level: int = 1):
        self.doc.add_heading(title, level=heading_level)
        paragraphs = content.split('\n\n')
        for para_text in paragraphs:
            if para_text.strip():
                if para_text.strip().startswith(('- ', '• ', '* ')):
                    for line in para_text.strip().split('\n'):
                        line = line.lstrip('-•* ').strip()
                        if line:
                            p = self.doc.add_paragraph(line, style='List Bullet')
                elif para_text.strip().startswith(('#')):
                    heading_text = para_text.strip().lstrip('#').strip()
                    self.doc.add_heading(heading_text, level=2)
                else:
                    self.doc.add_paragraph(para_text.strip())

    def format_proposal(self, proposal: Dict, company_info: Dict,
                        logo_path: Optional[str] = None) -> str:
        project_name = proposal.get('project_name', 'Untitled Project')
        owner = proposal.get('requirements', {}).get('owner_entity', '[Owner Name]')
        company_name = company_info.get('company_name', '[Company Name]')
        date = datetime.now().strftime('%B %d, %Y')
        
        self.add_cover_page(project_name, owner, company_name, date, logo_path)
        
        sections = proposal.get('sections', [])
        self.add_table_of_contents(sections)
        
        for section in sections:
            title = section.get('original_section_name', section['section_type'].replace('_', ' ').title())
            self.add_section(title, section['content'])
        
        output_path = f'proposal_{project_name.replace(" ", "_")[:50]}_{datetime.now().strftime("%Y%m%d_%H%M%S")}.docx'
        self.doc.save(output_path)
        return output_path


class PandaDocFormatter:
    def __init__(self):
        self.api_key = os.environ.get('PANDADOC_API_KEY')
        self.base_url = 'https://api.pandadoc.com/public/v1'
        self.headers = {
            'Authorization': f'API-Key {self.api_key}',
            'Content-Type': 'application/json'
        }

    def create_document_from_template(self, template_id: str, proposal: Dict,
                                       company_info: Dict, recipient: Dict) -> Dict:
        sections_content = {}
        for section in proposal.get('sections', []):
            field_name = section['section_type'].lower()
            sections_content[field_name] = section['content']
        
        payload = {
            'name': f"Proposal - {proposal.get('project_name', 'Untitled')}",
            'template_uuid': template_id,
            'recipients': [{
                'email': recipient.get('email'),
                'first_name': recipient.get('first_name'),
                'last_name': recipient.get('last_name'),
                'role': 'Client'
            }],
            'tokens': [
                {'name': 'project_name', 'value': proposal.get('project_name', '')},
                {'name': 'company_name', 'value': company_info.get('company_name', '')},
                {'name': 'date', 'value': datetime.now().strftime('%B %d, %Y')},
                {'name': 'contact_name', 'value': company_info.get('contact_name', '')},
                {'name': 'contact_email', 'value': company_info.get('contact_email', '')},
                {'name': 'contact_phone', 'value': company_info.get('contact_phone', '')}
            ],
            'fields': sections_content,
            'metadata': {
                'generated_by': 'proposal-ai-system',
                'generated_at': datetime.utcnow().isoformat()
            }
        }
        
        response = requests.post(
            f'{self.base_url}/documents',
            headers=self.headers,
            json=payload
        )
        response.raise_for_status()
        return response.json()

    def list_templates(self) -> List[Dict]:
        response = requests.get(
            f'{self.base_url}/templates',
            headers=self.headers
        )
        response.raise_for_status()
        return response.json().get('results', [])

# Procore API sync agent with Azure Functions timer trigger for daily
# incremental ingestion

# procore_sync.py
import os
import json
import requests
import tempfile
from datetime import datetime, timedelta
from typing import Dict, List, Optional
from ingest_proposals import ProposalIngestionPipeline

class ProcoreSync:
    def __init__(self):
        self.client_id = os.environ['PROCORE_CLIENT_ID']
        self.client_secret = os.environ['PROCORE_CLIENT_SECRET']
        self.company_id = os.environ['PROCORE_COMPANY_ID']
        self.base_url = 'https://api.procore.com/rest/v1.0'
        self.token = None
        self.ingestion = ProposalIngestionPipeline()

    def authenticate(self) -> str:
        response = requests.post(
            'https://login.procore.com/oauth/token',
            data={
                'grant_type': 'client_credentials',
                'client_id': self.client_id,
                'client_secret': self.client_secret
            }
        )
        response.raise_for_status()
        self.token = response.json()['access_token']
        return self.token

    def _headers(self) -> Dict:
        if not self.token:
            self.authenticate()
        return {
            'Authorization': f'Bearer {self.token}',
            'Procore-Company-Id': str(self.company_id)
        }

    def get_projects(self, status: str = 'Active', updated_since: Optional[str] = None) -> List[Dict]:
        params = {'company_id': self.company_id, 'filters[status]': status}
        if updated_since:
            params['filters[updated_at]'] = f'>={updated_since}'
        
        response = requests.get(
            f'{self.base_url}/projects',
            headers=self._headers(),
            params=params
        )
        response.raise_for_status()
        return response.json()

    def get_project_details(self, project_id: int) -> Dict:
        response = requests.get(
            f'{self.base_url}/projects/{project_id}',
            headers=self._headers()
        )
        response.raise_for_status()
        return response.json()

    def get_bid_packages(self, project_id: int) -> List[Dict]:
        response = requests.get(
            f'{self.base_url}/projects/{project_id}/bid_packages',
            headers=self._headers()
        )
        response.raise_for_status()
        return response.json()

    def sync_projects_to_knowledge_base(self, full_sync: bool = False):
        if full_sync:
            projects = self.get_projects(status='Active') + self.get_projects(status='Completed')
        else:
            yesterday = (datetime.utcnow() - timedelta(days=1)).strftime('%Y-%m-%dT00:00:00Z')
            projects = self.get_projects(updated_since=yesterday)

        print(f'Syncing {len(projects)} projects from Procore...')
        
        for project in projects:
            project_text = self._format_project_for_embedding(project)
            
            with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as tmp:
                tmp.write(project_text)
                tmp_path = tmp.name
            
            try:
                metadata = {
                    'source': 'procore',
                    'procore_project_id': str(project.get('id')),
                    'project_name': project.get('name', ''),
                    'project_status': project.get('status', ''),
                    'project_type': project.get('project_type', {}).get('name', '') if project.get('project_type') else '',
                }
                self.ingestion.ingest_file(tmp_path, 'procore_projects', metadata)
            finally:
                os.unlink(tmp_path)

    def _format_project_for_embedding(self, project: Dict) -> str:
        lines = [
            f"PROJECT: {project.get('name', 'Unknown')}",
            f"Status: {project.get('status', 'Unknown')}",
            f"Address: {project.get('address', 'N/A')}, {project.get('city', '')}, {project.get('state_code', '')}",
            f"Project Type: {project.get('project_type', {}).get('name', 'N/A') if project.get('project_type') else 'N/A'}",
            f"Start Date: {project.get('start_date', 'N/A')}",
            f"Estimated Completion: {project.get('estimated_completion_date', 'N/A')}",
            f"Total Value: ${project.get('total_value', 'N/A')}",
            f"Square Footage: {project.get('square_feet', 'N/A')}",
            f"Description: {project.get('description', 'No description available.')}",
            f"Owner: {project.get('owner', {}).get('name', 'N/A') if project.get('owner') else 'N/A'}",
        ]
        return '\n'.join(lines)


# Azure Function timer trigger for daily sync
# function.json:
# {
#   "scriptFile": "procore_sync_function.py",
#   "bindings": [
#     {
#       "name": "timer",
#       "type": "timerTrigger",
#       "direction": "in",
#       "schedule": "0 0 2 * * *"
#     }
#   ]
# }
#
# procore_sync_function.py:
# import azure.functions as func
# from procore_sync import ProcoreSync
# 
# def main(timer: func.TimerRequest) -> None:
#     sync = ProcoreSync()
#     sync.sync_projects_to_knowledge_base(full_sync=False)