import http.server
import ssl
import os
import urllib.request
import urllib.error
import json
import re
import subprocess
import tempfile
import threading

TURSO_URL = "https://second-brain-dumb-bug-baby.aws-us-west-2.turso.io"
TURSO_TOKEN = "eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJhIjoicnciLCJnaWQiOiI3MzlkODUzZS1mYTY1LTRiNTEtYjMzNC0xMmVmNTIzNDE4NGMiLCJpYXQiOjE3NzgyNzk4MDUsInJpZCI6ImRkM2RlMmQ0LTQwNDMtNDgyMy1hNGQzLTAzODJiYWNkMTQ0MiJ9.ivgVUS97cRk2KegeLqo0rJLvsZ3XtkSi8vCad3tr8xCUqsR-p1wruM4sJII3aSXEVj6Q6FNRMwIq9V1M0LO4CA"
RAPIDAPI_KEY = "f5f60ce59amshf959c091a474e92p1063ddjsn7fb5e4ab36cc"
APIPY_TOKEN = "apify_api_3AYc8MshFJclANE1uhywz5ofGcV7Ak0qY9A5"
ANTHROPIC_API_KEY = "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"

def turso_execute(sql, args=None):
    payload = {"requests": [{"type": "execute", "stmt": {"sql": sql, "args": args or []}}, {"type": "close"}]}
    req = urllib.request.Request(
        f"{TURSO_URL}/v2/pipeline",
        data=json.dumps(payload).encode(),
        headers={"Authorization": f"Bearer {TURSO_TOKEN}", "Content-Type": "application/json"},
        method="POST"
    )
    with urllib.request.urlopen(req) as resp:
        return json.loads(resp.read())

def fetch_zillow_full(zpid, url):
    req = urllib.request.Request(
        f"https://real-time-real-estate-data.p.rapidapi.com/property-details?zpid={zpid}",
        headers={
            "x-rapidapi-host": "real-time-real-estate-data.p.rapidapi.com",
            "x-rapidapi-key": RAPIDAPI_KEY
        }
    )
    with urllib.request.urlopen(req, timeout=30) as resp:
        d = json.loads(resp.read())
    data = d.get("data", {})

    # Extract photos - get highest res jpeg
    raw_photos = data.get("photos") or data.get("responsivePhotos") or []
    photos = []
    for p in raw_photos:
        if isinstance(p, dict):
            jpegs = p.get("mixedSources", {}).get("jpeg", [])
            if jpegs:
                best = max(jpegs, key=lambda x: x.get("width", 0))
                photos.append(best["url"])
            elif p.get("url"):
                photos.append(p["url"])
        elif isinstance(p, str):
            photos.append(p)
    photos = [p for p in photos if p and p.startswith("http")][:20]

    # Extract listing data
    addr = data.get("address", {})
    street = addr.get("streetAddress", "")
    city = addr.get("city", "")
    state = addr.get("state", "")
    zipcode = addr.get("zipcode", "")
    address = f"{street}, {city}, {state} {zipcode}".strip(", ")

    price_raw = data.get("price", "")
    price = f"${price_raw:,}/mo" if isinstance(price_raw, (int, float)) else str(price_raw)

    beds = data.get("bedrooms", "")
    baths = data.get("bathrooms", "")
    beds_str = f"{beds}bd / {baths}ba" if beds and baths else ""

    resoFacts = data.get("resoFacts", {}) or {}
    description = data.get("description") or resoFacts.get("description") or ""
    sqft = data.get("livingArea") or data.get("livingAreaValue") or ""
    notes = f"{sqft} sqft. " if sqft else ""
    pet_policy = resoFacts.get("petsAllowed")
    if pet_policy:
        notes += "Pets allowed. "
    parking = resoFacts.get("parkingFeatures")
    if parking:
        notes += f"Parking: {', '.join(parking) if isinstance(parking, list) else parking}. "

    # Feature detection
    desc_lower = (description or "").lower()
    reso_lower = str(resoFacts).lower()
    combined = desc_lower + " " + reso_lower

    sf_zip_to_hood = {
        "94102": "Civic Center", "94103": "SoMa", "94104": "Financial District",
        "94105": "Embarcadero", "94107": "Potrero Hill", "94108": "Chinatown",
        "94109": "Nob Hill", "94110": "Mission", "94111": "Embarcadero",
        "94112": "Excelsior", "94114": "Castro", "94115": "Pacific Heights",
        "94116": "West Portal", "94117": "Haight-Ashbury", "94118": "Richmond",
        "94121": "Outer Richmond", "94122": "Inner Sunset", "94123": "Marina",
        "94124": "Bayview", "94127": "West Portal", "94129": "Presidio",
        "94130": "Treasure Island", "94131": "Noe Valley", "94132": "Lake Merced",
        "94133": "North Beach", "94134": "Visitacion Valley"
    }
    hood = addr.get("neighborhood") or sf_zip_to_hood.get(zipcode, city or "")
    listing = {
        "address": address,
        "hood": hood,
        "price": price,
        "beds": beds_str,
        "url": url,
        "description": description,
        "notes": notes.strip(),
        "feat_light": 1 if any(w in combined for w in ["bay window","natural light","bright","sunny","windows"]) else 0,
        "feat_hw": 1 if any(w in combined for w in ["hardwood","wood floor","oak floor","parquet"]) else 0,
        "feat_dw": 1 if "dishwasher" in combined else 0,
        "feat_ferry": 0,
        "feat_access": 1 if any(w in combined for w in ["elevator","ground floor","first floor","no stairs"]) else 0,
        "feat_outdoor": 1 if any(w in combined for w in ["balcony","patio","deck","garden","yard","terrace","roof"]) else 0,
        "feat_tub": 1 if any(w in combined for w in ["bathtub","soaking tub","bath tub","shower/tub","tub/shower"]) else 0,
        "feat_gas": 1 if any(w in combined for w in ["gas stove","gas range","gas oven","gas cooking"]) else 0,
    }
    return listing, photos

def fetch_zillow_photos(zpid):
    import time
    start_req = urllib.request.Request(
        f"https://api.apify.com/v2/acts/zillowscraper~zillow-property-images-fetcher/runs?token={APIFY_TOKEN}",
        data=json.dumps({"zpids": [str(zpid)]}).encode(),
        headers={"Content-Type": "application/json"},
        method="POST"
    )
    with urllib.request.urlopen(start_req, timeout=30) as resp:
        run_data = json.loads(resp.read())
    run_id = run_data["data"]["id"]
    for _ in range(12):
        time.sleep(5)
        status_req = urllib.request.Request(
            f"https://api.apify.com/v2/actor-runs/{run_id}?token={APIFY_TOKEN}"
        )
        with urllib.request.urlopen(status_req, timeout=15) as resp:
            status = json.loads(resp.read())
        if status["data"]["status"] == "SUCCEEDED":
            break
    dataset_id = status["data"]["defaultDatasetId"]
    items_req = urllib.request.Request(
        f"https://api.apify.com/v2/datasets/{dataset_id}/items?token={APIFY_TOKEN}"
    )
    with urllib.request.urlopen(items_req, timeout=15) as resp:
        items = json.loads(resp.read())
    photos = []
    for item in items:
        if "images" in item:
            photos += [img.get("url") or img.get("src") or img for img in item["images"] if isinstance(img, (dict, str))]
        elif "photos" in item:
            photos += item["photos"]
    return [p for p in photos if isinstance(p, str) and p.startswith("http")][:20]

def fetch_photos_background(lid, url):
    try:
        photos = []
        if "zillow.com" in url:
            import re
            m = re.search(r'/(\d+)_zpid', url)
            if m:
                photos = fetch_zillow_photos(m.group(1))
        if not photos:
            _, photos = fetch_url(url)
        if photos:
            turso_execute(
                'UPDATE apartment_listings SET photos=?, updated_at=CURRENT_TIMESTAMP WHERE id=?',
                [arg(json.dumps(photos)), arg(lid)]
            )
    except Exception as e:
        pass

def fetch_url(url):
    import sys, json as _json
    sys.path.insert(0, '/home/ubuntu/.local/lib/python3.13/site-packages')
    from playwright.sync_api import sync_playwright
    with sync_playwright() as p:
        browser = p.chromium.launch()
        context_opts = {}
        if 'zillow.com' in url:
            try:
                with open('/home/ubuntu/second-brain/zillow_cookies.json') as f:
                    raw_cookies = _json.load(f)
                context = browser.new_context(
                    user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
                )
                cookies = []
                for c in raw_cookies:
                    cookie = {
                        'name': c['name'],
                        'value': c['value'],
                        'domain': c['domain'],
                        'path': c.get('path', '/'),
                        'secure': c.get('secure', False),
                        'httpOnly': c.get('httpOnly', False),
                    }
                    if c.get('expirationDate'):
                        cookie['expires'] = int(c['expirationDate'])
                    cookies.append(cookie)
                context.add_cookies(cookies)
            except Exception as e:
                context = browser.new_context()
        else:
            context = browser.new_context()
        page = context.new_page()
        page.goto(url, wait_until='domcontentloaded', timeout=60000)
        page.wait_for_timeout(5000)
        content = page.content()
        photos = page.evaluate("""() => {
            const imgs = Array.from(document.querySelectorAll('img'));
            return imgs
                .map(img => img.src)
                .filter(src => src && src.startsWith('http') && !src.includes('logo') && !src.includes('icon') && !src.includes('avatar') && !src.includes('sprite') && (src.includes('.jpg') || src.includes('.jpeg') || src.includes('.png') || src.includes('.webp')))
                .slice(0, 20);
        }""")
        browser.close()
    return content, photos


def claude_extract(html, url):
    truncated = html[:40000]
    with open('/home/ubuntu/second-brain/listing_prompt.txt', 'r') as f:
        prompt = f.read().replace('{url}', url).replace('{html}', truncated)

    payload = {
        "model": "claude-sonnet-4-5-20250929",
        "max_tokens": 1000,
        "messages": [{"role": "user", "content": prompt}]
    }
    req = urllib.request.Request(
        "https://api.anthropic.com/v1/messages",
        data=json.dumps(payload).encode(),
        headers={
            "x-api-key": ANTHROPIC_API_KEY,
            "anthropic-version": "2023-06-01",
            "Content-Type": "application/json"
        },
        method="POST"
    )
    with urllib.request.urlopen(req, timeout=30) as resp:
        data = json.loads(resp.read())
    text = data["content"][0]["text"].strip()
    text = re.sub(r"^```json\s*", "", text)
    text = re.sub(r"\s*```$", "", text)
    return json.loads(text)

def new_id():
    import binascii
    return binascii.hexlify(os.urandom(8)).decode()

def arg(v):
    if v is None or v == "":
        return {"type": "null"}
    return {"type": "text", "value": str(v)}

class Handler(http.server.SimpleHTTPRequestHandler):

    def do_GET(self):
        from urllib.parse import urlparse, parse_qs
        import urllib.request
        parsed = urlparse(self.path)
        if parsed.path == "/deploy":
            params = parse_qs(parsed.query)
            url = params.get("url", [""])[0]
            filename = params.get("file", [""])[0]
            if not url or not filename or ".." in filename:
                self.send_response(400)
                self.end_headers()
                return
            with urllib.request.urlopen(url) as r:
                data = r.read()
            dest = "/home/ubuntu/second-brain/" + filename
            open(dest, "wb").write(data)
            self.send_response(200)
            self.send_header("Access-Control-Allow-Origin","*")
            self.end_headers()
            self.wfile.write(b"Deployed: " + filename.encode())
            return
        return super().do_GET()
    def do_POST(self):
        length = int(self.headers.get("Content-Length", 0))
        body = self.rfile.read(length)

        print(f"DEBUG POST: {self.path}", flush=True)
        if self.path == "/turso":
            req = urllib.request.Request(
                f"{TURSO_URL}/v2/pipeline",
                data=body,
                headers={"Authorization": f"Bearer {TURSO_TOKEN}", "Content-Type": "application/json"},
                method="POST"
            )
            try:
                with urllib.request.urlopen(req) as resp:
                    result = resp.read()
                status = 200
            except urllib.error.HTTPError as e:
                result = e.read()
                status = e.code
            self._respond(status, result)

        elif self.path == "/fetch":
            try:
                data = json.loads(body)
                html = fetch_url(data["url"])
                self._respond(200, json.dumps({"html": html[:20000]}).encode())
            except Exception as e:
                self._respond(500, json.dumps({"error": str(e)}).encode())

        elif self.path == "/import":
            try:
                data = json.loads(body)
                url = data["url"]
                import re as _re
                if "zillow.com" in url:
                    m = _re.search(r'/(\d+)_zpid', url)
                    if m:
                        listing, photos = fetch_zillow_full(m.group(1), url)
                    else:
                        self._respond(422, json.dumps({"ok": False, "error": "Could not find ZPID in Zillow URL."}).encode())
                        return
                else:
                    html, photos = fetch_url(url)
                    listing = claude_extract(html[:40000], url)
                lid = new_id()
                sql = """INSERT INTO apartment_listings
                    (id,address,hood,price,beds,status,url,notes,description,photos,
                     feat_light,feat_hw,feat_dw,feat_ferry,feat_access,feat_outdoor,feat_tub,feat_gas)
                    VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"""
                args = [
                    arg(lid), arg(listing.get("address")), arg(listing.get("hood")),
                    arg(listing.get("price")), arg(listing.get("beds")), arg("new"),
                    arg(listing.get("url")), arg(listing.get("notes")),
                    arg(listing.get("description")), arg(json.dumps(photos)),
                    arg(listing.get("feat_light",0)), arg(listing.get("feat_hw",0)),
                    arg(listing.get("feat_dw",0)), arg(listing.get("feat_ferry",0)),
                    arg(listing.get("feat_access",0)), arg(listing.get("feat_outdoor",0)),
                    arg(listing.get("feat_tub",0)), arg(listing.get("feat_gas",0))
                ]
                if not listing.get("address"):
                    self._respond(422, json.dumps({"ok": False, "error": "Could not extract listing data — site may be blocking the request. Try the manual Add Listing form instead."}).encode())
                    return
                turso_execute(sql, args)
                self._respond(200, json.dumps({
                    "ok": True,
                    "address": listing.get("address", "Unknown"),
                    "price": listing.get("price", ""),
                    "id": lid
                }).encode())
            except Exception as e:
                self._respond(500, json.dumps({"ok": False, "error": str(e)}).encode())

        elif self.path == "/import-html":
            try:
                data = json.loads(body)
                html = data["html"]
                url = data.get("url", "")
                print(f"DEBUG import-html: url={url[:80]} html_len={len(html)} has_next_data={'__NEXT_DATA__' in html} has_zillow_photo={'zillowstatic' in html}", flush=True)
                # For Zillow, use Apify directly instead of HTML extraction
                if "zillow.com" in url:
                    import re
                    m = re.search(r'/(\d+)_zpid', url)
                    if m:
                        zpid = m.group(1)
                        listing, photos = fetch_zillow_full(zpid, url)
                    else:
                        listing, photos = {}, []
                else:
                    listing = claude_extract(html[:40000], url)
                    photos = listing.get("photos", [])
                    if not isinstance(photos, list): photos = []
                lid = new_id()
                sql = """INSERT INTO apartment_listings
                    (id,address,hood,price,beds,status,url,notes,description,photos,
                     feat_light,feat_hw,feat_dw,feat_ferry,feat_access,feat_outdoor,feat_tub,feat_gas)
                    VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"""
                args = [
                    arg(lid), arg(listing.get("address")), arg(listing.get("hood")),
                    arg(listing.get("price")), arg(listing.get("beds")), arg("new"),
                    arg(listing.get("url")), arg(listing.get("notes")),
                    arg(listing.get("description")), arg(json.dumps(photos)),
                    arg(listing.get("feat_light",0)), arg(listing.get("feat_hw",0)),
                    arg(listing.get("feat_dw",0)), arg(listing.get("feat_ferry",0)),
                    arg(listing.get("feat_access",0)), arg(listing.get("feat_outdoor",0)),
                    arg(listing.get("feat_tub",0)), arg(listing.get("feat_gas",0))
                ]
                turso_execute(sql, args)
                self._respond(200, json.dumps({
                    "ok": True,
                    "address": listing.get("address", "Unknown"),
                    "price": listing.get("price", ""),
                    "id": lid
                }).encode())
                # Background photo fetch
                if url:
                    threading.Thread(target=fetch_photos_background, args=(lid, url), daemon=True).start()
            except Exception as e:
                self._respond(500, json.dumps({"ok": False, "error": str(e)}).encode())

        else:
            self._respond(404, b"Not found")

    def _respond(self, status, body):
        self.send_response(status)
        self.send_header("Content-Type", "application/json")
        self.send_header("Access-Control-Allow-Origin", "*")
        self.end_headers()
        self.wfile.write(body)

    def do_OPTIONS(self):
        self.send_response(200)
        self.send_header("Access-Control-Allow-Origin", "*")
        self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
        self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization")
        self.end_headers()

    def log_message(self, format, *args):
        pass


os.chdir('/home/ubuntu/second-brain')

httpd2 = http.server.HTTPServer(('0.0.0.0', 7002), Handler)
threading.Thread(target=httpd2.serve_forever, daemon=True).start()

httpd = http.server.HTTPServer(('0.0.0.0', 7001), Handler)
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ctx.load_cert_chain('cert.pem', 'key.pem')
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
print("Serving HTTPS on 7001, HTTP on 7002")
httpd.serve_forever()