#!/usr/bin/env python3
"""How much of a Farcaster channel is a person?

I went looking for someone with a concrete request I could do work for, and found
that the channels with the followers are full of machine-written filler while the
channels where a human would ask a question have been dead for months. That is a
checkable claim, so this checks it, and reports the numbers that would refute it
as loudly as the ones that support it.

Metrics per channel, all computed from the public hub, no API key:
  casts        recent cast_adds fetched (newest first)
  authors      distinct FIDs among them
  newacct%     share of casts by FIDs above 700000 (registered recently; the
               farms mint accounts in blocks, so this is a weak signal alone)
  replied%     share of casts that got at least one reply from a DIFFERENT FID.
               This is the load-bearing metric: filler is talked at, not with.
  hook%        share opening with a stock engagement-bait construction
  dup%         share whose 12-most-common-word signature repeats another cast's
               by a different author (homogeneity of the writing)
  newest       hours since the most recent cast (a dead room is not a fake room)
A high newacct% with a high replied% would mean a healthy influx of newcomers.
It is the combination -- new accounts, no replies, identical shapes -- that means
nobody is home.
"""
import json, re, sys, time, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor

H = 'https://snap.farcaster.xyz:3381'
FC_EPOCH = 1609459200
UA = {'user-agent': 'Mozilla/5.0'}
HOOK = re.compile(r"^\s*(ever wonder|ever wondered|did you know|beware|what's your take|whats your take|"
                  r"here's why|heres why|let's talk|lets talk|the .{3,25} race is heating up|"
                  r"why .{3,30}\?|.{0,40} vs\. .{0,40}:|thoughts\?)", re.I)
STOP = set('the a an and or of to in is are was were be been for on with as at by that this it its from '
           'you your they their we our but not can could would should may might will just how what why '
           'who when which more most than then so if about into over under between'.split())


def get(url):
    return json.load(urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=30))


def fetch_channel(url, want=150):
    out, token = [], None
    while len(out) < want:
        q = f"{H}/v1/castsByParent?url={urllib.parse.quote(url, safe='')}&pageSize=100&reverse=true"
        if token:
            q += '&pageToken=' + token
        try:
            d = get(q)
        except Exception as e:
            print(f'    fetch error: {e}', file=sys.stderr)
            break
        ms = d.get('messages', [])
        out += ms
        token = d.get('nextPageToken')
        if not ms or not token:
            break
    return out[:want]


def has_foreign_reply(m):
    """True if some other FID replied to this cast."""
    fid, h = m['data']['fid'], m['hash']
    try:
        d = get(f"{H}/v1/castsByParent?fid={fid}&hash={h}&pageSize=10")
    except Exception:
        return None
    return any(r['data']['fid'] != fid for r in d.get('messages', []))


def sig(text):
    ws = [w for w in re.findall(r"[a-z']+", text.lower()) if w not in STOP and len(w) > 2]
    return tuple(sorted(set(ws))[:12])


def measure(name, url, want=150):
    ms = fetch_channel(url, want)
    if not ms:
        return None
    now = time.time()
    texts, fids, ages = [], [], []
    for m in ms:
        b = m['data'].get('castAddBody') or {}
        texts.append(b.get('text') or '')
        fids.append(m['data']['fid'])
        ages.append((now - (m['data']['timestamp'] + FC_EPOCH)) / 3600)
    # A cast half an hour old has had no time to be answered, while these dead
    # channels' casts have had months. Comparing raw reply rates across them
    # would measure age, not company. So the reply rate is computed only over
    # casts that have had at least MATURE hours to attract one, in both groups.
    MATURE = 6.0
    mature = [m for m, a in zip(ms, ages) if a >= MATURE]
    with ThreadPoolExecutor(max_workers=16) as ex:
        replied = list(ex.map(has_foreign_reply, mature))
    ok = [r for r in replied if r is not None]
    seen, dups = {}, 0
    for t, f in zip(texts, fids):
        s = sig(t)
        if len(s) < 6:
            continue
        if s in seen and seen[s] != f:
            dups += 1
        seen.setdefault(s, f)
    n = len(ms)
    return dict(channel=name, casts=n, authors=len(set(fids)),
                newacct=100.0 * sum(f > 700000 for f in fids) / n,
                replied=(100.0 * sum(ok) / len(ok)) if ok else float('nan'),
                hook=100.0 * sum(bool(HOOK.match(t)) for t in texts) / n,
                dup=100.0 * dups / n, newest=min(ages), median_fid=sorted(fids)[n // 2],
                n_mature=len(ok), span_h=max(ages) - min(ages),
                top_share=100.0 * max(fids.count(f) for f in set(fids)) / n)


CHANNELS = [
    ('dev',        'chain://eip155:1/erc721:0x7dd4e31f1530ac682c8ea4d8016e95773e08d8b0'),
    ('ai',         'chain://eip155:7777777/erc721:0x5747eef366fd36684e8893bf4fe628efc2ac2d10'),
    ('base',       'https://onchainsummer.xyz'),
    ('data',       'https://farcaster.group/data'),
    ('founders',   'https://farcaster.group/founders'),
    ('science',    'chain://eip155:8453/erc721:0xd953664a9b9e30fa7b3ccd00a2f9c21c7b75c5f0'),
    ('python',     'https://warpcast.com/~/channel/python'),
    ('programming','https://warpcast.com/~/channel/programming'),
    ('help',       'https://warpcast.com/~/channel/help'),
]

if __name__ == '__main__':
    rows = []
    hdr = f"{'channel':<12}{'casts':>6}{'authors':>8}{'newacct%':>9}{'replied%':>9}{'hook%':>7}{'dup%':>6}{'newest_h':>9}{'span_h':>8}{'top1%':>7}{'n>6h':>6}"
    print(hdr); print('-' * len(hdr))
    for name, url in CHANNELS:
        r = measure(name, url)
        if not r:
            print(f'{name:<12}  (no data)'); continue
        rows.append(r)
        print(f"{r['channel']:<12}{r['casts']:>6}{r['authors']:>8}{r['newacct']:>9.1f}"
              f"{r['replied']:>9.1f}{r['hook']:>7.1f}{r['dup']:>6.1f}{r['newest']:>9.1f}"
              f"{r['span_h']:>8.1f}{r['top_share']:>7.1f}{r['n_mature']:>6}")
    json.dump(rows, open('/home/agent/fcreal/result.json', 'w'), indent=1)