#!/usr/bin/env python3
"""The same liveness test, run on the fediverse, using the same rules.

Measuring Farcaster and stopping there would have been cheap in a bad way: a
number about someone else's platform with nothing to compare it to. So this
applies the identical metric to Lemmy communities -- share of posts answered by
an account other than the author, counted only over posts old enough to have had
the chance -- and the two tables can be read against each other.

The rules are kept deliberately identical, including the ones that hurt:
  * a reply counts only if its author differs from the post's author
  * only posts at least MATURE hours old are eligible
  * one snapshot, newest-first, no cherry-picking of communities
Lemmy hands out counts.comments for free, but that counts the author's own
replies too, so it is NOT used for the headline number; comments are fetched and
authors compared. That is 50 extra requests per community and costs nothing but
time, and it keeps the comparison honest.
"""
import json, sys, time, urllib.parse, urllib.request
from concurrent.futures import ThreadPoolExecutor

UA = {'user-agent': 'Mozilla/5.0 (liveness-measurement; see 144-31-195-17.sslip.io/fcreal.html)'}
MATURE = 6.0
LIMIT = 50

COMMUNITIES = [
    ('programming.dev', 'programming'),
    ('programming.dev', 'python'),
    ('lemmy.ml',        'asklemmy'),
    ('lemmy.ml',        'fediverse'),
    ('lemmy.ml',        'linux'),
    ('lemmy.ml',        'opensource'),
    ('lemmy.world',     'technology'),
    ('lemmy.world',     'science'),
]


def get(url, timeout=30):
    return json.load(urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=timeout))


def age_h(iso, now):
    iso = iso.replace('Z', '').split('.')[0]
    t = time.mktime(time.strptime(iso, '%Y-%m-%dT%H:%M:%S'))
    return (now - t) / 3600


def foreign_comment(inst, pv):
    """Did anyone other than the post's author comment on it?"""
    pid = pv['post']['id']
    author = pv['creator']['id']
    try:
        d = get(f"https://{inst}/api/v3/comment/list?post_id={pid}&limit=20&sort=Old&max_depth=1")
    except Exception:
        return None
    return any(c['creator']['id'] != author for c in d.get('comments', []))


def measure(inst, name):
    try:
        d = get(f"https://{inst}/api/v3/post/list?community_name={urllib.parse.quote(name)}"
                f"&sort=New&limit={LIMIT}&type_=Local")
    except Exception as e:
        print(f'{name}@{inst}: ERR {e}', file=sys.stderr)
        return None
    posts = d.get('posts', [])
    # Lemmy pins featured posts to the top of a New listing, so the "newest 50"
    # arrived carrying posts up to three years old -- which is what a span of
    # 27,724 hours in the first run was telling me. Pinned posts are mature and
    # heavily commented, so leaving them in would have flattered every number.
    posts = [p for p in posts if not (p['post'].get('featured_community')
                                      or p['post'].get('featured_local'))]
    if not posts:
        return None
    now = time.time()
    ages = [age_h(p['post']['published'], now) for p in posts]
    mature = [p for p, a in zip(posts, ages) if a >= MATURE]
    with ThreadPoolExecutor(max_workers=8) as ex:
        rep = list(ex.map(lambda p: foreign_comment(inst, p), mature))
    ok = [r for r in rep if r is not None]
    authors = {p['creator']['id'] for p in posts}
    ncom = sorted(p['counts']['comments'] for p in posts)
    # post_list's community object carries no counts, so the subscriber figure
    # came back 0 for every row in the first pass. It is the one number the
    # Farcaster comparison actually turns on -- follower count as a failed
    # liveness signal -- so it is worth a second request rather than a blank.
    try:
        cv = get(f"https://{inst}/api/v3/community?name={urllib.parse.quote(name)}")
        subs = cv['community_view']['counts'].get('subscribers')
    except Exception:
        subs = None
    return dict(community=f'{name}@{inst}', posts=len(posts), authors=len(authors),
                subscribers=subs, replied=(100.0 * sum(ok) / len(ok)) if ok else float('nan'),
                n_mature=len(ok), newest_h=min(ages), span_h=max(ages) - min(ages),
                median_comments=ncom[len(ncom) // 2],
                top_share=100.0 * max(sum(1 for p in posts if p['creator']['id'] == a)
                                      for a in authors) / len(posts))


if __name__ == '__main__':
    rows = []
    hdr = (f"{'community':<28}{'subs':>8}{'posts':>6}{'authors':>8}{'replied%':>9}"
           f"{'medcom':>7}{'newest_h':>9}{'span_h':>8}{'top1%':>7}{'n>6h':>6}")
    print(hdr); print('-' * len(hdr))
    for inst, name in COMMUNITIES:
        r = measure(inst, name)
        if not r:
            print(f'{name}@{inst:<20} (no data)'); continue
        rows.append(r)
        print(f"{r['community']:<28}{r['subscribers'] or 0:>8}{r['posts']:>6}{r['authors']:>8}"
              f"{r['replied']:>9.1f}{r['median_comments']:>7}{r['newest_h']:>9.1f}"
              f"{r['span_h']:>8.1f}{r['top_share']:>7.1f}{r['n_mature']:>6}")
    json.dump(rows, open('/home/agent/fcreal/lemmy_result.json', 'w'), indent=1)