"""Append-only 8 Oct research editions; reads preserved inputs, never calls a market API."""
import json,gzip,hashlib,statistics,io,csv
from pathlib import Path
from urllib.parse import urlparse,parse_qs
VERSION='2026-10-08.1'
CAPTURE=Path('docs/tools-research-project-2026-10-08/batch-04/evidence/compilation-inputs.json')
def read(p):return json.loads(Path(p).read_text())
def encoded(x):return (json.dumps(x,ensure_ascii=False,sort_keys=True,separators=(',',':'))+'\n').encode()
def sha(b):return hashlib.sha256(b).hexdigest()
def put(p,b):
 p=Path(p);p.parent.mkdir(parents=True,exist_ok=True)
 if p.exists():
  if p.read_bytes()!=b:raise ValueError(f'Immutable edition differs: {p}. Bump version.')
 else:p.write_bytes(b)
def med(v):return statistics.median(v) if v else None
def save(slug,payload,raw,csvrows):
 base=f'research/snapshots/{slug}/{VERSION}';priv=f'data/{base}';pub=f'public/{base}'
 payload['metadata']['immutableDownloads']={k:f'/{base}/{name}' for k,name in [('manifest','manifest.json'),('dataset','dataset.json'),('csv','data.csv'),('rawInputs','raw-inputs.json.gz')]}
 s=io.StringIO();w=csv.DictWriter(s,fieldnames=list(csvrows[0]));w.writeheader();w.writerows(csvrows)
 files={'dataset.json':encoded(payload),'raw-inputs.json.gz':gzip.compress(encoded(raw),mtime=0),'data.csv':s.getvalue().encode(),'reproduce.py':Path(__file__).read_bytes()}
 artifacts=[]
 for name,body in files.items():
  put(f'{pub}/{name}',body);artifacts.append({'path':f'/{base}/{name}','bytes':len(body),'sha256':sha(body)})
 put(f'{priv}/dataset.json',files['dataset.json'])
 manifest={'version':VERSION,'publishedAt':'2026-10-08','immutable':True,'overwritePolicy':'append-only; new corrections require a new version','artifacts':artifacts,'preservedRawInputs':[{'path':f'{priv}/dataset.json','bytes':len(files['dataset.json']),'sha256':sha(files['dataset.json'])}],'note':'reproduce.py records the compilation algorithm. raw-inputs preserves the source package; dated observations were not refreshed.'}
 put(f'{pub}/manifest.json',encoded(manifest));put(f'{priv}/manifest.json',encoded(manifest))
 return payload
p=read(CAPTURE)
# BTC: preserve inputs from the September study, revise documented boundary handling.
btc=read('data/research/snapshots/real-leverage-at-size/2026-09-18.1/raw/priority-contracts-btc.json')
manual=read('data/research/snapshots/real-leverage-at-size/2026-09-18.1/manual-inputs.json')
size=p['size'];rows=size['rows'];notionals=size['notionals']
for r in rows:
 if r['provider'] not in ['bybit','hyperliquid']:continue
 r['boundaryPolicy']='inclusive-upper-bound'
 c=next(c for c in btc['contracts'] if c['provider']==r['provider']);tiers=c['fields']['marginTiers']['value']
 for s in r['snapshots']:
  n=s['notionalUsd'];t=next((t for t in tiers if n>=t['notionalFloorUsd'] and (t.get('notionalCeilingUsd') is None or n<=t['notionalCeilingUsd'])),None)
  if not t:continue
  im=t['initialMarginPct'] or 100/t['maxLeverage'];eff=min(r['headlineLeverage'],t['maxLeverage'],100/im)
  s.update(tierLeverage=t['maxLeverage'],leverageImpliedByInitialMargin=100/im,effectiveLeverage=eff,initialMarginUsd=n/eff,retainedHeadlinePct=eff/r['headlineLeverage']*100,tierLabel=t['label'],tierFloorUsd=t['notionalFloorUsd'],tierCeilingUsd=t.get('notionalCeilingUsd'),initialMarginPct=t['initialMarginPct'],maintenanceMarginPct=t['maintenanceMarginPct'])
sm=[]
for n in notionals:
 ss=[(r,next(s for s in r['snapshots'] if s['notionalUsd']==n)) for r in rows];valid=[(r,s)for r,s in ss if s['effectiveLeverage'] is not None];v=[s['effectiveLeverage']for r,s in valid];reduced=sum(s['effectiveLeverage']<r['headlineLeverage'] for r,s in valid)
 sm.append(dict(notionalUsd=n,comparablePlatforms=len(v),reducedPlatforms=reduced,retainedHeadlinePlatforms=len(v)-reduced,medianLeverage=med(v),highestLeverage=max(v),lowestLeverage=min(v),orderLimitedPlatforms=sum(s['capacityIssue']=='order-limit' for r,s in ss),positionLimitedPlatforms=sum(s['capacityIssue']=='position-limit'for r,s in ss)))
meta={**size['metadata'],'datasetVersion':VERSION,'snapshotId':'btc-position-study-2026-10-08-v1','generatedAt':'2026-10-08T18:22:54.287Z','sourceDatasetVersion':'2026-09-18.1','revisionNote':'Analysis revision using preserved September inputs. Bybit and Hyperliquid documented caps are inclusive; other shared API boundaries retain the disclosed conservative convention. No new historical observations created.','methodology':'Total-position margin ceiling is the minimum of contract maximum, selected tier maximum and the inverse initial-margin rate. Position limits invalidate a value; order limits are separate flags. Bybit, Hyperliquid and WEEX use documented inclusive caps; other shared boundaries use the conservative next tier.'}
sizeout=save('real-leverage-at-size',dict(metadata=meta,notionals=notionals,rows=rows,exclusions=size['exclusions'],summaries=sm),dict(originalContracts=btc,manualInputs=manual,originalComputedRows=p['size'],methodologyVersion='btc-position-v2'),[dict(version=VERSION,provider=r['provider'],contract=r['venueSymbol'],marginMode=r['marginMode'],boundaryPolicy=r['boundaryPolicy'],**s) for r in rows for s in r['snapshots']])
# Statistics are a new compilation, not a replacement of the old October edition.
st=p['statistics'];st['cryptoLeverageStatisticsMetadata'].update(version=VERSION,snapshotDate='2026-10-08',realLeverageDatasetVersion=VERSION,methodology='Unweighted registry compilation frozen on 8 October. Underlying observations keep their own dates. 377 confirmed contracts from ten providers and forty requested assets; unequal provider coverage. Fee channels are separated, and the BTC cohort retains September observations.')
st['effectiveLeverageBySize']=sm
for metric in st['cryptoLeverageStatisticRows']:
 if metric['section']=='effective-leverage-by-size':metric['value']=next(s['medianLeverage']for s in sm if str(s['notionalUsd']) in metric['metric'].split('_'))
# Meaningful channel labels, retaining every recorded public rate but never merging them into retail promises.
channel={'mexc':'API','coinbase':'Institutional','orderly':'Builder base cost','lighter':'Order-class dependent','paradex':'Order-class dependent'}
facets=[]
for r in st['documentedBaseFeeRows']:
 r['channel']=channel.get(r['provider'],'Public CEX schedule' if r['type']=='cex' else 'Public DEX schedule')
for name in dict.fromkeys(r['channel']for r in st['documentedBaseFeeRows']):
 rr=[r for r in st['documentedBaseFeeRows']if r['channel']==name];facets.append(dict(channel=name,n=len(rr),makerMedianPct=med([r['makerPct']for r in rr]),takerMedianPct=med([r['takerPct']for r in rr])))
contracts=p['priorityContracts'];distribution=[]
for label,lo,hi in [('Below 25×',0,25),('25× to below 50×',25,50),('50× to below 100×',50,100),('100× to below 200×',100,200),('200× or more',200,float('inf'))]:
 values=[c['fields']['leverage']['value']for c in contracts if c['fields']['leverage']['state']=='retrieved' and c['fields']['leverage']['value'] is not None];distribution.append(dict(label=label,count=sum(lo<=v<hi for v in values),denominator=len(values)))
statpayload={'metadata':st['cryptoLeverageStatisticsMetadata'],'exports':st,'feeFacets':facets,'leverageDistribution':distribution}
statout=save('crypto-leverage-statistics',statpayload,dict(compilation=p,btcEdition=sizeout,feeChannelRules=channel),[dict(snapshot_version=VERSION,**r) for r in st['cryptoLeverageStatisticRows']])
# Execution: exact collector outputs are preserved, upstream raw order books are not claimed.
m=read('data/monthly-execution/2026-10.json');idx=read('data/research/raw/collector-responses/2026-10-04T02-41-44.621Z.json');captureRows=[];scenarios=[]
for o in m['observations']:
 c=next(x for x in idx['captures'] if parse_qs(urlparse(x['requestUrl']).query).get('provider')==[o['provider']] and parse_qs(urlparse(x['requestUrl']).query).get('asset')==[o['asset']]);body=Path(c['path']).read_bytes();assert sha(body)==c['sha256'];captureRows.append(dict(capture=c,response=json.loads(body)))
 for n,s in o['scenarios'].items():
  scenarios.append(dict(version=VERSION,provider=o['provider'],exchange=o['exchange'],asset=o['asset'],venueSymbol=o['venueSymbol'],observedAt=o['observedAt'],notionalUsd=int(n),**(s or dict.fromkeys(['longEntrySlippageBps','longEntryFillPct','longExitSlippageBps','longExitFillPct','entryVwap','exitVwap'])),sourceUrl=o['sourceUrl']))
heat=[]
for a in m['metadata']['assets']:
 for n in m['metadata']['notionalsUsd']:
  ss=[s for s in scenarios if s['asset']==a and s['notionalUsd']==n];full=[s for s in ss if s['longEntryFillPct']==100 and s['longExitFillPct']==100]
  heat.append(dict(asset=a,notionalUsd=n,requested=len(ss),pairedFull=len(full),partialOrMissing=len(ss)-len(full),buyMedianBps=med([s['longEntrySlippageBps']for s in full]),sellMedianBps=med([s['longExitSlippageBps']for s in full]),pairedMedianBps=med([s['longEntrySlippageBps']+s['longExitSlippageBps']for s in full])))
met={**m['metadata'],'title':'Crypto futures execution benchmark — 4 October observation window','version':VERSION,'analysisDate':'2026-10-08','upstreamOrderBooksArchived':False,'note':'One capture per pair between 02:41 and 02:50 UTC on 4 October. Historical collector responses retained; complete upstream books were not archived. The two sides use independent equal-USD orders, not a time-separated round trip.'}
exeout=save('monthly-execution',dict(metadata=met,observations=m['observations'],scenarios=scenarios,heatmap=heat,summary=dict(pairs=60,scenarios=len(scenarios),directionalResults=len(scenarios)*2,fullPaired=sum(s['longEntryFillPct']==100 and s['longExitFillPct']==100 for s in scenarios),partialPaired=sum(s['longEntryFillPct']!=100 or s['longExitFillPct']!=100 for s in scenarios))),dict(baseline=m,collectorResponses=captureRows,rawScope=idx['note']),scenarios)
print(json.dumps({'fees':facets,'distribution':distribution,'size':sm,'execution':exeout['summary'],'heatmap':heat},indent=2))