B
    d                 @   s  d Z dZdgZdZdZdZdddddd	Zd
ddgZdddgZddl	m
Z
 ddlmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ ed ddlmZmZ ddlmZmZmZmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* ddl+m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZF dd lGmHZH dd!lImJZJ e)d"d# ZKeKd$k rdd%lLmMZM ndd&lNmOZO d'ZPd(ZQe-d)d* d+ ZRePeQ ZSeSeR ZTeSeTgZUed,krd-nd.ZVd/d0d1d2eVd2d3d4d5d2d6d2d7d8d9d8d:d;d<d=d>eUiZWeXeW eYeZZ[d?d@ Z\dAdB Z]G dCdD dDZ^G dEdF dFZ_dGdH Z`dIdJ ZadKdL ZbdMdN ZcdcdQdRZddOdSgdfdTdUZedddVdWZfdXdY ZgdZd[ Zhd\d] Zid^d_ Zjd`da Zkeldbkre'e\  d"S )ez'
Postprocessing for the tiger pipeline
z
Tjonnie Liztjonnie.li@ligo.orgZ
Productionz titan2.atlas.aei.uni-hannover.dezpcdev2.phys.uwm.eduzldas-pcdev2.ligo.caltech.eduzldas-pcdev2.ligo-la.caltech.eduzldas-pcdev1.ligo-wa.caltech.edu)ZatlasZnemoZcitZlloZlhoblueredgreenz..z//z||    )ConfigParser)dumpload)datetime)combinations)cycle)usercParams__version__ZAgg)clffigure)accesspathR_OKmakedirs)ks_2samp)
mquantiles)PopenPIPE)exitstdoutversion_info
float_info)time)sqrtarrayemptyhstackminmaxreshapeshapeloadtxtvstackappendarangerandomcolumn_stackconcatenatesavetxtlogexpsizezerosargmaxargsortsortsumsubtractarray_split)findall)rangeN   )r9      )OptionParser)ArgumentParserg     @gRhV?   g      ?g       @z1.5.0z	font.sizeztext.fontsizebackendZPDFzaxes.labelsize   zlegend.fontsize   zxtick.labelsizezytick.labelsizez	axes.gridTztext.usetexzsavefig.dpi   zlines.markersize   zfigure.figsizec              C   sF  t dk r^t } | jddddtdd | jdd	dd
tdd | jddddtdd |  \}}nTtdd} | jddddtdd | jdd	dd
tdd | jddddtdd |  }|jdkr|jdkr|j	dkrt
d nh|jdkrtd|j  t|j n@|jdkrt|j n(|j	dkr2t|j	 nt
d t
d dS )zS
	-------------------------------------------------------------------------------
	)r9   r:   z-cz--configFILE
configfilezconfiguration file)metavardesttypehelpz-gz
--generate
examplecfgz#generate example configuration filez-pz--preprocesspreprocessfilez@preprocess data before sending to main script (NB: NOT WORKING!)zPost-processing for TIGER)descriptionNz<Specify either -c/--config, -g/--generate or -p/--preprocesszh****************************************
TIGER post-process
%s
****************************************
zUnknown options - check inputr   )
py_versionr;   
add_optionstr
parse_argsr<   add_argumentrD   rI   rJ   r   r   writeTigerPostProcessTigerCreateExampleConfigFileTigerPreProcess)parserargsoptions rX   h/work/yifan.wang/ringdown/master-ringdown-env/lib/python3.7/site-packages/lalinference/tiger/postproc.pymain[   s8    







rZ   c       7         s  t d t| tr&t }||  ntd t d |ddd}|dd}dd	 |d
ddD  |ddd}|ddd}|dd}|ddd}|dd}t	d|}	dd	 |	D }
t
| |ddd}|ddd}|dd}|dd}|dd}t|t|krBtd g xtt|D ]}|| dkrt|| d}dd	 | D }t d||   t||| |
| |||  d    d    d    n8|| d!krt||  |
| d  _ntd" qTW g }t d# xttD ]}|t|| jd$  | |d   t d%|d    |t|| jd&  | |d   t d%|d    q:W t d' xD ]}|j||d( qW t d) |dd
}|dd*}|dd+}|dd,}|d
d-}g }d.d/g}td0krx& D ]}t||d1}|| qRW t |} fd2d	d0d3 D }g }|d4krt d5 d6d	 |d
d7dD } | d8d8gkrht!d9}!x0D ](}x  D ]}t"|!|#|f}!qW qW t t$|!t%|!g}"t |"d9 d:|"d0 |"d9    |"d0 d:|"d0 |"d9    g}"~!n| }"x D ]|}t&  t'|}#|#(d;}$t)|$||"|d< xJ|D ]B}%|t|d=| d> t*| d? |%  |#j+|d  d@dA qW qrW g }&|d4krjt dB t&  t' }#|#(d;}$t,|$ x>|D ]6}'|&t|dC| dD |'  |#j+|&d  d@dA q0W g gt }(|d4krt dE xttD ]}g |(|< t&  t' }#|#(d;}$t-| |$ xL|D ]D}'|(| t|dF| j dD |'  |#j+|(| d  d@dA qW qW g gt })t   t dG |d4krht   d0k }*xttD ]}g gt|* |)|< xtt|*D ]}+g |)| |+< xt| j.|*|+  D ]},t&  t' }#|#(d;}$t/| |$|,|*|+  xp|D ]h}'|)| |+ t|dH| j dI t*|, dJ t*|*|+  dD |'  |#j+|)| |+ d  d@dA qW qW qW qhW t dK tt|dLdM}-dN}.t0 }/t12|/3dO}0|.dP|0 7 }.td0k	r|.dQ7 }.|.dR7 }.|.dS7 }.x" D ]}|.dTt*| dU 7 }.qW |.dV7 }.d9}1xttD ]}||1kr|.dW7 }.|.dT| j dU 7 }.xtt D ]z}+||1k rdX||+|d9f  dY dZ||+|d0f   }2n0dZ||+|d0 d9f  dY dZ||+|d0 d0f   }2|.dT|2 dU 7 }.qHW |.dV7 }.qW |.d[7 }.|.d\7 }.|.d]7 }.xttd0 D ]}x~tt D ]n}+|.dW7 }.|.d^t* |+  dU 7 }.|.dTdZ|| |+ d9   dU 7 }.|.dTdZ|| |+ d0   dU 7 }.|.dV7 }.	qW qW |.d[7 }.|.d_7 }.x>|D ]6}3d`|3k	rt4|3}4|.da|4 db dc |4 dd 7 }.	qW |.de7 }.x>|&D ]6}3d`|3k	rt4|3}4|.da|4 db dc |4 dd 7 }.	qW |.df7 }.x>ttD ].}|.da| j dg db | j dh 7 }.
q6W |.di7 }.x<ttD ]*}tt|| jdg dM}5dj}6|6dk| j dl 7 }6|6dm7 }6xB|(| D ]6}3d`|3k
rt4|3}4|6da|4 db dc |4 dd 7 }6
qW |6dn7 }6t   d0k }*x~tt|)| D ]j}+|6dot*|*|+  dp dq 7 }6xF|)| |+ D ]6}3d`|3kr`t4|3}4|6da|4 db dc |4 dd 7 }6q`W q2W |5|6 
qW |-|. t5dr d3S )sz
	Main TIGER post processing function. Calls the relevant function for source
	collection, odds ratio calculations, and plotting
	z"Configuration file: checking file
zFConfiguration file: checking file - file does not exist. Abort script
zConfiguration file: reading
runZ
hypotheses,	localdestc             S   s   g | ]}t |qS rX   )int).0irX   rX   rY   
<listcomp>   s    z$TigerPostProcess.<locals>.<listcomp>histnsourcesplottypesenginerunidfetchlabelslatexlabelsz	\$(.+?)\$c             S   s   g | ]}d | qS )z$%s$rX   )r_   srX   rX   rY   ra      s    rG   locscutZsnrthresZbayesgrthresseedz6Locations and types are not of the same length - exit
sourcerc             S   s   g | ]}|  qS rX   )split)r_   krX   rX   rY   ra      s    zFetching data: %s
picklezMFetch has to be either source or pickle. Check the configuration file - AbortzSaving data
z.picklez... Saving data: %s
z.datzApplying cuts
)stbtzProducing plots
Z	snrvsoddscumbayesZcumfreqbinsgffffff?gGz?   )Nbetac                s,   g | ]$ t d kr fddD qS )ry   c                s   g | ]}t d   |dqS )r   )rz   )TigerKSandPvalue)r_   ns)j	tigerrunsrX   rY   ra     s    z/TigerPostProcess.<locals>.<listcomp>.<listcomp>)len)r_   )	N_sourcesr   )r~   rY   ra     s    NTz... Histograms
c             S   s   g | ]}t |qS rX   )float)r_   itemrX   rX   rY   ra   
  s    xlimg        r   g?o   )rz   r   rx   Zhist__zsources.Ztight)Zbbox_inchesz... SNR vs. Odds
Z
snrVSodds_.z... Cumulative frequency
Zcumfreq_z... Cumulative Bayes
Z	cumbayes__catalogZ_sourcespercat_zCreating html pages
z
index.htmlwz
	<!DOCTYPE html>
	<html>

	<head>
	<title> TIGER post-processing page </title>
	</head>

	<body>

	<h1>TIGER post-processing page</h1>
	z%Y-%m-%d %H:%M:%SzLast updated on z
		<h2>Statistics</h2>
		z
		<h3>Efficiencies</h3>
		z2
		<table border="1">
		<tr>
		<td>catsize</td>
		z<td>z</td>z</tr>z<tr>z%.2f z | z%.2fz</table>z
		<h3>KS and p value</h3>
		zY
		<table border="1">
		<tr>
		<td></td>
		<td>ks stat</td>
		<td>p value</td>
		</tr>
		z<td>N=z
	<h2>Histograms</h2>
	z.pngz	<a href='z'>z
<img src='z' width=512></a>z
	<h2>SNR vs odds</h2>
	z
	<h2>Individual runs</h2>
	z.htmlz</a>&nbsp;&nbsp;&nbsp;&nbsp;z
	</body>
	</html>
	zj
		<!DOCTYPE html>
		<html>

		<head>
		<title> TIGER post-processing page </title>
		</head>

		<body>
		z<h1>z</h1>z8
		<h2>Cumulative frequency highest Bayes factor</h2>
		z5
		<h2>Cumulative Bayes factors (per catalog)</h2>
		z<h2>z sources per catalogz</h2>zScript finished)6r   rQ   r   r   r   readr   getrq   r7   ensuregetfloatgetintr   r8   open	readlinesr'   TigerSetsearchsources	pullbayespullsnrLoadPickledTigerSet
latexlabelr   joinlabelsavetopicklesavetoasciiapplycut
getbooleanTigerCalculateEfficiencyr   r   r    oddsr!   r"   r   r   Zadd_subplotTigerCreateHistogramrN   ZsavefigTigerCreateSNRvsOddsTigerCreateCumFreqrc   TigerCreateCumBayesr   r   fromtimestampstrftimebasenameprint)7rD   configtestcoefr]   re   rf   rg   ri   Zlatexlabels_strmatchesrj   Z
fetch_typeZ	fetch_locZfetch_snrthresZfetch_bayesgrthresZ
fetch_seedr`   Zlocfp	locationsZhtml_files_sourcestgZ	plot_histZplot_snrVSoddsZplot_cumbayesZplot_cumfreqZhist_odds_binsZhtml_data_efficienciesZbetasr}   tmpZhtml_data_ksZhtml_files_histZ	hist_xlimZtmplistr   ZfigZaxtypZhtml_files_snrvsoddsextZhtml_files_cumfreqZhtml_files_cumbayesZN_sources_catsr~   rr   ZhtmlfileZhtmltxtZtimesecZtimedate
backgroundZeffstrr   ZitembaseZindhtmlfileZ
indhtmltxtrX   )r   r   rY   rR      s   



	"








<


*








("



H2




*0  


"


".

"

(
rR   c               @   sJ   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d ZdddZ	dd Z
dS )TigerRunz"
	Class for a specific TIGER run
	c             C   sj   || _ |d dkr|d | _n|| _|| _|| _g | _g | _g | _g | _g | _d| _	g | _
g | _g | _dS )z
		Initialisation of class
		rs   /r   N)cluster	directoryrf   subhyp
bayesfilessnrfiles	detectorsposteriorfilesgpsrc   bayessnrposteriorsamples)selfr   r   rf   r   rX   rX   rY   __init__  s    zTigerRun.__init__c             C   sd  t d| j| jf  t   d}| jt krF|dt| j  d 7 }|d| j d | jd  7 }| jdkrv|d	7 }n| jd
kr|d7 }|d7 }xp| jD ]f}| jdkr|d| j d | 7 }|d	7 }n0| jd
kr|d| j d |	dd 7 }|d7 }|d7 }qW |d }| jt kr"|d7 }t
|ttdd}|j  }tdd |D | _~| jdkrvtdd | jD | _n"| jd
krtdd | jD | _| jdkrtdd | jD | _n"| jd
krtdd | jD | _| jdkrtdd | jD | _n"| jd
kr$tdd | jD | _tdd | jD | _t| j| _t d| j| j| jf  dS ) zP
		Search for sources completed sources in the specified cluster and location
		z... searching sources: %s	%s z
gsissh -C z 'zfor i in `ls -a r   r   lalnestz/nest/lalinferencez/posterior_samples/z"*_B.txt | xargs -n1 basename`; do ztest -f phichiz$i && zecho $i; done'T)r   stderrshellc             S   s   g | ]}| d qS )
)strip)r_   rr   rX   rX   rY   ra   L  s    z*TigerRun.searchsources.<locals>.<listcomp>c             S   s$   g | ]}| d d  dd qS )r   r9   r   r   )rq   )r_   rr   rX   rX   rY   ra   Q  s    c             S   s   g | ]}| d d qS )r   ry   )rq   )r_   rr   rX   rX   rY   ra   S  s    c             S   s   g | ]}| d dqS )z_B.txtr   )replace)r_   rr   rX   rX   rY   ra   W  s    c             S   s   g | ]}| d dqS )z_B.txtr   )r   )r_   rr   rX   rX   rY   ra   Y  s    c             S   s$   g | ]}| d d  dd qS )r   ry   r   r   )rq   )r_   rr   rX   rX   rY   ra   ]  s    c             S   s$   g | ]}| d d  dd qS )r   r9   -r   )rq   )r_   rr   rX   rX   rY   ra   _  s    c          
   S   s|   g | ]t}| d d dd dd|dd dd   d	|dd d d
  dt|dd d d
  qS )r   r   z-B.txtz_snr.txtZ	posteriorzlalinferencenest-%sry   r   r   z%srs   z%.1f)r   rq   r   )r_   rr   rX   rX   rY   ra   f  s    z*... searching sources: %s	%s - %d sources
N)r   rQ   r   r   flushclusterskeysr   rf   r   r   r   r   r   r   r   r   r   r   r   rc   )r   commandshplinesrX   rX   rY   r   .  sR    






zTigerRun.searchsourcesc             C   s  t d| j| jf  t   | jdkrBt d| j| jf  nd}x^| jD ]T}|d| j d d| j d 7 }| j	d	kr|d
7 }n| j	dkr|d7 }||7 }qNW | jdkrdnd}xzt
| |D ]h}| jt krddt| j dd|f nddd|f }t|ttdd}| jt|j dgd qW t| j| jt| jf| _td| j t d| j| jt| jd t| jd f  dS )zv
		Download Bayes factors from remote location. Only works after sources are
		found by the searchsources function.
		z!... pulling Bayes factors: %s	%sr   z4... pulling Bayes factors: %s	%s - nothing to fetch
r    {r\   }r   z/nest/r   z/posterior_samples/i     ry   z%s %s '%s %s'z	gsissh -Ccatz%s %sT)r   r   r   )usecolsztest.datz5... pulling Bayes factors: %s	%s - %d x %d dimension
N)r   rQ   r   r   r   rc   r   r   r   rf   r6   rq   r   r   r   r   r   extendr%   r#   r   r,   r$   )r   filesr   ZnsplitZfsplitr   r   rX   rX   rY   r   m  s(    
"


<zTigerRun.pullbayesc          	   C   s  t d| j| jf  | jdkr:t d| j| jf  nZd}| jdkr^|dt| j  d 7 }|d7 }d	| j}|d
| j | jd  d d | d 7 }| jdkr|d7 }t	|t
t
dd}|j  }|g krt d t| j| _dS d}xtt| jD ]}t| j| d }|dkrV| jt||| d  d| j|    |d7 }q| jt|||  d  ||d 7 }qW t| j| _t d| j| jt| jf  dS )zm
		Download SNRs from remote location. Only works after sources are found by
		the searchsources function.
		z... pulling SNRs: %s	%s
r   z+... pulling SNRs: %s	%s - nothing to fetch
r   localz
gsissh -C z 'r   r\   r   z/engine/r   r   r   T)r   r   r   z1... Warning: Cannot find SNR file. Writing zeros
Nr9   ry   z%s:zNetwork:z%... pulling SNRs: %s	%s - %d sources
)r   rQ   r   r   rc   r   r   r   r   r   r   r   r0   r   r8   r   r   r   r'   r   r   r   )r   r   Zsnrfilescommar   Z
snrrawdatacountrr   ZndetrX   rX   rY   r     s6    

(



0
"zTigerRun.pullsnrc             C   s^  | j dkrtd nDd}| jdkr:|dt| j  d 7 }|d7 }d| j}|d	| j | jd  d
 d | d 7 }| jdkr|d7 }t|t	t	dd}t
|jdgddd }tt| d}| jdkr|dt| j  d 7 }|d7 }d| j}|d	| j | jd  d
 d | d 7 }| jdkr0|d7 }t|t	t	dd}t
|j| _tt| j dS )z
		Download posterior pdfs from remote location. Only works after sources are
		found by the searchsources function.
		NB: NOT READY FOR USE YET!
		r   zNothing to fetchr   r   z
gsissh -C z 'zwc -lr\   r   z/nest/r   r   r   T)r   r   r   )r   Nrs   r   )rc   r   r   r   r   r   r   r   r   r   r%   r   r$   r   )r   r   Zposteriorfilescommar   ZposteriorfilesizesrX   rX   rY   pullposteriors  s0    

(

(zTigerRun.pullposteriors       @      @@c             C   sj   | j |k| j dkB | jdddf |k@ }| j|ddf | _| j | | _ | j| | _t| jd | _dS )z2
		Apply a Bayes factor and SNR cut on the data
		g        Nr   )r   r   r   r$   rc   )r   ru   rv   ZcondrX   rX   rY   r     s
    *zTigerRun.applycutc             C   s   t |d}t| |d dS )z.
		Pull data from remote or local locations
		wbr9   N)r   r   )r   rF   frX   rX   rY   r     s    
zTigerRun.savetopickleN)r   r   )__name__
__module____qualname____doc__r   r   r   r   r   r   r   rX   rX   rX   rY   r     s   ?%$
r   c               @   sl   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d ZdddZ	dddZ
dd Zdd Zdd Zdd ZdS )r   z`
	CLASS TO CONTAIN A SET OF RUNS UNDER A COMMON CHARACTERISTIC (E.G.
	BACKGROUND, FOREGROUND).
	c                s   | _ g  _x:tt j D ](}t||d } jdd |D  qW  jdd t j _|dkrr| _nt	d  fdd|D  _
| _| _d _tdt jf _td _d	S )
z
		INITIALISATION OF CLASS
		ry   c             S   s   g | ]}d  |qS )r   )r   )r_   r~   rX   rX   rY   ra     s    z%TigerSet.__init__.<locals>.<listcomp>r   GR)r   r   zEngine not recognisedc                sD   g | ]<}|d    t dg krt|d    |d  j jqS )r   r   ry   )lowerr   r   r   rf   r   )r_   loc)r   rX   rY   ra     s    N)r   r   r8   r   r	   r   insertnsubhyprf   r   r   r   r   rc   r   r   r   )r   r   r   r   r   rf   r`   rk   rX   )r   rY   r     s     zTigerSet.__init__c             C   s<   x&| j D ]}|  |  j|j7  _qW td| j  dS )z0
		FIND COMPLETED JOBS AND GET THE FILESNAMES
		z4--> searching sources: Total number of sources - %d
N)r   r   rc   r   rQ   )r   r   rX   rX   rY   r     s    zTigerSet.searchsourcesc             C   s^   x2| j D ](}|jdkr|  t| j|jf| _qW tdt| jd t| jd f  dS )z.
		PULL DATA FROM REMOTE OR LOCAL LOCATIONS
		r   z1--> pulling Bayes factors: matrix size - %d x %d
ry   N)r   rc   r   r&   r   r   rQ   r$   )r   r   rX   rX   rY   r     s
    
zTigerSet.pullbayesc             C   sL   x2| j D ](}|jdkr|  t| j|jf| _qW tdt| j  dS )z.
		PULL DATA FROM REMOTE OR LOCAL LOCATIONS
		r   z---> pulling SNRs: total number of sources %d
N)r   rc   r   r    r   r   rQ   r   )r   r   rX   rX   rY   r   #  s
    
zTigerSet.pullsnrc             C   s&   x | j D ]}|jdkr|  qW dS )z!
		PULLING THE POSTERIOR FILES
		r   N)r   rc   r   )r   r   rX   rX   rY   r   -  s    
zTigerSet.pullposteriorsry   c             C   sX   t d}xJt| j| D ]8}t|t| j|| |d | ddf t| j}qW |S )z2
		CALCULATE THE TOTAL ODDS FROM DATA STRUCTURE
		r   ry   N)r   r8   rc   r'   OddsFromBayesr   r   r   )r   rz   r   r`   rX   rX   rY   r   5  s    8zTigerSet.odds       @      @@c             C   s   t d| j| jf  x&| jD ]}|jdkr|j||d qW tdt| jf| _	td| _
x<| jD ]2}|jdkrdt| j	|j	f| _	t| j
|j
f| _
qdW t| j	d | _t d| j| jf  dS )z,
		APPLY DETECTION CUTS (SNR AND BAYESGR)
		z ... %s - before cut: %d sources
r   )ru   rv   z... %s - after cut: %d sources
N)r   rQ   r   rc   r   r   r   r   r   r   r   r&   r    r$   )r   ru   rv   r   rX   rX   rY   r   >  s    


zTigerSet.applycutc             C   sn   t dt| | jdkrj| jt| jd krjt| j}t| t| | j|ddf | _| j	| | _	dS )z,
		APPLY DETECTION CUTS (SNR AND BAYESGR)
		z... Shuufling with seedr   N)
r   rN   rc   r$   r   r(   r)   rn   shuffler   )r   rn   orderrX   rX   rY   shufflesourcesS  s    


zTigerSet.shufflesourcesc             C   s   t |d}t| |d dS )z2
		SAVE DATA TO PICKLE FOR QUICK FUTURE LOADING
		r   r9   N)r   r   )r   rF   r   rX   rX   rY   r   d  s    
zTigerSet.savetopicklec             C   s   t ddt| j f}xZ| jD ]P}|jdkrt|jg|j |jg|j f}t||j|j	|j
f}t||f}qW ttdddg| jdgf}t||f}t||ddd	 d
S )zR
		SAVE DATA TO ASCII FILE FOR QUICK FUTURE LOADING.
		NB: BROKEN - DO NOT USE!
		r      z#clusterfolderr   Znetsnr	z%s)	delimiterfmtN)r   r   r   r   rc   r*   r   r   r   r   r   r&   r   r+   r,   )r   filenameZsavedatar   ZlocsdataZsavedatatmpheaderrX   rX   rY   r   m  s    
 zTigerSet.savetoasciic             C   sd  x\| j D ]P}td d}t|d}|d |d ||j |d || j |d |t|j |  td d}|d7 }|d	7 }|t	7 }|d	7 }||7 }|d	7 }|t
|j d
 7 }t| t|ttdd}|  |jdkrTdt
|j  d tt	 d | d }t| t|ttdd}t|j  t|j  td q
W dS )z\
		PREPROCESS DATA ON THE CLUSTERS BEFORE SENDING DATA TO MASTER.
		NB: UNDER DEVELOPMENT
		zWriting preprocessfileztig_preprocess_tmp.txtr   r   r   zWriting preprocessfile - doner   z	gsiscp -Cr   z:~/T)r   r   r   r   zgsissh z	 'python z -p r   N)r   r   r   rQ   r   rf   rN   r   closescriptfilenamer   r   r   r   communicate
returncoder   r   r   r   r   r   )r   r   Zpreprocesstmppreprocessfpr   r   rX   rX   rY   
preprocess}  s>    




(zTigerSet.preprocessN)ry   )r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rX   rX   rX   rY   r     s    

	
	r   c             C   s   t d|  t| d}t|S )z
	LOAD FROM PICKLE
	z... Loading file from picklerb)r   r   r   )r   fprX   rX   rY   r     s    

r   c             C   s   t | st|  dS )z)
	CREATE DIRECTORY IF IT DOES NOT EXIST
	N)r   existsr   )r   rX   rX   rY   r     s    
r   c             C   sB   d}| |kr&| t dt||    }n|t dt| |   }|S )z(
	FUNCTION TO ADD IN LOGARITHMIC SPACE
	g        g      ?)r-   r.   )xyoutputrX   rX   rY   lnPLUS  s
    r  c             C   sn   t j }xJtt| d d D ]2}t|t| dd|d f | dddf  }qW |ttd|d 8 }|S )z0
	CALCULATE THE TOTAL ODDS FROM DATA STRUCTURE
	ry   Nr   r9   )r   r"   r8   r$   r  r4   r-   pow)matrixZNhypZ	TotalOddsr~   rX   rX   rY   r     s
    2r   ry   2   c             C   s"  xt t| D ]}|dkr|j| | |d| | jd tt| | | d dd||t| t| dd
 |	d |
d	 nv|| | jk r|j| | |d| | jd tt| | | d
 dd||t| t| dd
 |	d |
d | \}}|j||dd qW dS )z4
	CREATE TIGER HISTOGRAMS FROM A LIST OF TIGERRUNS
	ry   nonez
$\mathrm{(z\; sources)}$Z
stepfilledT)	Z	facecolorr   Zhisttype	linewidthr8   rx   ZhatchZ	edgecolorZdensityz$\ln{O_{GR}^{modGR}}$z$P(\ln{O_{GR}^{modGR}})$z\; catalogs)}$z$\ln{\mathcal{O}_{GR}^{modGR}}$z"$P(\ln{\mathcal{O}_{GR}^{modGR}})$z
upper left)r   N)r8   r   rb   r   r   rN   r/   hatch_defaultcolor_default
set_xlabel
set_ylabelrc   get_legend_handles_labelslegend)list_tigerrunsaxisrz   r   rx   r`   handlesri   rX   rX   rY   r     s:    &
&

r   gffffff?c                s2  g } fdd|D }t td t|f}xttD ]} | jkrtd| j f  |k rd||ddf< q@d||d ddf< q@|kr@|  }xxtt|D ]h}||| k}	||| k }
t||	 }t|}t|t| }|k r||||f< q|||d |f< qW q@W |S )z1
	CALCULATE EFFICIENCY FROM A LIST OF TIGERRUNS
	c                s$   g | ]}t   |gd qS ))Zprob)r   r   )r_   b)rz   r   r  rX   rY   ra     s    z,TigerCalculateEfficiency.<locals>.<listcomp>ry   zb... Warning: Not sufficient events (%s) to calculate the efficiency for %s sources. Writing zeros
g        N)r   r   r8   rc   r   rQ   r   r   )r  rz   r{   r   ZefficienciesZOddsBetar`   r   r~   ZmskZnmskZ
nabovebetaZntotalZeffrX   )rz   r   r  rY   r     s,    
r   c             C   s^   | j |k s|j |k r4td| j |j |f  ddgS | |}||}t||\}}||gS )z4
	CALCULATE KS STATISTICS FROM A LIST OF TIGERRUNS
	zc... Warning: Not sufficient events (%s,%s) to calculate KS-statistic for %s sources. Writing zeros
g        )rc   r   rQ   r   r   )Z	tigerrun1Z	tigerrun2rz   Zdata1Zdata2ZksstatZpvaluerX   rX   rY   r|     s    

r|   c          
   C   s   ddg}ddg}| d |d x\tt| D ]L}|j| | j| |  || || | | jd t| | j	 d d	d
 q2W |
dd | \}}|j||dd dS )z8
	CREATE SNR VS ODDS DIAGRAMS FROM A LIST OF TIGERRUNS
	r   or   r   z$\mathrm{SNR}$z'$\ln{O_{\mathrm{GR}}^{\mathrm{modGR}}}$z
$\mathrm{(z\; sources)}$@   )colormarkerr   rk   g      @g      I@best)r   N)r  r  r8   r   Zscatterr   r   r   rN   rc   set_xlimr  r  )r  r  markerscolorsr`   r  ri   rX   rX   rY   r   '  s    

r   c          
   C   s  ddg}t |d |d d}t|}t|| jf}xxt|D ]l}|| t| jkr<| j|| k }| j|ddf }tt	|dd}	x$t| jD ]}
|	
|
|||
f< qW q<W t|dddf | jkrtd|d |d | jt|dddf  f  d	td
< tddddg}tddddddddg}tdddddddg}xt| jD ]}| j| dd dkr|j||dd|f d d!d"d# n| jd$krd%d&| j| dd d' d( }n4| jd)krd%d&| j| dd d* d( }|j||dd|f || dd+| d, qHW |d- |d. || | \}}|j||d/d0d1 | }|j| j||d2dd3 |d4 |d5 || dS )6zA
	CREATE CUMULATIVE FREQUENCY DIAGRAMS FROM A LIST OF TIGERRUNS
	g      @g      Y@r   ry   N)r  rs   zmWarning, some sources maybe out of the SNR range and are not plotted: SNR \in [%.1f:%.1f], %i sources missing   zlegend.fontsizer   z--z-.:+*r   <d^r   rk   r  grp   cmr  rr   r   r   z$\mathcal{H}_{\mathrm{GR}}$blackg      @)r   r  r  r   z$H_{r   dphiz}$r   dchig      ?)r   r  	linestyler  r  z$\mathrm{SNR}$z)$\mathrm{Cumulative\;frequency\;(lines)}$r  r   )r   ncolg      ?)r8   rx   alphar  Fz1$\mathrm{\#\;of\;sources\;per\;SNR\;bin\;(hist)}$)r(   r   r0   r   r8   r!   r   r   listr1   r   r4   rc   r   r   r
   r   rq   rd   rf   r   nextr  r  r  r  r  Ztwinxrb   grid)tigerrunr  r   ZsnrrangeZ	nsnrstepsZfreqsr`   ZsnrmaskZbayesmaskedmaxarrayr~   Z
linestylesmarkerstylesZcolorstylescurlabelr  ri   Zax2rX   rX   rY   r   @  sP    0"*(




r   c          	      s  j || |d |  }j|| |d | ddf }t|}t|}||ddf  t fddt|D }t fddt|D }	tdddd	d
dddg}
|d |d xtj	d D ]}j
dkrddj|d  dd d d }n8j
dkrDddj|d  dd d d }|j||	dd|f ||
 ddd qW |j||dddd |d d! | \}}|j||d"d#d$ dS )%z]
	CREATE CUMULATIVE BAYES FACTOR/ODDS DIAGRAMS FROM A LIST OF TIGERRUNS (FOR
	EACH CATALOG)
	ry   Nc                s0   g | ](}t  d |d d d f tjqS )Nry   )r   r   r   )r_   r`   )bayescatsortedr.  rX   rY   ra     s    z'TigerCreateCumBayes.<locals>.<listcomp>c          	      sF   g | ]>}t t d |d dd f j d |d df jddqS )Nry   r   )r  )r4   r5   T)r_   r`   )r2  rX   rY   ra     s    r  r  r   r  r   r!  r   rk   z$\mathrm{SNR}$z5$\ln{{}^{(cat)}B^{i_1 i_2 \ldots i_k}_{\mathrm{GR}}}$r   z$H_{r   r   rs   r&  z}$r   r'  z--g      ?)r   r  r(  r  z$\ln{O_{GR}^{modGR}}$g       @r%  )r   r  r  g      @g      I@r  r   )r   r)  )r   r   r2   r3   r   r8   r
   r  r  r   rf   r   r   rq   rd   r,  r  r  r  )r.  r  ZnthcatZnsourcespercatZsnrcatZbayescatZ	sortorderZsnrcatsortedZcumoddsrw   r0  r`   r1  r  ri   rX   )r2  r.  rY   r   y  s0    "

.,r   c             C   sL   t | }t | }|dkr d}t| tt ||d}d}|| dS )zA
	CREATE EXAMPLE CONFIGURATION FILE (TO BE USED WITH -G OPTION)
	r   z./r   aR  ###############################################################################
#
# Configuration file for tiger_postproc.py
#
###############################################################################

#------------------------------------------------------------------------------
[run]
#------------------------------------------------------------------------------

# Engine for TIGER runs: lalnest or lalinference
engine=engine1,engine2

# Subhypotheses: Notation depends on engine.
#		lalnest: dphi1,dphi2,...
#		lalinference: dchi1,dchi2,...
hypotheses=dphi1,dphi2,dphi3

# Run identifier (used in filenames for plots)
runid=name_for_run

# Local destination folder for postproc files: created if non-existing
localdest=/home/user/public_html/tiger/inj_tt4spin_tt4all_rec_tf2spin/

#------------------------------------------------------------------------------
[fetch]
#------------------------------------------------------------------------------

# Fetch data from (one per type of runs)
#		- source : directly from output of engine
#		- pickle : saved state
type=source,source

# Location files.(one per type of runs)
# 	source: ascii file two columns: cluster directory
# 		cluster: (atlas, nemo, cit, lho, llo, or local)
# 	pickle: location to python pickle file
locs=/path/to/the/location/file.txt,/path/to/the/location/file.txt

# Labels (one for each type of runs)
labels=tt4spin,tt4all

# Latex labels for different runs for plot legends (one for each run type).
# Currently not working with commas in the latex itself!
latexlabels=$\mathrm{TaylorT4+spin}$,$\mathrm{TaylorT4+all}$

# Seed for shuffling the sources. seed=0 means no shuffling
# NB: NOT WORKING!
seed=0

#------------------------------------------------------------------------------
[cut]
#------------------------------------------------------------------------------

# SNR threshold
snrthres=8.0

# B^GR_noise cutoff
bayesgrthres=32.0

#------------------------------------------------------------------------------
[plot]
#------------------------------------------------------------------------------

# Plot histograms (see [hist] for plotting options)
hist=true

# Plot SNR vs odds scatter plot (see [snrvsodds] for plotting options)
snrvsodds=true

# Plot cumulative frequency of a subhypothesis having the highest Bayes factor
# (see [cumfreq] for plotting options)
cumfreq=true

# Plot cumulative Bayes factor, per subhypothesis, and the odds ratio, as a
# function of SNR, per catalog. (see [cumbayes] for plotting options).
cumbayes=false

# File output extensions: png required for html page
# Other supported formats: pdf, ps, eps, svg
types=png,pdf

#------------------------------------------------------------------------------
[hist]
#------------------------------------------------------------------------------

# Number of bins in the histograms
bins=50

# Number of sources per catalog for the histograms
nsources=1,15

# Limits on the x-axis. Use xlim=0,0 for automatic determination (range will be
# the same for ALL sizes of the catalog).
xlim=0,0

#------------------------------------------------------------------------------
[snrvsodds]
#------------------------------------------------------------------------------


#------------------------------------------------------------------------------
[cumbayes]
#------------------------------------------------------------------------------

N)r   r   dirnamer   r   r   rQ   )r   r   r4  outfileZexamplecfgtxtrX   rX   rY   rS     s    

lrS   c             C   s   t | trt| d}| d}|d dd ddddd	}t|d
  |d |d |}|  |	  |
  |d td
 ntd dS )z6
	TIGER PREPROCESSOR FUNCTION
	NB: UNDER DEVELOPMENT
	rp   r   r   ry   rs   r   r   r   r\   r   r9   ztigerloc_preprocess.picklez'Preprocess file cannot be found - abortN)r   r   r   r   rq   r   r   r   r   r   r   r   r   )rJ   r   rV   r   ZtglocrX   rX   rY   rT      s    

&

rT   __main__)ry   Nr  )ry   )mr   
__author____credits__Z__maintainer__	__email__
__status__r   r
  r	  configparserr   rt   r   r   r   	itertoolsr	   r
   Z
matplotlibr   r   r   Zmpl_versionZmatplotlib.pyplotr   r   osr   r   r   r   Zscipy.statsr   Zscipy.stats.mstatsr   
subprocessr   r   sysr   r   r   r   r   numpyr   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   rer7   Z	six.movesr8   rL   optparser;   argparser<   Zfig_width_ptZinches_per_ptZgolden_meanZ	fig_widthZ
fig_heightZfig_sizeZfontsizeparamsupdaterealpath__file__r   rZ   rR   r   r   r   r   r  r   r   r   r|   r   r   r   rS   rT   r   rX   rX   rX   rY   <module>   s   

p


-    U E
(
9*}
