B
    d4                 @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ d dlmZmZmZ d dlmZm Z  d dlm!Z! d dl"m#Z# G dd de j$Z%G dd de&Z'G dd de'Z(G dd de(Z)G dd de&Z*G dd de&Z+G dd de&Z,G dd de&Z-G dd  d e&Z.d!d"d#d$d%d&d'd(gZ/d)d* Z0d+d, Z1d-d. Z2d/d0 Z3d1d2 Z4d3d4 Z5d5d6 Z6dd7ej7fd8d9Z8d?d:d;Z9d@d<d=Z:d>Z;dS )A    N)chain)BytesIO)LIGOTimeGPSYRJUL_SI)ligolw)	lsctables)utils)process)version)return_search_summaryreturn_empty_sngl)eventsconversionspnutils)rankingveto)mean_if_greater_than_zero)mass1_mass2_to_mchirp_etac               @   s   e Zd ZdZdd ZdS )HFilezF Low level extensions to the capabilities of reading an hdf5 File
    c                sj  i i  x |D ]}| | |< g  |< qW | dd}tjg tjd}| dtd}t| }dx|k r| |k r| n|fdd|D }	||	 }
|rt|t|
 g}x(t||	D ]\}} | 	||
  qW |7 qhW t|d	kr4t |d  }|r.|
tj|fS |S n2t fd
d|D }|rb|
tjf| S |S dS )a   Return arrays from an hdf5 file that satisfy the given function

        Parameters
        ----------
        fcn : a function
            A function that accepts the same number of argument as keys given
            and returns a boolean array of the same length.

        args : strings
            A variable number of strings that are keys into the hdf5. These must
            refer to arrays of equal length.

        chunksize : {1e6, int}, optional
            Number of elements to read and process at a time.

        return_indices : bool, optional
            If True, also return the indices of elements passing the function.

        Returns
        -------
        values : np.ndarrays
            A variable number of arrays depending on the number of keys into
            the hdf5 file that are given. If return_indices is True, the first
            element is an array of indices of elements passing the function.

        >>> f = HFile(filename)
        >>> snr = f.select(lambda snr: snr > 6, 'H1/snr')
        return_indicesF)dtype	chunksizeg    .Ar   c                s   g | ]}|   qS  r   ).0arg)irrefsr   Y/work/yifan.wang/ringdown/master-ringdown-env/lib/python3.7/site-packages/pycbc/io/hdf.py
<listcomp>P   s    z HFile.select.<locals>.<listcomp>   c             3   s   | ]}t  | V  qd S )N)npconcatenate)r   r   )datar   r   	<genexpr>c   s    zHFile.select.<locals>.<genexpr>N)getr!   arrayZuint64intlenr"   flatnonzerozipappendastypetuple)selfZfcnargskwdsr   r   indicesr   sizepartialkeeppartresr   )r#   r   r   r   r   select   s8    

zHFile.selectN)__name__
__module____qualname____doc__r7   r   r   r   r   r      s   r   c               @   sJ   e Zd ZdZdddZdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dS )	DictArraya$   Utility for organizing sets of arrays of equal length.

    Manages a dictionary of arrays of equal length. This can also
    be instantiated with a set of hdf5 files and the key values. The full
    data is always in memory and all operations create new instances of the
    DictArray.
    Nc             C   s   |r|rt d|r |s t d|| _|| _|ri | _x|D ]}g | j|< q<W xN|D ]F}t|}x0|D ](}||krf| j| || dd  qfW |  qTW x6| jD ],}t| j| dkst| j| | j|< qW x | jD ]}t	| || j|  qW dS )au   Create a DictArray

        Parameters
        ----------
        data: dict, optional
            Dictionary of equal length numpy arrays
        files: list of filenames, optional
            List of hdf5 file filenames. Incompatibile with the `data` option.
        groups: list of strings
            List of keys into each file. Required by the files option.
        z9DictArray can only have data or files as input, not both.z$If files are given then need groups.Nr   )
RuntimeErrorr#   groupsr   r+   closer(   r!   r"   setattr)r.   r#   filesr>   gfdkr   r   r   __init__r   s*    


 zDictArray.__init__c             C   s   | j |dS )N)r#   )	__class__)r.   r#   r   r   r   _return   s    zDictArray._returnc             C   s   t | jt| j d  S )Nr   )r(   r#   r-   keys)r.   r   r   r   __len__   s    zDictArray.__len__c          	   C   sf   i }xT| j D ]J}y"t| j | |j | g||< W q tk
rT   td|  Y qX qW | j|dS )Nz%s does not exist in other data)r#   )r#   r!   r"   KeyErrorlogginginforH   )r.   otherr#   rE   r   r   r   __add__   s    "zDictArray.__add__c             C   s2   i }x | j D ]}| j | | ||< qW | j|dS )zC Return a new DictArray containing only the indexed values
        )r#   )r#   rH   )r.   idxr#   rE   r   r   r   r7      s    zDictArray.selectc             C   s@   i }x.| j D ]$}t| j | tj|td||< qW | j|dS )zI Return a new DictArray that does not contain the indexed values
        )r   )r#   )r#   r!   deleter&   r'   rH   )r.   rP   r#   rE   r   r   r   remove   s    $zDictArray.removec             C   s`   t |d}x| jD ]}| j| |j|< qW x(| jD ]}|j|| j| dddd q2W |  d S )Nwgzip	   T)r#   compressionZcompression_optsshuffle)r   attrsr#   create_datasetr?   )r.   outnamerC   rE   r   r   r   save   s    
zDictArray.save)NNN)r8   r9   r:   r;   rF   rH   rJ   rO   r7   rR   r[   r   r   r   r   r<   j   s   
(	r<   c                   s:   e Zd Zd fdd	Zdd Zdd Z fd	d
Z  ZS )StatmapDataNstattime1time2Ztrigger_id1Ztrigger_id2template_iddecimation_factortimeslide_idc                sP   t t| j|||d |r(|| _|| _n$|rLt|d d}|d | _|j| _d S )N)r#   rA   r>   r   r   segments)superr\   rF   segrX   r   )r.   r#   rf   rX   rA   r>   rC   )rG   r   r   rF      s    
zStatmapData.__init__c             C   s   | j || j| jdS )N)r#   rX   rf   )rG   rX   rf   )r.   r#   r   r   r   rH      s    zStatmapData._returnc             C   sZ   t | jdkst | jdkr | S ddlm} | jd }|| j| j| j| j||}| |S )zy Cluster the dict array, assuming it has the relevant Coinc colums,
        time1, time2, stat, and timeslide_id
        r   )cluster_coincstimeslide_interval)	r(   r_   r`   pycbc.eventsrg   rX   r^   rc   r7   )r.   windowrg   intervalcidr   r   r   cluster   s    
zStatmapData.clusterc          	      sz   t t| | t|dV}xN| j D ]@}| j| d d d  |d| < | j| d d d  |d| < q(W W d Q R X d S )NrS   startzsegments/%s/startendzsegments/%s/end)re   r\   r[   r   rf   rI   )r.   rZ   rC   key)rG   r   r   r[      s
    zStatmapData.save)NNNNr]   )r8   r9   r:   rF   rH   rm   r[   __classcell__r   r   )rG   r   r\      s
    r\   c                   s.   e Zd Zd fdd	Zdd Zdd Z  ZS )	MultiifoStatmapDataNc                sT   ddddg}x(|D ] }|d| g7 }|d| g7 }qW t t| j|||||d d S )Nrb   r^   ra   rc   z%s/timez%s/trigger_id)r#   rA   r>   rX   rf   )re   rr   rF   )r.   r#   rf   rX   rA   ifosr>   ifo)rG   r   r   rF      s    
zMultiifoStatmapData.__init__c             C   s&   | j d d}| j|| j | j|dS )Nrs    )r#   rX   rf   rs   )rX   splitrG   rf   )r.   r#   Zifolistr   r   r   rH      s    zMultiifoStatmapData._returnc             C   s   | j d }| j d }t| jd|  dks@t| jd|  dkrD| S ddlm} | j d }|| j| jd|  | jd|  | j||}| |S )zy Cluster the dict array, assuming it has the relevant Coinc colums,
        time1, time2, stat, and timeslide_id
        ZpivotZfixedz%s/timer   )rg   rh   )rX   r(   r#   ri   rg   r^   rc   r7   )r.   rj   Z	pivot_ifoZ	fixed_iforg   rk   rl   r   r   r   rm      s    

,
zMultiifoStatmapData.cluster)NNNNN)r8   r9   r:   rF   rH   rm   rq   r   r   )rG   r   rr      s    
rr   c               @   s2   e Zd Zd
ddZdd Zedd Zdd	 ZdS )FileDataNc             C   s   |st d|| _t|d| _|dkrNt| j dkrF| j \}nt d|| _| j| | _|dk	rl|nt| j | _	|| _
d| _dS )a  
        Parameters
        ----------
        group : string
            Name of group to be read from the file
        columnlist : list of strings
            Names of columns to be read; if None, use all existing columns
        filter_func : string
            String should evaluate to a Boolean expression using attributes
            of the class instance derived from columns: ex. 'self.snr < 6.5'
        zDidn't get a file!r   Nr    zDidn't get a group!)r=   fnamer   h5filer(   rI   	group_keygrouplistcolumnsfilter_func_mask)r.   rx   r{   
columnlistr~   r   r   r   rF     s     zFileData.__init__c             C   s   | j   d S )N)ry   r?   )r.   r   r   r   r?   0  s    zFileData.closec             C   sh   | j dkrtdnP| jdkr^x2| jD ](}|| j kr&t| || j| dd  q&W t| j | _| jS dS )z
        Create a mask implementing the requested filter on the datasets

        Returns
        -------
        array of Boolean
            True for dataset indices to be returned by the get_column method
        Nz+Can't get a mask without a filter function!)r~   r=   r   r}   r@   r{   eval)r.   columnr   r   r   mask3  s    




zFileData.maskc             C   sB   t | j stg S | j| }| jr2|| j S |dd S dS )z
        Parameters
        ----------
        col : string
            Name of the dataset to be returned

        Returns
        -------
        numpy array
            Values from the dataset, filtered if requested
        N)r(   r{   rI   r!   r&   r~   r   )r.   colvalsr   r   r   
get_columnI  s    


zFileData.get_column)NNN)r8   r9   r:   rF   r?   propertyr   r   r   r   r   r   rw     s   
rw   c               @   s   e Zd ZdddZdd ZdS )DataFromFilesNc             C   s   || _ || _|| _|| _d S )N)rA   r{   r}   r~   )r.   filelistr{   r   r~   r   r   r   rF   a  s    zDataFromFiles.__init__c             C   sv   t d|  g }x<| jD ]2}t|| j| j| jd}||| |	  qW t dt
dd |D   t|S )a^  
        Loop over files getting the requested dataset values from each

        Parameters
        ----------
        col : string
            Name of the dataset to be returned

        Returns
        -------
        numpy array
            Values from the dataset, filtered if requested and
            concatenated in order of file list
        z
getting %s)r{   r   r~   z- got %i valuesc             s   s   | ]}t |V  qd S )N)r(   )r   vr   r   r   r$     s    z+DataFromFiles.get_column.<locals>.<genexpr>)rL   rM   rA   rw   r{   r}   r~   r+   r   r?   sumr!   r"   )r.   r   r   rC   rD   r   r   r   r   g  s    
zDataFromFiles.get_column)NNN)r8   r9   r:   rF   r   r   r   r   r   r   _  s   
r   c               @   s  e Zd ZdZdPddZdd Zdd Zd	d
 Zedd Z	dd Z
dQddZedd Zedd Zedd Zedd Zedd Zedd Zedd Zed d! Zed"d# Zed$d% Zed&d' Zed(d) Zed*d+ Zed,d- Zed.d/ Zed0d1 Zed2d3 Zed4d5 Zed6d7 Zed8d9 Z ed:d; Z!ed<d= Z"ed>d? Z#ed@dA Z$edBdC Z%edDdE Z&edFdG Z'edHdI Z(edJdK Z)dLdM Z*dNdO Z+dS )RSingleDetTriggerszQ
    Provides easy access to the parameters of single-detector CBC triggers.
    Nc             C   s  t d t|d| _| j| | _|| _|| _|rJt d t|d| _nt d i | _|d k	rj|| _nt	j
t| jd td| _|rt d t d| j  tjj| j|g||d	\| _}t	| j| j }	d
| jd d < d| j|	< t dt| j |rt d x<| j D ].}
|
|krt| d|
 | j|
 d d   qW xL| j D ]>}
|
|krXt| d|
 t	| j|
 | jd d d    qXW t|dd| _x8t| j | j D ]}
|
|krt| d|
  qW | j| j@ | _t dt| j| d S )NzLoading triggersr   zLoading bankzNo bank file givenend_time)r   zApplying veto segmentsz%i triggers before vetoes)rt   segment_nameFTz%i triggers remain after vetoeszSetting up filter function_ra   zself.zself._z"%i triggers remain after cut on %s)rL   rM   r   Ztrigs_ftrigsrt   detectorbankr   r!   Zonesr(   boolr   r   r   Zindices_outside_segmentsr   Z	veto_maskr)   rI   r@   r&   r   replaceZfilter_maskr   delattr)r.   Z	trig_file	bank_fileZ	veto_filer   r~   r   Zpremaskr   rP   cr   r   r   rF     sP    






$

*
 zSingleDetTriggers.__init__c          
   C   s   y
|  |S  tk
r   Y nX y"| | | j| d d  | j S  ttfk
r| } zd|}t||W d d }~X Y nX d S )NzCannot find {} in input files)r   rK   	checkbankr   ra   r=   format
ValueError)r.   rp   excerr_msgr   r   r   __getitem__  s    


zSingleDetTriggers.__getitem__c             C   s   | j i krtd| S d S )Nz'Can't get %s values without a bank file)r   r=   )r.   paramr   r   r   r     s    
zSingleDetTriggers.checkbankc             C   sj   i }x`| j D ]V}t| j | t| j d kr| jdk	rL| j | | j ||< q| j | dd ||< qW |S )z*Returns dict of the masked trigger valuse r   N)r   r(   r   )r.   ZmtrigsrE   r   r   r   	trig_dict  s    
zSingleDetTriggers.trig_dictc             C   s   dd t | D S )z3Returns a list of plottable CBC parameter variablesc             S   s$   g | ]}t |d  tkr|d qS )r    r   )typer   )r   mr   r   r   r     s    z5SingleDetTriggers.get_param_names.<locals>.<listcomp>)inspect
getmembers)clsr   r   r   get_param_names  s    z!SingleDetTriggers.get_param_namesc             C   s^   t | jdrD| jjdkrD| j d | }d| jdd< d| j|< ntt| j| | _dS )z,Apply a boolean array to the set of triggersr   r   r   FNT)hasattrr   r   nonzeror|   r!   r&   )r.   Z
logic_maskorig_indicesr   r   r   
apply_mask  s
    zSingleDetTriggers.apply_mask
   c             C   s   | | j|  f}t|dkr0tg | _dS | j}| ddd }g }g }xV|D ]N}	||	 }
x2|D ]}t	|
| |k rhP qhW |
|	 |
|
 t||krVP qVW t|}|  || | _t| jdr| jjdkrt| j| }t|| _n$t| jtrtt| j| | _dS )zEdits the mask property of the class to point to the N loudest
        single detector events as ranked by ranking statistic. Events are
        clustered so that no more than 1 event within +/- cluster-window will
        be considered.r   Nr   r   )Zrank_stat_singlert   r   r(   r!   r&   r^   r   argsortabsr+   sortr   r   r   r)   r|   
isinstance)r.   Zrank_method	n_loudestZcluster_windowr^   timesindexZ	new_timesZ	new_indexZcurr_idxZ	curr_timetimer   r   r   r   "mask_to_n_loudest_clustered_events  s2    	





z4SingleDetTriggers.mask_to_n_loudest_clustered_eventsc             C   s   |  dtS )Nra   )r   r,   r'   )r.   r   r   r   ra     s    zSingleDetTriggers.template_idc             C   s"   |  d | jd d d  | j S )Nmass1)r   r   ra   )r.   r   r   r   r     s    
zSingleDetTriggers.mass1c             C   s"   |  d | jd d d  | j S )Nmass2)r   r   ra   )r.   r   r   r   r   "  s    
zSingleDetTriggers.mass2c             C   s"   |  d | jd d d  | j S )Nspin1z)r   r   ra   )r.   r   r   r   r   '  s    
zSingleDetTriggers.spin1zc             C   s"   |  d | jd d d  | j S )Nspin2z)r   r   ra   )r.   r   r   r   r   ,  s    
zSingleDetTriggers.spin2zc             C   s"   |  d | jd d d  | j S )Nspin2x)r   r   ra   )r.   r   r   r   r   1  s    
zSingleDetTriggers.spin2xc             C   s"   |  d | jd d d  | j S )Nspin2y)r   r   ra   )r.   r   r   r   r   6  s    
zSingleDetTriggers.spin2yc             C   s"   |  d | jd d d  | j S )Nspin1x)r   r   ra   )r.   r   r   r   r   ;  s    
zSingleDetTriggers.spin1xc             C   s"   |  d | jd d d  | j S )Nspin1y)r   r   ra   )r.   r   r   r   r   @  s    
zSingleDetTriggers.spin1yc             C   s"   |  d | jd d d  | j S )Ninclination)r   r   ra   )r.   r   r   r   r   E  s    
zSingleDetTriggers.inclinationc             C   s"   |  d | jd d d  | j S )Nf_lower)r   r   ra   )r.   r   r   r   r   J  s    
zSingleDetTriggers.f_lowerc             C   s"   |  d | jd d d  | j S )Napproximant)r   r   ra   )r.   r   r   r   r   O  s    
zSingleDetTriggers.approximantc             C   s   | j | j S )N)r   r   )r.   r   r   r   mtotalT  s    zSingleDetTriggers.mtotalc             C   s   t | j| jS )N)r   Zmchirp_from_mass1_mass2r   r   )r.   r   r   r   mchirpX  s    zSingleDetTriggers.mchirpc             C   s   t | j| jS )N)r   Zeta_from_mass1_mass2r   r   )r.   r   r   r   eta\  s    zSingleDetTriggers.etac             C   s   t | j| j| j| jS )N)r   Zchi_effr   r   r   r   )r.   r   r   r   effective_spin`  s    z SingleDetTriggers.effective_spinc             C   s   t d| j| j| j| jS )NZfSEOBNRv2Peak)r   get_freqr   r   r   r   )r.   r   r   r   f_seobnrv2_peaki  s    z!SingleDetTriggers.f_seobnrv2_peakc             C   s   t d| j| j| j| jS )NZfSEOBNRv4Peak)r   r   r   r   r   r   )r.   r   r   r   f_seobnrv4_peakn  s    z!SingleDetTriggers.f_seobnrv4_peakc             C   s
   |  dS )Nr   )r   )r.   r   r   r   r   s  s    zSingleDetTriggers.end_timec             C   s
   |  dS )Ntemplate_duration)r   )r.   r   r   r   r   w  s    z#SingleDetTriggers.template_durationc             C   s
   |  dS )Nsnr)r   )r.   r   r   r   r   {  s    zSingleDetTriggers.snrc             C   s
   |  dS )Nsg_chisq)r   )r.   r   r   r   sgchisq  s    zSingleDetTriggers.sgchisqc             C   s
   |  dS )Nu_vals)r   )r.   r   r   r   r     s    zSingleDetTriggers.u_valsc             C   s   |  d|  dd d  S )Nchisq	chisq_dof   )r   )r.   r   r   r   rchisq  s    zSingleDetTriggers.rchisqc             C   s
   |  dS )Npsd_var_val)r   )r.   r   r   r   r     s    zSingleDetTriggers.psd_var_valc             C   s   t | j| jS )N)r   newsnrr   r   )r.   r   r   r   r     s    zSingleDetTriggers.newsnrc             C   s   t | j| j| jS )N)r   newsnr_sgvetor   r   r   )r.   r   r   r   r     s    zSingleDetTriggers.newsnr_sgvetoc             C   s   t | j| j| j| jS )N)r   newsnr_sgveto_psdvarr   r   r   r   )r.   r   r   r   r     s    z&SingleDetTriggers.newsnr_sgveto_psdvarc             C   s   t | j| j| j| jS )N)r   newsnr_sgveto_psdvar_thresholdr   r   r   r   )r.   r   r   r   r     s    z0SingleDetTriggers.newsnr_sgveto_psdvar_thresholdc             K   s   t j| |f|S )N)r   Zget_sngls_ranking_from_trigs)r.   Z	rank_namekwargsr   r   r   get_ranking  s    zSingleDetTriggers.get_rankingc             C   s   d}| j d k	rJt| j ts:t| j  d t| j | k rJ| j| | j  S | j d k	rl| j| d d  | j  S | j| d d  S d S )Ng333333?r   )r   r   r|   r(   r   r   )r.   cnameZMFRACr   r   r   r     s     
zSingleDetTriggers.get_column)N)r   r   ),r8   r9   r:   r;   rF   r   r   r   classmethodr   r   r   r   ra   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r     sP   
:
 
&	r   c               @   sz   e Zd ZdddZedd Zedd	 Zed
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )ForegroundTriggersN
foregroundTc       
         s  t ||d _d jjjkr4 jjjd d _n
td|i  _|d k	rrx$|D ]}t |}|j}	| j|	< qRW t	 fdd jD st
d| t
d j td	t j t jkstd
 t|d _| _| _d  _d  _d  _   d S )N)r{   rs   ru   z&File doesn't have an 'ifos' attribute!c                s   g | ]}| j  kqS r   )
sngl_filesrI   )r   rt   )r.   r   r   r     s    z/ForegroundTriggers.__init__.<locals>.<listcomp>zsngl_files: {}zself.ifos: {}zJIFOs in statmap file not all represented by single-detector trigger files.zMWARNING: Single-detector trigger files given for IFOs not in the statmap filer   )rw   
coinc_filery   rX   rv   rs   r   r   rz   allprintr   r=   sortedrI   rL   warningr   r   r   
_inclusive	_sort_arr_template_id	_trig_idsget_active_segments)
r.   r   r   r   r   r{   Zhas_incZ	sngl_fileZcurr_datZcurr_ifor   )r.   r   rF     s0    

zForegroundTriggers.__init__c             C   s   | j d kr| jrTy| jd}W q` tk
rP   td | jd}d| _Y q`X n| jd}| d d d }| jr|d | j }|| _ | j S )NifarzCWARNING: Can't find inclusive IFAR!Using exclusive IFAR instead ...ifar_excFr   )	r   r   r   r   rK   rL   r   r   r   )r.   r   Zsortingr   r   r   sort_arr  s    

zForegroundTriggers.sort_arrc             C   s&   | j d kr | d}|t| _ | j S )Nra   )r   get_coincfile_arrayr,   r'   )r.   ra   r   r   r   ra     s    

zForegroundTriggers.template_idc             C   s@   | j d k	r| j S i | _ x"| jD ]}| |d | j |< qW | j S )Nz/trigger_id)r   rs   r   )r.   rt   r   r   r   trig_id  s    
zForegroundTriggers.trig_idc             C   s   | j || j S )N)r   r   r   )r.   variabler   r   r   r     s    z&ForegroundTriggers.get_coincfile_arrayc             C   sL   y| j | d d  | j S  tk
rF   t| jdkr@tg S  Y nX d S )Nr   )r   ra   
IndexErrorr(   r!   r&   )r.   r   r   r   r   get_bankfile_array  s    
z%ForegroundTriggers.get_bankfile_arrayc       	   	   C   s   i }x| j D ]}y| j| }|dk}t|}d||< t|dk r~g }| j| j| }x|D ]}|||  q\W t|}n| j| 	|| }W n< t
k
r   t| j| dkrtg }|dk}n Y nX |t|f||< qW |S )Nr   r   i  )rs   r   r!   copyr(   r   r{   r+   r&   r   r   logical_not)	r.   r   Zreturn_dictrt   tidZlgccurrZhdf_datasetrP   r   r   r   get_snglfile_array_dict  s*    




z*ForegroundTriggers.get_snglfile_array_dictc             C   sL   i | _ x@| jD ]6}| j| d}| j| d}t||| j |< qW d S )Nzsearch/start_timezsearch/end_time)active_segmentsrs   r   r   r   start_end_to_segments)r.   rt   ZstartsZendsr   r   r   r   (  s    z&ForegroundTriggers.get_active_segmentsc                s0    fdd j D }tdd t| D }|S )Nc             3   s   | ]}  d |V  qdS )z{}/timeN)r   r   )r   rt   )r.   r   r   r$   1  s   z2ForegroundTriggers.get_end_time.<locals>.<genexpr>c             S   s   g | ]}t |d  qS )r   )r   )r   tr   r   r   r   3  s   z3ForegroundTriggers.get_end_time.<locals>.<listcomp>)rs   r!   r&   r*   )r.   Z	times_genZ	ref_timesr   )r.   r   get_end_time0  s
    


zForegroundTriggers.get_end_timec             C   sX   g }x4| j D ]*}t| |d dk d|}|| qW dd tt|jD }|S )z
        Returns
        -------
        ifos_list
             List of lists of ifo names involved in each foreground event.
             Ifos will be listed in the same order as self.ifos
        z/timer   -c             S   s   g | ]}t ||d k qS )r   )r|   )r   trigr   r   r   r   G  s   z/ForegroundTriggers.get_ifos.<locals>.<listcomp>)rs   r!   wherer   r+   iterr&   T)r.   Zifo_or_minusrt   Z	ifo_trigsZ	ifos_listr   r   r   get_ifos7  s    zForegroundTriggers.get_ifosc       1      C   s  t  }|t   t| j }tj|di |dt	j
dt	j t	jdj}ttj}| jj}y<|d d d d d   }|d d d d d   }W n tk
r$   g }	g }
xZ|d D ]N}|d	krq|d | }|	|d d d    |
|d d d    qW t|	}t|
}Y nX t| j}t||||}|| |jd
 | ttj}ttj}ttj}ttj}ttj}ttj }t!d
}x8|D ]0}t" }||_#||_$d
|_%||_|| qW t&d
}t' }d|_(d|_)||_*d
|_+|| ddddg}i }x|D ]}| ,|||< q.W ddddg}i }x4|D ],}|dkrx| - ||< n| .|||< q\W dddddddddddg}i } x|D ]}| /|| |< qW d
}!x~t0t| jD ]j}"t1|"}#g }$g }%d
}&g }'xh|D ]^}| d | d  |" s&q|'|g7 }'t2|!}(|!d 7 }!t3 })|(|)_4||)_5|&| d | d
 |" d! 7 }&xD|D ]<}| | | d
 |" }*|dkrt6|*|)_7nt8|)||* qvW x&|D ]}|| |" }*t8|)||* qW t9:|)j;|)j<\|)_=|)_>t9?|)j;|)j<\|)_@}+|)jAd" |)jB |)_C|$|)j@g7 }$|%|)j=g7 }%||) tD },d#|,_E|#|,_F|(|,_4||, qW tGH|$}-tGH|%}.tI }/tJ }0||/_*t|'|/_Kd$LtM|'|/_N|'|0_N||/_$||/_|#|/_F|#|0_F|-|0_@|.|0_Ot6|d |" |0_7|&d" |0_B|d |" |0_Pd%|d |"  |0_Q|0jQtR |0_Q|d |" |/_Sd&|0_T||/ ||0 qW |jd
 | |jd
 | |jd
 | |jd
 | |jd
 | |jd
 | tUV|| d S )'Npycbc zpycbc/)instrumentscommentr
   Zcvs_repositoryZcvs_entry_timerd   Zcoincrn   ro   Zforeground_vetor   Zinspiralz+sngl_inspiral<-->sngl_inspiral coincidencesr   r   r   r   r   r   fapr^   r   r   r   
bank_chisqbank_chisq_dof
cont_chisqcont_chisq_dofr   r   Z	coa_phasesigmasqr    r   g      ?Zsngl_inspiral,g      ?g        )Wr   ZDocumentZappendChildZLIGO_LWr|   r   rI   ligolw_processZregister_to_xmldocpycbc_versionZgit_hashZ
git_branchdateZ
process_idr   ZNewZSearchSummaryTabler   ry   minmaxrK   r+   r(   r   r   Z
childNodesZSnglInspiralTableZCoincDefTableZ
CoincTableZCoincInspiralTableZCoincMapTableZTimeSlideTableZTimeSlideIDZ	TimeSlideZ
instrumenttime_slide_idoffsetZ
CoincDefIDZCoincDefsearchdescriptioncoinc_def_idZsearch_coinc_typer   r   r   r   rangeZCoincIDZSnglInspiralIDr   event_idrt   r   ro   r@   r   Zmass1_mass2_to_mtotal_etar   r   r   r   r   r   r  r   Zeff_distanceZCoincMapZ
table_nameZcoinc_event_idr!   ZmeanZCoincZCoincInspiralZneventsjoinr   r   ZmassZfalse_alarm_rateZcombined_farr   Z
likelihoodZminimum_durationligolw_utilsZwrite_filename)1r.   	file_nameZoutdocrs   Zproc_idZsearch_summ_tableZcoinc_h5file
start_timer   Zstart_timesZ	end_timesZifo_combZ	seg_groupZ	num_trigsZsearch_summaryZsngl_inspiral_tableZcoinc_def_tableZcoinc_event_tableZcoinc_inspiral_tableZcoinc_event_map_tableZtime_slide_tabler  rt   Ztime_slide_rowr  Zcoinc_def_rowZbank_col_namesZbank_col_valsnameZcoinc_event_namesZcoinc_event_valsZsngl_col_namesZsngl_col_valsZsngl_event_countrP   Zcoinc_idZsngl_mchirpsZ
sngl_mtotsZ	net_snrsqZtriggered_ifosr  Zsnglvalr   Zcoinc_map_rowZsngl_combined_mchirpZsngl_combined_mtotZcoinc_event_rowZcoinc_inspiral_rowr   r   r   to_coinc_xml_objectK  s   
 





















z&ForegroundTriggers.to_coinc_xml_objectc          	      s  t |d}td   |jdtjd  jrR 	d}|jd|tjd  	d}|jd|tjd  jr 	d}|jd	|tjd  	d
}|jd|tjd x(dD ] } 	|}|j||tjd qW td x(dD ] } 
|}|j||tjd qW  
d}	 
d}
t|	|
\}}|jd|tjd td td  d} d}xh jD ]^}|| d }|| d }|| d }|d| d  }d|t|< |j|d |tjd qnW xdD ]}t| y |}W n$ tk
r   t|d  Y nX xP jD ]F}|| d }|| d }d|t|< |j|d | |tjd q W qW  dt jd  d }xd jD ]Z}| d }| d }d|t|< |j|d |tjd ||  || d 7  < qW t|}|jd |tjd td! fd"d# jD }d$d# t| D }|jd%|d&d td'  fd(d# jD }d)d# t| D }|jd*|d&d |  d S )+NrS   zOutputting search resultsr   )r#   r   r   r   Zifar_exclusiver  Zp_valuefap_excZp_value_exclusive)r^   zOutputting template information)r   r   r   r   r   r   Z
chirp_massz%Outputting single-trigger informationzreduced chisquaredr   r   r   r    g       @g      Z_chisq)r   r   r  r   z, is not present in the single-detector filesr   r   Z_snrnetwork_snrzTriggered detectorsc                s&   g | ]  fd d  d D qS )c                s   g | ]}|r d  ndqS )r   r   r   )r   r   )rt   r   r   r   J  s   zEForegroundTriggers.to_coinc_hdf_object.<locals>.<listcomp>.<listcomp>r    r   )r   )snr_vals_valid)rt   r   r   J  s   z:ForegroundTriggers.to_coinc_hdf_object.<locals>.<listcomp>c             S   s   g | ]}d  |dqS )r   ascii)r  encode)r   Z	triggeredr   r   r   r   N  s   r   z<S3zactive detectorsc                s    g | ]  fd dD qS )c                s&   g | ]}|j   kr d  ndqS )r   r   )r   )r   r   )rt   r.   r   r   r   V  s   zEForegroundTriggers.to_coinc_hdf_object.<locals>.<listcomp>.<listcomp>r   )r   )r.   r   )rt   r   r   V  s   c             S   s   g | ]}d  |dqS )r   r  )r  r  )r   Zactive_at_timer   r   r   r   Z  s   Zobs)h5pyFilerL   rM   r   rY   r!   Zfloat32r   r   r   r   r   rs   r   float64rK   Z
zeros_likesqrtr*   r?   )r.   r  Zofdr   r   r  r  fieldr   r   r   r   r   Zchisq_vals_validZchisq_dof_vals_validrt   Z
chisq_valsZchisq_validZchisq_dof_valsr   Z
vals_validvalidZnetwork_snr_sqr  Ztriggered_matrixZtriggered_detectorsZactive_matrixZactive_detectorsr   )r.   r  r   r   to_coinc_hdf_object  s    




























z&ForegroundTriggers.to_coinc_hdf_object)NNr   T)r8   r9   r:   rF   r   r   ra   r   r   r   r   r   r   r   r  r&  r   r   r   r   r     s    
 
 *r   c               @   s6   e Zd Zdddi fddZdd Zdd Zdd	 ZdS )
ReadByTemplateNc             C   s  || _ t|d| _t| j d | _d | _|r>t|dni | _d| j }| j|d  d d  | j|d  d d   }}t	
|| | _|d krg }|d krg }x8t||D ]*\}	}
t	j|	| j|
d}| j|  | _qW | j|kr|| j d}t|d }t|d }|dks(|dk r0td	|dkrD|dkst| j| jd
  d d  }| jd | jkr| j| jd  d d  }ng }t||f}t	
|| ||  }| j|  | _t	| j| _d S )Nr   r   z
%s/search/r  r   )rt   r   r  r    zQGating veto window values must be negative before gates and positive after gates.z/gating/auto/timez/gating/filez/gating/file/time)filenamer   r!  filer-   rI   rt   r%  r   r   r   ZcoalesceZsegsr*   Zselect_segments_by_definerrv   floatr   r!   uniquer"   Zsegments_to_start_end)r.   r(  r   r   Z
veto_filesZgating_veto_windowsrp   seZvfiler  Z	veto_segsZgating_vetoZgveto_beforeZgveto_afterZautogate_timesZdetgate_timesZ
gate_timesZgating_veto_segsr   r   r   rF   d  sD    
.
zReadByTemplate.__init__c             C   s0   | j d| j|f  | }| j d| j|f  | S )a<  Get a column of data for template with id 'num'.

        Parameters
        ----------
        col: str
            Name of column to read
        num: int
            The template id to read triggers for

        Returns
        -------
        data: numpy.ndarray
            The requested column of data
        z%s/%s_templatez%s/%s)r)  rt   )r.   r   numrefr   r   r   get_data  s    zReadByTemplate.get_datac             C   s   || _ | d|}| jr6t|| jd | jd | _ntdt|| _| j	i kri | _
d| j	jkrxR| j	jd D ]}| j	| | j  | j
|< qrW n&x$| j	D ]}| j	| | j  | j
|< qW | j| jd| j  |  }|S )a  Set the active template to read from.

        Parameters
        ----------
        num: int
            The template id to read triggers for.

        Returns
        -------
        trigger_id: numpy.ndarray
            The indices of this templates triggers.
        r   r   r    
parametersz%s/template_boundaries)template_numr0  r%  r   Zindices_within_timesr4   r!   Zaranger(   r   r   rX   r)  rt   )r.   r.  r   r   Z
trigger_idr   r   r   set_template  s     
zReadByTemplate.set_templatec             C   s8   | j dkrtd| || j }| jr0|| j n|}|S )a   Return the column of data for current active template after
        applying vetoes

        Parameters
        ----------
        col: str
            Name of column to read

        Returns
        -------
        data: numpy.ndarray
            The requested column of data
        NzGYou must call set_template to first pick the template to read data from)r2  r   r0  r%  r4   )r.   r   r#   r   r   r   r     s
    
zReadByTemplate.__getitem__)r8   r9   r:   rF   r0  r3  r   r   r   r   r   r'  b  s
   *)r'  traditionalcontr   max_cont_tradsgmax_bank_contmax_bank_tradmax_bank_cont_tradc             C   sZ  | }|dkr<|d d d  }|d d d  }||d d  }|dkrl|d d d  }|d d d  }|| }|dkr|d	 d d  }|d
 d d  }|| }|dkr|d d d  }	n|dkr|}	n|dkr|}	n|dkr|}	nv|dkrt ||}	n`|dkrt ||}	nH|dkr&t ||}	n0|dkrFt t |||}	nd| }
t|
|	S )N)r4  r6  r9  r:  r   r   r   )r5  r6  r8  r:  r  r  )r   r8  r9  r:  r  r  r7  r   r4  r5  r   r6  r8  r9  r:  z"Do not recognize --chisq-choice %s)r!   maximumr   )ZhdfileZchisq_choicerC   Z
trad_chisqZtrad_chisq_dofr  r  r  r  r   r   r   r   r   get_chisq_from_file_choice  s@    


r<  c          	   C   s(   t |d}t|d|  W dQ R X dS )z
    Parameters
    ----------
    dic:
        python dictionary to be converted to hdf5 format
    filename:
        desired name of hdf5 file
    rS   /N)r   r!  'recursively_save_dict_contents_to_group)dicr(  ry   r   r   r   save_dict_to_hdf5  s    	r@  c             C   s~   xx|  D ]l\}}t|tjtjtjtttt	t
tf	rF|| |t| < q
t|trft| || d | q
tdt| q
W dS )z
    Parameters
    ----------
    h5file:
        h5py file to be written to
    path:
        path within h5py file to saved dictionary
    dic:
        python dictionary to be converted to hdf5 format
    r=  zCannot save %s typeN)itemsr   r!   ZndarrayZint64r"  strr'   r*  bytesr-   r|   dictr>  r   r   )ry   pathr?  rp   itemr   r   r   r>    s    
r>  c             C   sp   i }xf| |   D ]V\}}t|tjr4|d ||< qt|tjrXt| || d ||< qtdt| qW |S )z
    Parameters
    ----------
    h5file:
        h5py file to be loaded as a dictionary
    path:
        path within h5py file to load: '/' for the whole h5py file

    Returns
    -------
    dic:
        dictionary with hdf5 file group content
    r   r=  zCannot load %s type)rA  r   r   DatasetGroupload_hdf5_to_dictr   r   )ry   rE  r?  rp   rF  r   r   r   rI  +  s    rI  c                s2   t |ttfstt fdd|D |  < dS )z@ Combine the same column from multiple files and save to a thirdc                s4   g | ],} |kr |  d d  nt jg t jdqS )N)r   )r!   r&   Zuint32)r   fi)r{   r   r   r   G  s   z$combine_and_copy.<locals>.<listcomp>N)r   r|   r-   AssertionErrorr!   r"   )rC   rA   r{   r   )r{   r   combine_and_copyC  s    rL  c             C   s:   t | ttfstg }x| D ]}|t|d7 }qW t|S )Nr=  )r   r|   r-   rK  get_all_subkeysset)rA   ZdatasetsrJ  r   r   r   name_all_datasetsJ  s
    
rO  c             C   sp   g }|}|dkr| }n| | }xL|  D ]@}|d | }t| | tjrZ||d q(|t| |7 }q(W |S )Nr   r=  )rI   r   r   rG  r+   lstriprM  )grprp   Zsubkey_listZsubkey_startZgrpkskrE  r   r   r   rM  Q  s    rM  statec             C   s*   t  }tj| ||d t||||d dS )a(  Dumps the given state to an hdf5 file handler.

    The state is stored as a raw binary array to ``{path}/{dsetname}`` in the
    given hdf5 file handler. If a dataset with the same name and path is
    already in the file, the dataset will be resized and overwritten with the
    new state data.

    Parameters
    ----------
    state : any picklable object
        The sampler state to dump to file. Can be the object returned by
        any of the samplers' `.state` attribute (a dictionary of dictionaries),
        or any picklable object.
    fp : h5py.File
        An open hdf5 file handler. Must have write capability enabled.
    path : str, optional
        The path (group name) to store the state dataset to. Default (None)
        will result in the array being stored to the top level.
    dsetname : str, optional
        The name of the dataset to store the binary array to. Default is
        ``state``.
    protocol : int, optional
        The protocol version to use for pickling. See the :py:mod:`pickle`
        module for more details.
    )protocol)rE  dsetnameN)r   pickledumpdump_pickle_to_hdf)rS  fprE  rU  rT  memfpr   r   r   
dump_statej  s    r[  c             C   s   |  d tj|  dd}|dk	r0|d | }||krP|j||jd|jd n&|j|| jd krv|| |jf ||| dd< dS )a  Dumps pickled data to an hdf5 file object.

    Parameters
    ----------
    memfp : file object
        Bytes stream of pickled data.
    fp : h5py.File
        An open hdf5 file handler. Must have write capability enabled.
    path : str, optional
        The path (group name) to store the state dataset to. Default (None)
        will result in the array being stored to the top level.
    dsetname : str, optional
        The name of the dataset to store the binary array to. Default is
        ``state``.
    r   ZS1)r   Nr=  )N)shapeZmaxshaper   )	seekr!   Z
frombufferreadrY   r\  r   r2   resize)rZ  rY  rE  rU  bdatar   r   r   rX    s    
rX  c             C   s.   |dk	r| | } | | d   }tt|S )a  Loads a sampler state from the given hdf5 file object.

    The sampler state is expected to be stored as a raw bytes array which can
    be loaded by pickle.

    Parameters
    ----------
    fp : h5py.File
        An open hdf5 file handler.
    path : str, optional
        The path (group name) that the state data is stored to. Default (None)
        is to read from the top level.
    dsetname : str, optional
        The name of the dataset that the state data is stored to. Default is
        ``state``.
    Nr   )tobytesrV  loadr   )rY  rE  rU  r`  r   r   r   
load_state  s    rc  )r   r<   r\   rr   rw   r   r   r   r'  chisq_choicesr<  r@  r>  rI  rL  rO  rM  r[  rX  rc  )NrS  )NrS  )<r   numpyr!   rL   r   rV  	itertoolsr   ior   Zlalr   r   Zligo.lwr   r   r   r  Zligo.lw.utilsr	   r  r   r
   r	  Zpycbc.io.ligolwr   r   r   r   r   ri   r   r   r   Zpycbc.pnutilsr   r!  r   objectr<   r\   rr   rw   r   r   r   r'  rd  r<  r@  r>  rI  rL  rO  rM  HIGHEST_PROTOCOLr[  rX  rc  __all__r   r   r   r   <module>   sZ   N\(%L$  7   -~

(

