B
    d                 @   s  d Z ddlZddlZddlZddlZddlmZ ddlmZ	 ddl
mZ ddlZddlmZ ddlmZ ddlZdd	lmZ d
ZdZdZdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Z d#d$ Z!G d%d& d&e"Z#d'd( Z$d)d* Z%d+d, Z&d-d. Z'd/d0 Z(G d1d2 d2ej)Z*G d3d4 d4ej+Z,G d5d6 d6ej)ej-Z.G d7d8 d8ej+ej/Z0G d9d: d:ej)ej-Z1G d;d< d<ej/Z2G d=d> d>ej3Z3G d?d@ d@ej)Z4G dAdB dBej+Z5G dCdD dDej)Z6G dEdF dFej+Z7G dGdH dHej)Z8G dIdJ dJej+Z9G dKdL dLej)Z:G dMdN dNej)Z;G dOdP dPej+Z<G dQdR dRej)Z=G dSdT dTej+Z>G dUdV dVej)Z?G dWdX dXej+Z@daAdaBdaCdaDdaEdaFdaGdaHdaIdaJdaKddZd[ZLd\d] ZMd^d_ ZNd`da ZOdbdc ZPddde ZQdfZRdgdh ZSddjdkZTi fdldmZUddndoZVddpdqZWddrdsZXddtduZYddvdwZZddxdyZ[dzd{ Z\d|d} Z]dd~dZ^e_ difddZ`e_ difddZadddZbdS )z+
Excess power pipeline construction tools.
    N)segments)utils)pipeline)	iterutils)
CacheEntry   )cafezSDuncan Brown <duncan@gravity.phys.uwm.edu>, Kipp Cannon <kipp@gravity.phys.uwm.edu>z$Date$z
$Revision$c             C   s   |  ddS )NcondorZuniverse)get)config_parser r   [/work/yifan.wang/ringdown/master-ringdown-env/lib/python3.7/site-packages/lalburst/power.pyget_universe=   s    r   c             C   s   |  ddS )Nr	   accounting_group)r
   )r   r   r   r   get_accounting_groupA   s    r   c             C   s   |  d|S )Nr	   )r
   )r   namer   r   r   get_executableE   s    r   c             C   s   |  ddS )Nr   Zout_dir)r
   )r   r   r   r   get_out_dirI   s    r   c             C   s   |  ddS )Nr   	cache_dir)r
   )r   r   r   r   get_cache_dirM   s    r   c             C   s   |  ddS )Nr   Ztriggers_dir)r
   )r   r   r   r   get_triggers_dirQ   s    r   c          
   C   sF   yt |  W n2 tk
r@ } z|jtjkr0|W d d }~X Y nX d S )N)osmkdirOSErrorerrnoEEXIST)direr   r   r   make_dir_if_not_existsU   s
    r   c             C   s   t t|  t t|  d S )N)r   r   r   )r   r   r   r   make_dag_directories^   s    r   c             C   s   |  ddS )Nr   files_per_bucluster)getint)r   r   r   r   get_files_per_buclusterc   s    r"   c             C   s   |  ddS )Nr   files_per_bucut)r!   )r   r   r   r   get_files_per_bucutg   s    r$   c             C   s   |  ddS )Nr   files_per_burca)r!   )r   r   r   r   get_files_per_burcak   s    r&   c             C   s   |  ddS )Nr   files_per_binjfind)r!   )r   r   r   r   get_files_per_binjfindo   s    r(   c               @   s   e Zd ZdZdd ZdS )TimingParametersz,
	A class to hold timing parameter values.
	c          	   C   s   | dd| _|dd| _t| dd| j | _| dd| _|dd| _| dd| _t	
| j| j| j|dd\| _| _| _| _| _d S )	Nlalapps_powerzresample-ratezwindow-lengthzmax-tile-durationztile-stride-fractionzfilter-corruptionzmax-tile-bandwidthzpsd-average-points)getfloatresample_rater!   Zwindow_lengthintZmax_tile_lengthZtile_stride_fractionfilter_corruptionZmax_tile_bandwidthlalburstZEPGetTimingParameters
psd_length	psd_shiftZwindow_shiftZ
window_padZtiling_length)selfr   r   r   r   __init__w   s    zTimingParameters.__init__N)__name__
__module____qualname____doc__r3   r   r   r   r   r)   s   s   r)   c             C   s   t  }x| D ]}||jO }qW | }d |kr:|d  |  x| D ]}|| sL||= qLW |rxdtj| }nd}td	|pd ||
 |S )Nzfile://localhost%szfile://localhost/dev/null+)r   Zsegmentlistdictkeysremovesortr   pathabspathr   joinZ
extent_all)input_cachedescriptionr<   seglistscZinstruments
instrumenturlr   r   r   make_cache_entry   s    


rE   c             C   s"   dd | D }|j dd d |S )Nc             S   s"   g | ]}|  D ]}||fqqS r   )get_output_cache).0parentcache_entryr   r   r   
<listcomp>   s    z)collect_output_caches.<locals>.<listcomp>c             S   s
   | d j S )Nr   )segment)xr   r   r   <lambda>       z'collect_output_caches.<locals>.<lambda>)key)r;   )parentscacher   r   r   collect_output_caches   s    rR   c       	      C   s   t | } i }x&| D ]}x| D ]}|||< q W qW dd |D }x2t||D ]$\}}x|D ]}|||  q\W qNW t| tt j|  }||fS )z
	For each cache, get the set of nodes whose output files it
	contains.  A node is allowed to provide more than one output file,
	and thus can be listed in more than one set.
	c             S   s   g | ]
}t  qS r   )set)rG   rQ   r   r   r   rJ      s    z)match_nodes_to_caches.<locals>.<listcomp>)rS   rF   zipaddlenunion)	nodescachesindexnoderI   Znode_groupsZ
node_grouprQ   unusedr   r   r   match_nodes_to_caches   s    

r]   c             C   s0   t dd | D }tdd | D }t||S )Nc             S   s   g | ]}|j d  qS )r   )rK   )rG   rI   r   r   r   rJ      s    zcache_span.<locals>.<listcomp>c             S   s   g | ]}|j d  qS )r   )rK   )rG   rI   r   r   r   rJ      s    )minmaxr   rK   )rQ   abr   r   r   
cache_span   s    rb   c             C   s4   t |d}x$t| D ]\}}tt||d qW d S )Nw)file)rd   rR   printstr)rX   filenamefrI   r[   r   r   r   write_output_cache   s    
ri   c               @   s   e Zd Zdd ZdS )RMJobc             C   sv   t j| dd | tjt|d | tjt|d | 	dd | 	dt
| | dd	 | d
 dS )z)
		config_parser = ConfigParser object
		localz/bin/rmzrm-$(cluster)-$(process).outzrm-$(cluster)-$(process).errgetenvTruer   force zrm.subN)r   CondorDAGJobr3   set_stdout_filer   r<   r>   r   set_stderr_fileadd_condor_cmdr   Zadd_optset_sub_file)r2   r   r   r   r   r3      s    zRMJob.__init__N)r4   r5   r6   r3   r   r   r   r   rj      s   rj   c               @   s$   e Zd Zdd Zdd Zdd ZdS )RMNodec             C   s0   t j| | t | _t | _t | jd< d S )N
initialdir)	r   CondorDAGNoder3   rS   r?   output_cacher   getcwd_CondorDAGNode__macros)r2   jobr   r   r   r3      s    zRMNode.__init__c             C   s0   |  j |O  _ x|D ]}tj| |j qW d S )N)r?   r   rw   add_file_argr<   )r2   rQ   rI   r   r   r   add_input_cache   s    
zRMNode.add_input_cachec             C   s   t  S )N)rS   )r2   r   r   r   rF      s    zRMNode.get_output_cacheN)r4   r5   r6   r3   r}   rF   r   r   r   r   ru      s   ru   c               @   s   e Zd ZdZdd ZdS )BurstInjJoba?  
	A lalapps_binj job used by the power pipeline. The static options
	are read from the [lalapps_binj] section in the ini file. The
	stdout and stderr from the job are directed to the logs directory.
	The job runs in the universe specified in the ini file.  The path
	to the executable is determined from the ini file.
	c             C   s   t j| t|t|d t j| | |ddrD|dd| _nd| _| 	|d | 
tjt|d | tjt|d | dd | d	t| | d
 d| _|dd| _dS )z)
		config_parser = ConfigParser object
		Zlalapps_binjr   injection_bandsNzNlalapps_binj-$(macrogpsstarttime)-$(macrogpsendtime)-$(cluster)-$(process).outzNlalapps_binj-$(macrogpsstarttime)-$(macrogpsendtime)-$(cluster)-$(process).errrl   rm   r   zlalapps_binj.sub.z	time-step)r   rp   r3   r   r   AnalysisJob
has_optionr!   r   add_ini_optsrq   r   r<   r>   r   rr   rs   r   rt   
output_dirr+   	time_step)r2   r   r   r   r   r3     s    
zBurstInjJob.__init__N)r4   r5   r6   r7   r3   r   r   r   r   r~     s   r~   c               @   sl   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd ZdS )BurstInjNodec             C   sR   t j| | t j|  d | _g | _tjt	 | 
 j| _t	 | jd< d S )Nrv   )r   rw   r3   AnalysisNode_BurstInjNode__usertagrx   r   r<   r>   ry   r{   r   rz   )r2   r{   r   r   r   r3   '  s    zBurstInjNode.__init__c             C   s   || _ | d| j  d S )Nzuser-tag)r   add_var_opt)r2   tagr   r   r   set_user_tag/  s    zBurstInjNode.set_user_tagc             C   s   | j rtd| jS )Nz5cannot change attributes after computing output cache)rx   AttributeErrorr   )r2   r   r   r   get_user_tag3  s    zBurstInjNode.get_user_tagc             C   s   |  d| d S )Nztime-slide-file)r   )r2   rg   r   r   r   set_time_slide_file8  s    z BurstInjNode.set_time_slide_filec             C   s   |   dd S )NZmacrotimeslidefile)get_optsr
   )r2   r   r   r   get_time_slide_file;  s    z BurstInjNode.get_time_slide_filec             C   s   | j rtd| d| d S )Nz5cannot change attributes after computing output cachezgps-start-time)rx   r   r   )r2   startr   r   r   	set_start>  s    zBurstInjNode.set_startc             C   s   |   dd S )NZmacrogpsstarttime)r   r
   )r2   r   r   r   	get_startC  s    zBurstInjNode.get_startc             C   s   | j rtd| d| d S )Nz5cannot change attributes after computing output cachezgps-end-time)rx   r   r   )r2   endr   r   r   set_endF  s    zBurstInjNode.set_endc             C   s   |   dd S )NZmacrogpsendtime)r   r
   )r2   r   r   r   get_endK  s    zBurstInjNode.get_endc          
   C   sL   | j sFtd| jtt|  t|  dt	j
|   g| _ | j S )z
		Returns a LAL cache of the output file name.  Calling this
		method also induces the output name to get set, so it must
		be at least once.
		zG1+H1+H2+L1+T1+V1zfile://localhost)rx   r   r   r   rK   lalLIGOTimeGPSr   r   r   r<   r=   
get_output)r2   r   r   r   rF   N  s    @zBurstInjNode.get_output_cachec             C   s   t d S )N)NotImplementedError)r2   r   r   r   get_output_filesY  s    zBurstInjNode.get_output_filesc             C   s   | j d krd |  |  | jfkr*tdtt|  t|  }| 	t
j| jd| jt|  t|  |   f  | j S )Nz7start time, end time, ifo, or user tag has not been setz,G1+H1+H2+L1+T1+V1-INJECTIONS_%s-%d-%d.xml.gz)_AnalysisNode__outputr   r   r   
ValueErrorr   rK   r   r   
set_outputr   r<   r>   r   r-   )r2   segr   r   r   r   \  s    
 :zBurstInjNode.get_outputN)r4   r5   r6   r3   r   r   r   r   r   r   r   r   rF   r   r   r   r   r   r   r   &  s   r   c               @   s   e Zd ZdZdd ZdS )PowerJoba^  
	A lalapps_power job used by the power pipeline. The static options
	are read from the [lalapps_power] and [lalapps_power_<inst>]
	sections in the ini file. The stdout and stderr from the job are
	directed to the logs directory.  The job runs in the universe
	specified in the ini file. The path to the executable is determined
	from the ini file.
	c             C   s   t j| t|t|d t j| | | |d | tj	
t|d | tj	
t|d | dd | dt| | d d| _d	S )
z)
		config_parser = ConfigParser object
		r*   z'lalapps_power-$(cluster)-$(process).outz'lalapps_power-$(cluster)-$(process).errrl   rm   r   zlalapps_power.subr   N)r   rp   r3   r   r   r   r   rq   r   r<   r>   r   rr   rs   r   rt   r   )r2   r   r   r   r   r3   n  s    
zPowerJob.__init__N)r4   r5   r6   r7   r3   r   r   r   r   r   e  s   r   c               @   sT   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd ZdS )	PowerNodec             C   sR   t j| | t j|  d | _g | _tjt	 | 
 j| _t	 | jd< d S )Nrv   )r   rw   r3   r   _PowerNode__usertagrx   r   r<   r>   ry   r{   r   rz   )r2   r{   r   r   r   r3     s    zPowerNode.__init__c             C   sL   | j rtdtj| | x*|  jd| D ]}| d|  q2W dS )zS
		Load additional options from the per-instrument section in
		the config file.
		z5cannot change attributes after computing output cachezlalapps_power_%sz--%s %sN)	rx   r   r   r   set_ifor{   Z_AnalysisJob__cpitemsadd_var_arg)r2   rC   Zoptvaluer   r   r   r     s
    zPowerNode.set_ifoc             C   s&   | j rtd|| _| d| j d S )Nz5cannot change attributes after computing output cachezuser-tag)rx   r   r   r   )r2   r   r   r   r   r     s    zPowerNode.set_user_tagc             C   s   | j S )N)r   )r2   r   r   r   r     s    zPowerNode.get_user_tagc          
   C   sP   | j sJt|  | jtt|  t| 	 dt
j|   g| _ | j S )z
		Returns a LAL cache of the output file name.  Calling this
		method also induces the output name to get set, so it must
		be at least once.
		zfile://localhost)rx   r   get_ifor   r   rK   r   r   r   r   r   r<   r=   r   )r2   r   r   r   rF     s    DzPowerNode.get_output_cachec             C   s   t d S )N)r   )r2   r   r   r   r     s    zPowerNode.get_output_filesc             C   s   | j d krd |  |  |  | jfkr0tdtt	|  t	|  }| 
tj| jd|  | jt|  t|  t|   f  | j S )Nz7start time, end time, ifo, or user tag has not been setz%s-POWER_%s-%d-%d.xml.gz)r   r   r   r   r   r   r   rK   r   r   r   r   r<   r>   r   r-   )r2   r   r   r   r   r     s    
 DzPowerNode.get_outputc             C   s   |  d| | | dS )z
		Set the LAL frame cache to to use. The frame cache is
		passed to the job with the --frame-cache argument.  @param
		file: calibration file to use.
		z	mdc-cacheN)r   add_input_file)r2   rd   r   r   r   set_mdccache  s    zPowerNode.set_mdccachec             C   s   |  d| | | dS )zV
		Set the name of the XML file from which to read a list of
		software injections.
		zinjection-fileN)r   r   )r2   rd   r   r   r   set_injection_file  s    zPowerNode.set_injection_fileN)r4   r5   r6   r3   r   r   r   rF   r   r   r   r   r   r   r   r   r   ~  s   
	r   c               @   sh   e Zd Zdd ZdddZdd Zdd	 Zdd
dZdd Zdd Z	dd Z
dd Zdd Zdd ZdS )LigolwAddNodec             G   sz   t jj| |f|  g | _g | _tjt | 	 j
| _
tjt d| _t | jd< t|| _| jrv| d d S )Nr   rv   z--remove-input)r   r   r3   r?   rx   r   r<   r>   ry   r{   r   r   rz   boolremove_inputr   )r2   r{   r   argsr   r   r   r3     s    
zLigolwAddNode.__init__Nc             C   sH   | j d d = t| jd | j}|d k	r*||_|d k	r8||_| j |}d S )N)rx   rE   r?   r   observatoryrK   append)r2   r   rK   rI   r   r   r   Z__update_output_cache  s    z#LigolwAddNode.__update_output_cachec             G   s>   t jj| f|  tj| jd|   | _| 	d| j d S )Nz%s.cachezinput-cache)
r   r   set_namer   r<   r>   r   get_name
cache_namer   )r2   r   r   r   r   r     s    zLigolwAddNode.set_namec             C   s   | j | |   d S )N)r?   extend#_LigolwAddNode__update_output_cache)r2   rQ   r   r   r   r}     s    zLigolwAddNode.add_input_cachec             C   s    t j| | | j||d d S )N)r   rK   )r   r   r   r   )r2   r<   r   rK   r   r   r   r     s    zLigolwAddNode.set_outputc             C   s(   | j r$x|D ]}| d|j  qW d S )Nz--remove-input-except %s)r   r   r<   )r2   rQ   rB   r   r   r   add_preserve_cache  s    
z LigolwAddNode.add_preserve_cachec             C   s   | j S )N)r?   )r2   r   r   r   get_input_cache  s    zLigolwAddNode.get_input_cachec             C   s   | j S )N)rx   )r2   r   r   r   rF     s    zLigolwAddNode.get_output_cachec             G   sB   t | jd}x| jD ]}tt||d qW tjj| f|  d S )Nrc   )rd   )rd   r   r?   re   rf   r   r   write_input_files)r2   r   rh   rB   r   r   r   r     s    zLigolwAddNode.write_input_filesc             C   s   t d S )N)r   )r2   r   r   r   r     s    zLigolwAddNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r     s    zLigolwAddNode.get_output)NN)NNN)r4   r5   r6   r3   r   r   r}   r   r   r   rF   r   r   r   r   r   r   r   r     s   
	
r   c               @   s   e Zd Zdd ZdS )BucutJobc             C   s   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | d	d
 | |d t|| _| jdk rtdd S )NvanillaZlalapps_bucutzlalapps_bucut.subz'lalapps_bucut-$(cluster)-$(process).outz'lalapps_bucut-$(cluster)-$(process).errrl   rm   r   RequirementszMemory > 1100r   zfiles_per_bucut < 1)r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   r$   r#   r   )r2   r   r   r   r   r3     s    


zBucutJob.__init__N)r4   r5   r6   r3   r   r   r   r   r     s   r   c               @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )	BucutNodec             G   s2   t jj| f|  g | _| j| _t | jd< d S )Nrv   )r   rw   r3   r?   rx   r   ry   rz   )r2   r   r   r   r   r3     s    zBucutNode.__init__c             C   s<   | j | x*|D ]"}|j}tj| | | | qW d S )N)r?   r   r<   r   rw   r|   add_output_file)r2   rQ   rB   rg   r   r   r   r}     s
    
zBucutNode.add_input_cachec             C   s   t d S )N)r   )r2   rg   r   r   r   r|     s    zBucutNode.add_file_argc             C   s   | j S )N)r?   )r2   r   r   r   r     s    zBucutNode.get_input_cachec             C   s   | j S )N)rx   )r2   r   r   r   rF      s    zBucutNode.get_output_cachec             C   s   t d S )N)r   )r2   r   r   r   r   #  s    zBucutNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r   &  s    zBucutNode.get_outputN)
r4   r5   r6   r3   r}   r|   r   rF   r   r   r   r   r   r   r     s   r   c               @   s   e Zd Zdd ZdS )BuclusterJobc             C   s   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | d	d
 | |d t|| _t|| _| jdk rtdd S )Nr   Zlalapps_buclusterzlalapps_bucluster.subz+lalapps_bucluster-$(cluster)-$(process).outz+lalapps_bucluster-$(cluster)-$(process).errrl   rm   r   r   zMemory > 1100r   zfiles_per_bucluster < 1)r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   r   r   r"   r    r   )r2   r   r   r   r   r3   +  s    



zBuclusterJob.__init__N)r4   r5   r6   r3   r   r   r   r   r   *  s   r   c               @   sT   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd ZdS )BuclusterNodec             G   sL   t jj| f|  g | _| j| _tjt | 	 j
| _
t | jd< d S )Nrv   )r   rw   r3   r?   rx   r   r<   r>   ry   r{   r   rz   )r2   r   r   r   r   r3   =  s
    zBuclusterNode.__init__c             G   s>   t jj| f|  tj| jd|   | _| 	d| j d S )Nz%s.cachezinput-cache)
r   rw   r   r   r<   r>   r   r   r   r   )r2   r   r   r   r   r   D  s    zBuclusterNode.set_namec             C   s   | j | d S )N)r?   r   )r2   rQ   r   r   r   r}   I  s    zBuclusterNode.add_input_cachec             C   s   t d S )N)r   )r2   rg   r   r   r   r|   L  s    zBuclusterNode.add_file_argc             G   sB   t | jd}x| jD ]}tt||d qW tjj| f|  d S )Nrc   )rd   )rd   r   r?   re   rf   r   rw   r   )r2   r   rh   rB   r   r   r   r   O  s    zBuclusterNode.write_input_filesc             C   s   | j S )N)r?   )r2   r   r   r   r   U  s    zBuclusterNode.get_input_cachec             C   s   | j S )N)rx   )r2   r   r   r   rF   X  s    zBuclusterNode.get_output_cachec             C   s   t d S )N)r   )r2   r   r   r   r   [  s    zBuclusterNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r   ^  s    zBuclusterNode.get_outputN)r4   r5   r6   r3   r   r}   r|   r   r   rF   r   r   r   r   r   r   r   <  s   r   c               @   s   e Zd Zdd ZdS )BinjfindJobc             C   s   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | |d t|| _| jd	k rtd
d S )Nr   Zlalapps_binjfindzlalapps_binjfind.subz*lalapps_binjfind-$(cluster)-$(process).outz*lalapps_binjfind-$(cluster)-$(process).errrl   rm   r   r   zfiles_per_binjfind < 1)r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   r(   r'   r   )r2   r   r   r   r   r3   c  s    


zBinjfindJob.__init__N)r4   r5   r6   r3   r   r   r   r   r   b  s   r   c               @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )BinjfindNodec             G   s2   t jj| f|  g | _| j| _t | jd< d S )Nrv   )r   rw   r3   r?   rx   r   ry   rz   )r2   r   r   r   r   r3   r  s    zBinjfindNode.__init__c             C   s<   | j | x*|D ]"}|j}tj| | | | qW d S )N)r?   r   r<   r   rw   r|   r   )r2   rQ   rB   rg   r   r   r   r}   x  s
    
zBinjfindNode.add_input_cachec             C   s   t d S )N)r   )r2   rg   r   r   r   r|     s    zBinjfindNode.add_file_argc             C   s   | j S )N)r?   )r2   r   r   r   r     s    zBinjfindNode.get_input_cachec             C   s   | j S )N)rx   )r2   r   r   r   rF     s    zBinjfindNode.get_output_cachec             C   s   t d S )N)r   )r2   r   r   r   r     s    zBinjfindNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r     s    zBinjfindNode.get_outputN)
r4   r5   r6   r3   r}   r|   r   rF   r   r   r   r   r   r   r   q  s   r   c               @   s   e Zd Zdd ZdS )BurcaJobc             C   s   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | d	d
 | |d t|| _| jdk rtdd S )Nr   lalapps_burcazlalapps_burca.subz'lalapps_burca-$(cluster)-$(process).outz'lalapps_burca-$(cluster)-$(process).errrl   rm   r   r   zMemory >= $(macrominram)r   zfiles_per_burca < 1)r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   r&   r%   r   )r2   r   r   r   r   r3     s    


zBurcaJob.__init__N)r4   r5   r6   r3   r   r   r   r   r     s   r   c               @   s   e Zd Zdd ZdS )	Burca2Jobc             C   s   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | |d	 t|| _d S )
Nr   r   zlalapps_burca2.subz(lalapps_burca2-$(cluster)-$(process).outz(lalapps_burca2-$(cluster)-$(process).errrl   rm   r   Zlalapps_burca2)r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   r   r   )r2   r   r   r   r   r3     s    
zBurca2Job.__init__N)r4   r5   r6   r3   r   r   r   r   r     s   r   c               @   sL   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dS )	BurcaNodec             G   s2   t jj| f|  g | _| j| _t | jd< d S )Nrv   )r   rw   r3   r?   rx   r   ry   rz   )r2   r   r   r   r   r3     s    zBurcaNode.__init__c             C   s   | j | x*|D ]"}|j}tj| | | | qW tdd | j D }|dkrb| dd n"|dkrx| dd n| dd d S )	Nc             s   s   | ]}t |jV  qd S )N)absrK   )rG   rI   r   r   r   	<genexpr>  s    z,BurcaNode.add_input_cache.<locals>.<genexpr>ia  Zmacrominrami  i'  i   r   )	r?   r   r<   r   rw   r|   r   r_   	add_macro)r2   rQ   rB   rg   Zlongest_durationr   r   r   r}     s    
zBurcaNode.add_input_cachec             C   s   t d S )N)r   )r2   rg   r   r   r   r|     s    zBurcaNode.add_file_argc             C   s   | j S )N)r?   )r2   r   r   r   r     s    zBurcaNode.get_input_cachec             C   s   | j S )N)rx   )r2   r   r   r   rF     s    zBurcaNode.get_output_cachec             C   s   t d S )N)r   )r2   r   r   r   r     s    zBurcaNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r     s    zBurcaNode.get_outputc             C   s   |  ddt|  d S )Nz--coincidence-segments %s,)r   r>   segmentsUtilsZto_range_strings)r2   seglistr   r   r   set_coincidence_segments  s    z"BurcaNode.set_coincidence_segmentsN)r4   r5   r6   r3   r}   r|   r   rF   r   r   r   r   r   r   r   r     s   r   c               @   s   e Zd Zdd ZdS )	SQLiteJobc             C   s|   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | |d d S )	Nr   Zligolw_sqlitezligolw_sqlite.subz'ligolw_sqlite-$(cluster)-$(process).outz'ligolw_sqlite-$(cluster)-$(process).errrl   rm   r   )r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   )r2   r   r   r   r   r3     s    
zSQLiteJob.__init__N)r4   r5   r6   r3   r   r   r   r   r     s   r   c               @   sL   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dS )
SQLiteNodec             G   s0   t jj| f|  g | _g | _t | jd< d S )Nrv   )r   rw   r3   r?   rx   r   ry   rz   )r2   r   r   r   r   r3     s    zSQLiteNode.__init__c             C   sJ   | j rtd| j| x*|D ]"}|j}tj| | | | q W d S )Nz5cannot change attributes after computing output cache)	rx   r   r?   r   r<   r   rw   r|   r   )r2   rQ   rB   rg   r   r   r   r}     s    
zSQLiteNode.add_input_cachec             C   s   t d S )N)r   )r2   rg   r   r   r   r|     s    zSQLiteNode.add_file_argc             C   s   | j rtd| d| d S )Nz5cannot change attributes after computing output cachemacrodatabase)rx   r   r   )r2   rg   r   r   r   r     s    zSQLiteNode.set_outputc             C   s   | j S )N)r?   )r2   r   r   r   r     s    zSQLiteNode.get_input_cachec             C   s&   | j s t| jd |  d g| _ | j S )Nr   )rx   rE   r?   r   )r2   r   r   r   rF     s    zSQLiteNode.get_output_cachec             C   s   t d S )N)r   )r2   r   r   r   r     s    zSQLiteNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r     s    zSQLiteNode.get_outputN)r4   r5   r6   r3   r}   r|   r   r   rF   r   r   r   r   r   r   r     s   	r   c               @   s   e Zd Zdd ZdS )BurcaTailorJobc             C   s   t j| dt|d | d | tjt	|d | 
tjt	|d | dd | dt| | |d t|| _d	| _d S )
Nr   Zlalapps_burca_tailorzlalapps_burca_tailor.subz.lalapps_burca_tailor-$(cluster)-$(process).outz.lalapps_burca_tailor-$(cluster)-$(process).errrl   rm   r   r   )r   rp   r3   r   rt   rq   r   r<   r>   r   rr   rs   r   r   r   r   r   )r2   r   r   r   r   r3   
  s    

zBurcaTailorJob.__init__N)r4   r5   r6   r3   r   r   r   r   r   	  s   r   c               @   s\   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd Zdd ZdS )BurcaTailorNodec             G   sd   t jj| f|  g | _g | _tjt | 	 j
| _
tjt | 	 j| _t | jd< d S )Nrv   )r   rw   r3   r?   rx   r   r<   r>   ry   r{   r   r   rz   )r2   r   r   r   r   r3     s    zBurcaTailorNode.__init__c             G   s0   t jj| f|  tj| jd|   | _d S )Nz%s.cache)	r   rw   r   r   r<   r>   r   r   r   )r2   r   r   r   r   r      s    zBurcaTailorNode.set_namec             C   sJ   | j rtd| j| x |D ]}|j}tj| | q W | | d S )Nz5cannot change attributes after computing output cache)	rx   r   r?   r   r<   r   rw   r|   r   )r2   rQ   rB   rg   r   r   r   r}   $  s    
zBurcaTailorNode.add_input_cachec             C   s   t d S )N)r   )r2   rg   r   r   r   r|   -  s    zBurcaTailorNode.add_file_argc          
   C   s   | j rtdt| j|d}tj| jd|j|j	t
|jd t
t|jf }| d| dtj| |_| j d d = | j | |S )Nz5cannot change attributes after computing output cachero   z%s-%s-%d-%d.xml.gzr   outputzfile://localhost)rx   r   rE   r?   r   r<   r>   r   r   r@   r-   rK   r   r   r=   rD   r   )r2   r@   rI   rg   r   r   r   r   0  s    4zBurcaTailorNode.set_outputc             C   s   | j S )N)r?   )r2   r   r   r   r   ;  s    zBurcaTailorNode.get_input_cachec             C   s   | j std| j S )Nz'must call set_output(description) first)rx   r   )r2   r   r   r   rF   >  s    z BurcaTailorNode.get_output_cachec             G   s^   xX|   D ]L}d|kr
t| jd}x| jD ]}tt||d q*W tjj| f|  P q
W d S )Nz--add-from-cacherc   )rd   )	get_argsrd   r   r?   re   rf   r   rw   r   )r2   r   argrh   rB   r   r   r   r   C  s    z!BurcaTailorNode.write_input_filesc             C   s   t d S )N)r   )r2   r   r   r   r   M  s    z BurcaTailorNode.get_output_filesc             C   s   t d S )N)r   )r2   r   r   r   r   P  s    zBurcaTailorNode.get_outputN)r4   r5   r6   r3   r   r}   r|   r   r   rF   r   r   r   r   r   r   r   r     s   	
r   datafindrmbinjpowerlladdbinjfind	buclusterbucutburcaburca2sqliteburcatailorc             C   s
  d|kr:t tjt t| tjt t| | ad|krJt	| a
d|krZt| ad|krjt| ad|krt tjt| | at| t_d|krt| ad|krt| ad|krt| ad	|krt| ad
|krt| ad|krt| ad|krt| adS )z.
	Construct definitions of the submit files.
	r   r   r   r   r   r   r   r   r   r   r   r   N) r   ZLSCDataFindJobr   r<   r>   ry   r   r   datafindjobrj   rmjobr~   binjjobr   powerjobZLigolwAddJoblladdjobr   r   binjfindjobr   bucutjobr   buclusterjobr   burcajobr   	burca2jobr   	sqlitejobr   burcatailorjob)r   Z	job_typesr   r   r   init_job_typeso  s2    2

r   c             C   sF   |dk rt ||| j d| j  }|| jk r2dS || j | j d S )z{
	Return the number of PSDs that can fit into a job of length t
	seconds.  In general, the return value is a non-integer.
	r      r   )r   r,   r.   r0   r1   )timing_paramstr   r   r   psds_from_job_length  s    
r   c             C   s<   |dk rt ||d | j | j }|d| j 7 }|| j S )z_
	From the analysis parameters and a count of PSDs, return the length
	of the job in seconds.
	r   r   )r   r1   r0   r.   r,   )r   Zpsdsresultr   r   r   job_length_from_psds  s
    r   c          	   C   s   t | |}d| j | j| j  }|| j }t }|d }x:|| |d krt|t||| |@  ||| 7 }q<W t	t
| t|d | }|r|t||t | |  |S )z
	Split the data segment into correctly-overlaping segments.  We try
	to have the numbers of PSDs in each segment be equal to
	psds_per_job, but with a short segment at the end if needed.
	r   r   r   )r   r.   r0   r1   r,   r   Zsegmentlistr   rK   r-   r   float)r   rK   psds_per_jobZ	joblengthZ
joboverlapsegsr   Z
extra_psdsr   r   r   split_segment  s    

r   c             C   s   t | tt|dkS )zC
	Return True if the segment can be analyzed using lalapps_power.
	g      ?)r   r   r   )r   rK   r   r   r   
segment_ok  s    r   c                s*   x$|   D ]}t fdd| q
W dS )z|
	Remove segments from seglistdict that are too short to analyze.

	CAUTION:  this function modifies seglistdict in place.
	c                s
   t  | S )N)r   )r   )r   r   r   rM     rN   z+remove_too_short_segments.<locals>.<lambda>N)valuesr   Zinplace_filter)seglistdictr   r   r   )r   r   remove_too_short_segments  s    r   i   c             C   s   t t}|d|t|d tt|f  ||d t  ||d d  ||_	|
|d  | d kr|t dd|  |d | | t|gS )Nzligo_data_find-%s-%d-%dr   r   r   ztype_%s   )r   ZLSCDataFindNoder   r   r-   r   r   datafind_padr   Z_AnalysisNode__ifoZset_observatoryget_typeset_typeZget_config_filer
   	set_retryadd_noderS   )dagrC   r   r[   r   r   r   make_datafind_fragment  s    
$

r  Fc             C   s
  t t|d}x|D ]}	||	 qW |d krJx&|D ]}	||	  q2W n
|| |d k	rf|| |d k	rx|| | \}
|d kr|
j}|d|t|d tt	|f  |j
tj|jd|
j|t|d tt	|f |d |d | | t|gS )N)r   zlladd_%s_%d_%dr   z%s-%s-%d-%d.xml.gz)rK   r   )r   r   
add_parentr}   rF   r   rK   r   r-   r   r   r   r<   r>   r   r   r   r  rS   )r  rP   r   rK   r?   r   Zpreserve_cacheZextra_input_cacher[   rH   rI   r   r   r   make_lladd_fragment  s&    





$8

r  c       
   	   C   s   t t}|d||t|d tt|f  t|j| || || |	|d  |
|d  || x&| D ]\}}	|d||	f  q~W | | t|gS )Nzlalapps_power_%s_%s_%d_%dr   r   z--%s %s)r   r   r   r-   r   mapr  Z	set_cacher   r   r   r   	iteritemsr   r  rS   )
r  rP   rC   r   r   
framecacheinjargsr[   r   valuer   r   r   make_power_fragment6  s    &



r  c       	      C   s   |d |d t j  t j|  }tt }||j || ||d  |d k	rr|d|t|t|f  n|d|t|f  |	| |d k	r|
d| |d k	r|
d| |
dtt d |  | | t|gS )	Nr   r   zlalapps_binj_%s_%d_%dzlalapps_binj_%s_%dZ	macroflowZ
macrofhighZ	macroseedd   )r   r   r   r   r<   r   r   r   r-   r   r   timer  rS   )	r  r   Ztime_slides_cache_entryr   offsetZflowZfhighr   r[   r   r   r   make_binj_fragmentJ  s      


r  c       	      C   s   t |}t }x|rtt}|dd |d tj D  x.tdd |d tj D D ]}|| qXW |d tj= t| }|	d|t
|d t
t|f  |d| | | || qW |S )Nc             S   s   g | ]\}}|qS r   r   )rG   rI   rH   r   r   r   rJ   e  s    z*make_binjfind_fragment.<locals>.<listcomp>c             s   s   | ]\}}|V  qd S )Nr   )rG   rI   rH   r   r   r   r   f  s    z)make_binjfind_fragment.<locals>.<genexpr>zlalapps_binjfind_%s_%d_%dr   macrocomment)rR   rS   r   r   r}   r'   r  rb   r   r   r-   r   r   r  rU   )	r  rP   r   verboser?   rX   r[   rH   r   r   r   r   make_binjfind_fragment`  s    "$
r  c       	      C   s   t |}t }x|rtt}|dd |d tj D  x.tdd |d tj D D ]}|| qXW |d tj= t| }|	d|t
|d t
t|f  |d| |d | | || qW |S )	Nc             S   s   g | ]\}}|qS r   r   )rG   rI   rH   r   r   r   rJ   v  s    z+make_bucluster_fragment.<locals>.<listcomp>c             s   s   | ]\}}|V  qd S )Nr   )rG   rI   rH   r   r   r   r   w  s    z*make_bucluster_fragment.<locals>.<genexpr>zlalapps_bucluster_%s_%d_%dr   r  r   )rR   rS   r   r   r}   r    r  rb   r   r   r-   r   r   r   r  rU   )	r  rP   r   r  r?   rX   r[   rH   r   r   r   r   make_bucluster_fragmentq  s    "$

r  c       	      C   s   t |}t }x|rtt}|dd |d tj D  x.tdd |d tj D D ]}|| qXW |d tj= t| }|	d|t
|d t
t|f  |d| | | || qW |S )Nc             S   s   g | ]\}}|qS r   r   )rG   rI   rH   r   r   r   rJ     s    z'make_bucut_fragment.<locals>.<listcomp>c             s   s   | ]\}}|V  qd S )Nr   )rG   rI   rH   r   r   r   r     s    z&make_bucut_fragment.<locals>.<genexpr>zlalapps_bucut_%s_%d_%dr   r  )rR   rS   r   r   r}   r#   r  rb   r   r   r-   r   r   r  rU   )	r  rP   r   r  r?   rX   r[   rH   r   r   r   r   make_bucut_fragment  s    "$
r  c       
      C   s   t |}|d k	r t|dks tt }x|rtt}|dd |d tj D  x.tdd |d tj D D ]}|| qpW |d tj= t	|
 }	|d|t|	d tt|	f  |d k	r|| |d| | | || q(W |S )	Nr   c             S   s   g | ]\}}|qS r   r   )rG   rI   rH   r   r   r   rJ     s    z'make_burca_fragment.<locals>.<listcomp>c             s   s   | ]\}}|V  qd S )Nr   )rG   rI   rH   r   r   r   r     s    z&make_burca_fragment.<locals>.<genexpr>zlalapps_burca_%s_%d_%dr   r  )rR   rV   AssertionErrorrS   r   r   r}   r%   r  rb   r   r   r-   r   r   r   r  rU   )
r  rP   r   Zcoincidence_segmentsr  r?   rX   r[   rH   r   r   r   r   make_burca_fragment  s$    "$

r  c       	      C   s   t |}t }xl|D ]d\}}tt}||g || |d|t|f  ||j	
dd | | || qW |S )Nzligolw_sqlite_%s_%dz.xml.gzz.sqlite)rR   rS   r   r   r}   r  r   rV   r   r<   replacer  rU   )	r  rP   r   r  r?   rX   rI   rH   r[   r   r   r   make_sqlite_fragment  s    

r  c             C   s  t |}|jdd t }d}x|rg }d}x<|rh||krh||  |tt|d jd d 7 }q.W tt	}|
| |d|t|d tt|t|f  |d	|t|f  | | || q W tt	}|d
|t|d tt|f  x&|D ]}	||	 |
|	  qW | d d = |d|j  || | | t| t|  }
|
rtt}|d|t|d tt|f  || |
|
 | | t|gS )NT)reverse   r   g     @r   z lalapps_burca_tailor_%s_%d_%d_%dz%s_%dzlalapps_burca_tailor_%s_%d_%dz--add-from-cache %sz lalapps_burca_tailor_rm_%s_%d_%d)listr;   rS   r   popr   r   rK   r   r   r}   r   r-   rV   r   r  rU   r  rF   r   r   r   r   ru   r   )r  r?   r   r   rX   max_cost_per_jobrQ   costr[   rH   Zdelete_cacheZrmnoder   r   r   make_burca_tailor_fragment  sB    "
*
$



$


r   c             C   s(  t |}|jdd tjtjd| }t|d}x&dd |D D ]}tt	||d qBW t
 }d}x|r"g }	d	}
x<|r|
|kr|	|  |
tt|	d
 jd d 7 }
qvW tt}|d|t|f  |d| |d|  ||	 x|D ]}|| qW | | || qfW |S )NT)r  zburca2_%s.cacherc   c             S   s   g | ]}|  D ]}|qqS r   )rF   )rG   r[   rI   r   r   r   rJ     s    z(make_burca2_fragment.<locals>.<listcomp>)rd   
   r   r  g     @r   zlalapps_burca2_%s_%dr  z--likelihood-data-cache %s)r  r;   r   r<   r>   r   r   rd   re   rf   rS   r   r  r   r   rK   r   r   rV   r   r   r}   r  r  rU   )r  Zcoinc_cacheZlikelihood_parentsr   Zlikelihood_data_cache_filenameZlikelihood_data_cache_filerI   rX   r  rQ   r  r[   rH   r   r   r   make_burca2_fragment  s0    
"


r"  c       	      C   s   |rt dtjd | td td }dd | D }|  t	 }x>|D ]6\}}|rvt d||f tjd t
| ||}||O }qTW |S )Nz building ligo_data_find jobs ...)rd   r   c             S   s"   g | ]\}}|D ]}||fqqS r   r   )rG   rC   r   r   r   r   r   rJ     s    z'make_datafind_stage.<locals>.<listcomp>z&making datafind job for %s spanning %s)re   sysstderrcopyZprotractr   Zcontractr  r;   rS   r  )	r  rA   r  Zfilledr   rX   r   rC   Z	new_nodesr   r   r   make_datafind_stage  s    	r&  c	             C   s   dd |D \}	|r.dd |D \}
d|
i}ni }t |||}|rXtdt| tjd t }x*|D ]"}|t| ||B ||||	|dO }qdW |S )zp
	Construct a DAG fragment for an entire segment, splitting the
	segment into multiple trigger generator jobs.
	c             S   s   g | ]}|  qS r   )r   )rG   r[   r   r   r   rJ   A  s    z/make_power_segment_fragment.<locals>.<listcomp>c             S   s    g | ]}|  D ]
}|jqqS r   )rF   r<   )rG   r[   rI   r   r   r   rJ   C  s    zinjection-filezSegment split: )rd   )r	  )r   re   rf   r#  r$  rS   r  )r  ZdatafindnodesrC   rK   r   r   r   	binjnodesr  r  Zsimfiler	  r   rX   r   r   r   r   make_power_segment_fragment9  s    

"r(  c                s   g }x|  D ]\ }	x|	D ]x|r>td tf tjd t fdd|D }
t|
dkrvtdt f |t| |
 |||||d	7 }qW qW |S )Nzgenerating %s fragment %s)rd   c                s4   g | ],}|   krt| | kr|qS r   )r   r   rK   r   r   )rG   r[   )rC   r   r   r   rJ   ]  s    z0make_single_instrument_stage.<locals>.<listcomp>r   zOerror, not exactly 1 datafind is suitable for trigger generator job at %s in %s)r'  r  )	r  re   rf   r#  r$  rS   rV   r   r(  )r  Z	datafindsr   r   r   r   r'  r  rX   r   Zdfnodesr   )rC   r   r   make_single_instrument_stageU  s    
&r)  c             C   s  |sg S |rt dtjd tjdd | D |||d\}}dd |D }tt|t|ks`tdd |D }d gt| }|d k	r,dd |D }	xt|	D ]\}
}|
d	ks|	|
d
  	|rt
j}n|d	 }|
t|	d
 ks|	|
d
  	|rt
j}n|d
 }|t
jk	s|t
jk	rt
||||
< qW |r@t dtjd t| |\}}|rt|rtt d|t| f tjd t||||S )Nz'Grouping jobs for coincidence analysis:)rd   c             S   s   g | ]}|  D ]}|qqS r   )rF   )rG   rH   rI   r   r   r   rJ   ~  s    z'group_coinc_parents.<locals>.<listcomp>)extentlimitr  c             S   s   g | ]}t |jqS r   )	frozensetobjects)rG   binr   r   r   rJ     s    c             S   s   g | ]}t |jqS r   )rb   r,  )rG   r-  r   r   r   rJ     s    c             S   s   g | ]
}|j qS r   )extent)rG   r-  r   r   r   rJ     s    r   r   zMatching jobs to caches ...zRNotice:  %d jobs (of %d) produce output that will not be used by a coincidence job)re   r#  r$  r   Zligolw_caferV   rS   r  	enumerateZdisjointr   ZNegInfinityZPosInfinityrK   r]   rT   )rP   Zoffset_vectorsr*  r  rA   ZbinsrY   r   ZclipsegsZextentsir.  lohiZparent_groupsr\   r   r   r   group_coinc_parentsq  s4     
"r3  )r   )NNFNN)NN)F)F)F)NF)F)F)NF)cr7   r   r   r#  r  Zligor   Zligo.segmentsr   r   Zgluer   r   r   Z	lal.utilsr   r/   ro   r   
__author____date____version__r   r   r   r   r   r   r   r   r"   r$   r&   r(   objectr)   rE   rR   r]   rb   ri   rp   rj   rw   ru   r   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r  r   r"  r&  rS   r(  r)  r3  r   r   r   r   <module>   s   	!"?D:&+'K
A
"





%)
2