B ß‹dÿrã@sdZddlmZdZddlmZejZejZ ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZddlZddlZddlZddlZddlmZyddlmZWn ek rÄdd lmZYnXy ddlZWn YnXdd lm Z m!Z!d d „Z"Gd d„de#ƒZ$Gdd„de$ƒZ%Gdd„de$ƒZ&Gdd„de$ƒZ'Gdd„de$ƒZ(Gdd„de$ƒZ)Gdd„de#ƒZ*Gdd„de+ƒZ,Gdd„de,ƒZ-Gdd „d e+ƒZ.Gd!d"„d"e+ƒZ/Gd#d$„d$e/ƒZ0Gd%d&„d&e+ƒZ1Gd'd(„d(e+ƒZ2Gd)d*„d*e+ƒZ3Gd+d,„d,e+ƒZ4Gd-d.„d.e+ƒZ5Gd/d0„d0e+ƒZ6Gd1d2„d2e+ƒZ7Gd3d4„d4e-e2ƒZ8Gd5d6„d6e/e3ƒZ9Gd7d8„d8e-e2ƒZ:Gd9d:„d:e/e3ƒZ;Gd;d<„d„d>e/e3ƒZ=Gd?d@„d@e-e2ƒZ>GdAdB„dBe/e3ƒZ?GdCdD„dDe-e2ƒZ@GdEdF„dFe/e3ƒZAGdGdH„dHe@ƒZBGdIdJ„dJeAƒZCGdKdL„dLe jDƒZEdS)Ma* This modules contains objects that make it simple for the user to create python scripts that build Condor DAGs to run code on the LSC Data Grid. This file is part of the Grid LSC User Environment (GLUE) GLUE is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . é)Úprint_functionz*Duncan Brown )Ú git_versionN)Úurllib)Úmd5)Údecode)Úloads)Ú configparserÚStringIOcCs|dddkS)zd Return True if t is in the S2 playground, False otherwise t = GPS time to test if playground i Õw+iâiX©)Útr r úZ/work/yifan.wang/ringdown/master-ringdown-env/lib/python3.7/site-packages/glue/pipeline.pyÚs2play6sr c@seZdZdZddd„ZdS)Ú CondorErrorzError thrown by Condor JobsNcCs ||_dS)N)Úargs)Úselfrr r r Ú__init__@szCondorError.__init__)N)Ú__name__Ú __module__Ú __qualname__Ú__doc__rr r r r r>src@s eZdZdS)ÚCondorJobErrorN)rrrr r r r rBsrc@s eZdZdS)ÚCondorSubmitErrorN)rrrr r r r rDsrc@s eZdZdS)ÚCondorDAGErrorN)rrrr r r r rFsrc@s eZdZdS)ÚCondorDAGJobErrorN)rrrr r r r rHsrc@s eZdZdS)ÚCondorDAGNodeErrorN)rrrr r r r rJsrc@seZdZddd„ZdS)Ú SegmentErrorNcCs ||_dS)N)r)rrr r r rMszSegmentError.__init__)N)rrrrr r r r rLsrc@sPeZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd d „Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd„Zd d!„Zd"d#„Zd$d%„Zd&d'„Zd(d)„Zd*d+„Zd,d-„Zd.d/„Zd0d1„Zd2d3„Zd4d5„Zd6d7„Zd8d9„Zd:d;„Z dd?„Z"d@dA„Z#dBdC„Z$dDdE„Z%dFdG„Z&dHdI„Z'dJdK„Z(dLdM„Z)dNdO„Z*dPdQ„Z+dRS)SÚ CondorJobz{ Generic condor job class. Provides methods to set the options in the condor submit file for a particular executable cCs|||_||_||_i|_i|_g|_i|_d|_d|_d|_ d|_ d|_ d|_ g|_ g|_g|_d|_d|_d|_d|_dS)zš @param universe: the condor universe to run the job in. @param executable: the executable to run. @param queue: number of jobs to queue. NT)Ú_CondorJob__universeÚ_CondorJob__executableÚ_CondorJob__queueÚ_CondorJob__optionsÚ_CondorJob__short_optionsÚ_CondorJob__argumentsÚ_CondorJob__condor_cmdsÚ_CondorJob__notificationÚ_CondorJob__log_fileÚ_CondorJob__in_fileÚ_CondorJob__err_fileÚ_CondorJob__out_fileÚ_CondorJob__sub_file_pathÚ_CondorJob__output_filesÚ_CondorJob__input_filesÚ_CondorJob__checkpoint_filesÚ_CondorJob__grid_typeÚ_CondorJob__grid_serverÚ_CondorJob__grid_schedulerZ _CondorJob__executable_installed)rÚuniverseÚ executableÚqueuer r r rVs(zCondorJob.__init__cCs|jS)z9 Return the name of the executable for this job. )r)rr r r Úget_executablesszCondorJob.get_executablecCs ||_dS)z6 Set the name of the executable for this job. N)r)rr1r r r Úset_executableyszCondorJob.set_executablecCs|jS)z> Return the condor universe that the job will run in. )r)rr r r Ú get_universeszCondorJob.get_universecCs ||_dS)zt Set the condor universe for the job to run in. @param universe: the condor universe to run the job in. N)r)rr0r r r Ú set_universe…szCondorJob.set_universecCs|jS)z* Return the grid type of the job. )r-)rr r r Ú get_grid_typeŒszCondorJob.get_grid_typecCs ||_dS)za Set the type of grid resource for the job. @param grid_type: type of grid resource. N)r-)rZ grid_typer r r Ú set_grid_type’szCondorJob.set_grid_typecCs|jS)z; Return the grid server on which the job will run. )r.)rr r r Úget_grid_server™szCondorJob.get_grid_servercCs ||_dS)zk Set the grid server on which to run the job. @param grid_server: grid server on which to run. N)r.)rZ grid_serverr r r Úset_grid_serverŸszCondorJob.set_grid_servercCs|jS)z$ Return the grid scheduler. )r/)rr r r Úget_grid_scheduler¦szCondorJob.get_grid_schedulercCs ||_dS)z\ Set the grid scheduler. @param grid_scheduler: grid scheduler on which to run. N)r/)rZgrid_schedulerr r r Úset_grid_scheduler¬szCondorJob.set_grid_schedulercCs||j|<dS)z¢ Add a Condor command to the submit file (e.g. a class add or evironment). @param cmd: Condor command directive. @param value: value for command. N)r#)rÚcmdÚvaluer r r Úadd_condor_cmd³szCondorJob.add_condor_cmdcCs|jS)zD Return the dictionary of condor keywords to add to the job )r#)rr r r Úget_condor_cmds»szCondorJob.get_condor_cmdscCs||jkr|j |¡dS)zo Add filename as a necessary input file for this DAG node. @param filename: input filename to add N)r+Úappend)rÚfilenamer r r Úadd_input_fileÁs zCondorJob.add_input_filecCs||jkr|j |¡dS)zg Add filename as a output file for this DAG node. @param filename: output filename to add N)r*rA)rrBr r r Úadd_output_fileÊs zCondorJob.add_output_filecCs||jkr|j |¡dS)z= Add filename as a checkpoint file for this DAG job. N)r,rA)rrBr r r Úadd_checkpoint_fileÓs zCondorJob.add_checkpoint_filecCs|jS)z7 Return list of input files for this DAG node. )r+)rr r r Úget_input_filesÚszCondorJob.get_input_filescCs|jS)z8 Return list of output files for this DAG node. )r*)rr r r Úget_output_filesàszCondorJob.get_output_filescCs|jS)z= Return a list of checkpoint files for this DAG node )r,)rr r r Úget_checkpoint_filesæszCondorJob.get_checkpoint_filescCs|j |¡dS)z™ Add an argument to the executable. Arguments are appended after any options and their order is guaranteed. @param arg: argument to add. N)r"rA)rÚargr r r Úadd_argìszCondorJob.add_argcCs&|j |¡||jkr"|j |¡dS)z÷ Add a file argument to the executable. Arguments are appended after any options and their order is guaranteed. Also adds the file name to the list of required input data for this job. @param filename: file to add as argument. N)r"rAr+)rrBr r r Ú add_file_argôs  zCondorJob.add_file_argcCs|jS)zO Return the list of arguments that are to be passed to the executable. )r")rr r r Úget_argsÿszCondorJob.get_argscCs||j|<dS)aÀ Add a command line option to the executable. The order that the arguments will be appended to the command line is not guaranteed, but they will always be added before any command line arguments. The name of the option is prefixed with double hyphen and the program is expected to parse it with getopt_long(). @param opt: command line option to add. @param value: value to pass to the option (None for no argument). N)r )rÚoptr>r r r Úadd_opts zCondorJob.add_optcCs||jkr|j|SdS)z± Returns the value associated with the given command line option. Returns None if the option does not exist in the options list. @param opt: command line option N)r )rrMr r r Úget_opts  zCondorJob.get_optcCs$||j|<||jkr |j |¡dS)aÀ Add a command line option to the executable. The order that the arguments will be appended to the command line is not guaranteed, but they will always be added before any command line arguments. The name of the option is prefixed with double hyphen and the program is expected to parse it with getopt_long(). @param opt: command line option to add. @param value: value to pass to the option (None for no argument). N)r r+rA)rrMrBr r r Ú add_file_opts  zCondorJob.add_file_optcCs|jS)z4 Return the dictionary of opts for the job. )r )rr r r Úget_opts)szCondorJob.get_optscCs||j|<dS)a{ Add a command line option to the executable. The order that the arguments will be appended to the command line is not guaranteed, but they will always be added before any command line arguments. The name of the option is prefixed with single hyphen and the program is expected to parse it with getopt() or getopt_long() (if a single character option), or getopt_long_only() (if multiple characters). Long and (single-character) short options may be mixed if the executable permits this. @param opt: command line option to add. @param value: value to pass to the option (None for no argument). N)r!)rrMr>r r r Ú add_short_opt/s zCondorJob.add_short_optcCs|jS)z= Return the dictionary of short options for the job. )r!)rr r r Úget_short_opts=szCondorJob.get_short_optscCs6x0| |¡D]"}t| ||¡ƒ ¡}||j|<q WdS)zè Parse command line options from a given section in an ini file and pass to the executable. @param cp: ConfigParser object pointing to the ini file. @param section: section of the ini file to add to the options. N)ÚoptionsÚstrÚgetÚstripr )rÚcpÚsectionrMrIr r r Ú add_ini_optsCszCondorJob.add_ini_optscCs ||_dS)zv Set the email address to send notification to. @param value: email address or never for no notification. N)r$)rr>r r r Úset_notificationNszCondorJob.set_notificationcCs ||_dS)zE Set the Condor log file. @param path: path to log file. N)r%)rÚpathr r r Ú set_log_fileUszCondorJob.set_log_filecCs ||_dS)zk Set the file from which Condor directs the stdin of the job. @param path: path to stdin file. N)r&)rr\r r r Úset_stdin_file\szCondorJob.set_stdin_filecCs|jS)zF Get the file from which Condor directs the stdin of the job. )r&)rr r r Úget_stdin_filecszCondorJob.get_stdin_filecCs ||_dS)zk Set the file to which Condor directs the stderr of the job. @param path: path to stderr file. N)r')rr\r r r Úset_stderr_fileiszCondorJob.set_stderr_filecCs|jS)zE Get the file to which Condor directs the stderr of the job. )r')rr r r Úget_stderr_filepszCondorJob.get_stderr_filecCs ||_dS)zk Set the file to which Condor directs the stdout of the job. @param path: path to stdout file. N)r()rr\r r r Úset_stdout_filevszCondorJob.set_stdout_filecCs|jS)zE Get the file to which Condor directs the stdout of the job. )r()rr r r Úget_stdout_file}szCondorJob.get_stdout_filecCs ||_dS)z Set the name of the file to write the Condor submit file to when write_sub_file() is called. @param path: path to submit file. N)r))rr\r r r Ú set_sub_fileƒszCondorJob.set_sub_filecCs|jS)zw Get the name of the file which the Condor submit file will be written to when write_sub_file() is called. )r))rr r r Ú get_sub_file‹szCondorJob.get_sub_filecCsL|jstdƒ‚|jstdƒ‚|js*tdƒ‚|js8tdƒ‚yt|jdƒ}Wntd|jƒ‚YnX|jdkrØ|jdkr‚td ƒ‚nV|jd kr |jdkrØtd ƒ‚n8|jd krÐ|jdkr¼td ƒ‚|j dkrØtd ƒ‚ntdƒ‚|  d|jd¡|  d|j d¡|jdkrT|jd kr.|  d|j|jf¡|jd krT|  d|j|j|j f¡|jdkr~|  d¡|  d¡|  d¡t |j  ¡ƒs¦t |j ¡ƒs¦|jrv|  d¡xL|j  ¡D]>}|j |rê|  d|d|j |¡n|  d|¡q¼WxL|j ¡D]>}|j|r8|  d|d|j|¡n|  d|¡q Wx|jD]}|  d|¡qTW|  d¡x8|j ¡D]*}|  t|ƒdt|j|ƒd¡q‚W|  d|jd¡|jdk rä|  d|jd¡|  d|jd¡|  d |jd¡|jr(|  d!|jd¡|  d"t|jƒd¡| ¡dS)#z2 Write a submit file for this Condor job. zLog file not specified.zError file not specified.zOutput file not specified.zNo path for submit file.ÚwzCannot open file ÚgridNzNo grid type specified.Zgt2z&No server specified for grid resource.Zgt4z)No scheduler specified for grid resource.zUnsupported grid resource.z universe = Ú z executable = zgrid_resource = %s %s zgrid_resource = %s %s %s z"when_to_transfer_output = ON_EXIT z'transfer_output_files = $(macrooutput) z%transfer_input_files = $(macroinput) z arguments = "z --ú z -z " z = zlog = zinput = zerror = z output = znotification = zqueue )r%rr'r(r)Úopenrr-r.r/ÚwriterÚlistr Úkeysr!r"r#rUr&r$rÚclose)rZsubfileÚcr=r r r Úwrite_sub_file’sz                    (     * zCondorJob.write_sub_fileN),rrrrrr3r4r5r6r7r8r9r:r;r<r?r@rCrDrErFrGrHrJrKrLrNrOrPrQrRrSrZr[r]r^r_r`rarbrcrdrerpr r r r rQsR      rcsDeZdZdZ‡fdd„Zdd„Zd dd„Zd d „Zdd d „Z‡Z S)Ú CondorDAGJobz° A Condor DAG job never notifies the user on completion and can have variable options that are set for a particular node in the DAG. Inherits methods from a CondorJob. csHtt|ƒ ||d¡t |d¡g|_d|_g|_g|_t   d¡|_ dS)zn universe = the condor universe to run the job in. executable = the executable to run in the DAG. éÚneverrz[_-]N) Úsuperrqrrr[Ú_CondorDAGJob__var_optsÚ_CondorDAGJob__arg_indexÚ_CondorDAGJob__var_argsÚ_CondorDAGJob__var_cmdsÚreÚcompileÚ_CondorDAGJob__bad_macro_chars)rr0r1)Ú __class__r r rès zCondorDAGJob.__init__cCst|ƒS)a> Create a condor node from this job. This provides a basic interface to the CondorDAGNode class. Most jobs in a workflow will subclass the CondorDAGNode class and overwrite this to give more details when initializing the node. However, this will work fine for jobs with very simp input/output. )Ú CondorDAGNode)rr r r Ú create_nodeõszCondorDAGJob.create_nodeFcCsV||jkrR|j |¡|j d|¡}|r>| |d|d¡n| |d|d¡dS)zß Add a variable (or macro) option to the condor job. The option is added to the submit file and a different argument to the option can be set for each node in the DAG. @param opt: name of option to add. Úz$(macroú)N)rurAr{ÚsubrRrN)rrMÚshortÚmacror r r Ú add_var_optÿs   zCondorDAGJob.add_var_optcCs<||jkr8|j |¡|j d|¡}| |d|d¡dS)z| Add a condor command to the submit file that allows variable (macro) arguments to be passes to the executable. rz$(macror€N)rxrAr{rr?)rÚcommandrƒr r r Úadd_var_condor_cmds  zCondorDAGJob.add_var_condor_cmdcCsˆy|j|Wnttk r‚||jkr0tdƒ‚|rJ|j dt|ƒ¡n|j dt|ƒ¡| |j|j¡|jd7_YnXdS)zr Add a command to the submit file to allow variable (macro) arguments to be passed to the executable. z+mismatch between job and node var_arg indexz'$(macroargument%s)'z$(macroargument%s)rrN)rwÚ IndexErrorrvrrArUrJ)rÚ arg_indexÚquoter r r Ú add_var_args zCondorDAGJob.add_var_arg)F)F) rrrrrr~r„r†rŠÚ __classcell__r r )r|r rqâs    rqc@sReZdZdZddd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dS)ÚCondorDAGManJobzU Condor DAGMan job class. Appropriate for setting up DAGs to run within a DAG. NcCs||_d|_||_dS)zn dag = the name of the condor dag file to run dir = the diretory in which the dag file is located N)Ú_CondorDAGManJob__dagÚ_CondorDAGManJob__notificationÚ_CondorDAGManJob__dag_directory)rZdagÚdirr r r r/szCondorDAGManJob.__init__cCst|ƒS)aD Create a condor node from this job. This provides a basic interface to the CondorDAGManNode class. Most jobs in a workflow will subclass the CondorDAGManNode class and overwrite this to give more details when initializing the node. However, this will work fine for jobs with very simp input/output. )ÚCondorDAGManNode)rr r r r~8szCondorDAGManJob.create_nodecCs ||_dS)zy Set the directory where the dag will be run @param dir: the name of the directory where the dag will be run N)r)rrr r r Úset_dag_directoryBsz!CondorDAGManJob.set_dag_directorycCs|jS)z5 Get the directory where the dag will be run )r)rr r r Úget_dag_directoryIsz!CondorDAGManJob.get_dag_directorycCs ||_dS)zv Set the email address to send notification to. @param value: email address or never for no notification. N)rŽ)rr>r r r r[Osz CondorDAGManJob.set_notificationcCs|jS)zp Return the name of the dag as the submit file name for the SUBDAG EXTERNAL command in the uber-dag )r)rr r r reVszCondorDAGManJob.get_sub_filecCsdS)zm Do nothing as there is not need for a sub file with the SUBDAG EXTERNAL command in the uber-dag Nr )rr r r rp]szCondorDAGManJob.write_sub_filecCs|jS)z4 Return the name of any associated dag file )r)rr r r Úget_dagdszCondorDAGManJob.get_dag)N) rrrrrr~r’r“r[rerpr”r r r r rŒ*s  rŒc@sžeZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd d „Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd„Zd d!„Zd"d#„Zd$d%„Zd&d'„Zd(d)„Zd*d+„Zd,d-„Zd.d/„Zd0d1„Zd2d3„Zd4d5„Zd6d7„Zd8d9„Zdfd;d<„Z dgd=d>„Z!dhd?d@„Z"dAdB„Z#dCdD„Z$dEdF„Z%dGdH„Z&dIdJ„Z'dKdL„Z(dMdN„Z)dOdP„Z*dQdR„Z+dSdT„Z,dUdV„Z-dWdX„Z.dYdZ„Z/d[d\„Z0d]d^„Z1d_d`„Z2dadb„Z3dcdd„Z4deS)ir}zõ A CondorDAGNode represents a node in the DAG. It corresponds to a particular condor job (and so a particular submit file). If the job has variable (macro) options, they can be set here so each nodes executes with the correct options. cCst|tƒst|tƒstdƒ‚d|_||_d|_d|_d|_g|_ d|_ g|_ i|_ i|_ g|_d|_d|_g|_t d¡|_g|_g|_g|_d|_t|tƒr²| ¡dkr²d|_nd|_ttt ¡dƒƒ}ttt ¡dƒƒ}t|jƒ}t ||| !¡ƒ "¡|_|j|_#dS) zB @param job: the CondorJob that this node corresponds to. zCA DAG node must correspond to a Condor DAG job or Condor DAGMan jobNrz[_-]ÚstandardÚnoneièl;á )$Ú isinstancerqrŒrÚ_CondorDAGNode__nameÚ_CondorDAGNode__jobÚ_CondorDAGNode__categoryÚ_CondorDAGNode__priorityÚ_CondorDAGNode__pre_scriptÚ_CondorDAGNode__pre_script_argsÚ_CondorDAGNode__post_scriptÚ _CondorDAGNode__post_script_argsÚ_CondorDAGNode__macrosÚ_CondorDAGNode__optsÚ_CondorDAGNode__argsÚ_CondorDAGNode__arg_indexÚ_CondorDAGNode__retryÚ_CondorDAGNode__parentsryrzÚ_CondorDAGNode__bad_macro_charsÚ_CondorDAGNode__output_filesÚ_CondorDAGNode__input_filesÚ _CondorDAGNode__checkpoint_filesZ_CondorDAGNode__vds_groupr5Z_CondorDAGNode__grid_startrUÚintÚtimeÚrandomr|rÚencodeÚ hexdigestZ_CondorDAGNode__md5name)rÚjobr ÚrÚar r r rrs>    zCondorDAGNode.__init__cCs|jS)N)r˜)rr r r Ú__repr__™szCondorDAGNode.__repr__cCs|jS)zA Return the CondorJob that this node is associated with. )r™)rr r r r¯œszCondorDAGNode.jobcCs ||_dS)z| Sets the name of the pre script that is executed before the DAG node is run. @param script: path to script N)rœ)rÚscriptr r r Úset_pre_script¢szCondorDAGNode.set_pre_scriptcCs|j |¡dS)z] Adds an argument to the pre script that is executed before the DAG node is run. N)rrA)rrIr r r Úadd_pre_script_argªsz CondorDAGNode.add_pre_script_argcCs ||_dS)z} Sets the name of the post script that is executed before the DAG node is run. @param script: path to script N)rž)rr³r r r Úset_post_script±szCondorDAGNode.set_post_scriptcCs|jS)z€ returns the name of the post script that is executed before the DAG node is run. @param script: path to script )rž)rr r r Úget_post_script¹szCondorDAGNode.get_post_scriptcCs|j |¡dS)z^ Adds an argument to the post script that is executed before the DAG node is run. N)rŸrA)rrIr r r Úadd_post_script_argÁsz!CondorDAGNode.add_post_script_argcCs|jS)zl Returns and array of arguments to the post script that is executed before the DAG node is run. )rŸ)rr r r Úget_post_script_argÈsz!CondorDAGNode.get_post_script_argcCst|ƒ|_dS)z0 Set the name for this node in the DAG. N)rUr˜)rÚnamer r r Úset_nameÏszCondorDAGNode.set_namecCs|jS)z0 Get the name for this node in the DAG. )r˜)rr r r Úget_nameÕszCondorDAGNode.get_namecCst|ƒ|_dS)z4 Set the category for this node in the DAG. N)rUrš)rÚcategoryr r r Ú set_categoryÛszCondorDAGNode.set_categorycCs|jS)z4 Get the category for this node in the DAG. )rš)rr r r Ú get_categoryászCondorDAGNode.get_categorycCst|ƒ|_dS)z4 Set the priority for this node in the DAG. N)rUr›)rÚpriorityr r r Ú set_priorityçszCondorDAGNode.set_prioritycCs|jS)z4 Get the priority for this node in the DAG. )r›)rr r r Ú get_priorityíszCondorDAGNode.get_prioritycCsB||jkr>|j |¡t| ¡tƒs>| ¡ ¡dkr>| |¡dS)zo Add filename as a necessary input file for this DAG node. @param filename: input filename to add rgN)r¨rAr—r¯rŒr5Úadd_input_macro)rrBr r r rCós   zCondorDAGNode.add_input_filecCsB||jkr>|j |¡t| ¡tƒs>| ¡ ¡dkr>| |¡dS)zg Add filename as a output file for this DAG node. @param filename: output filename to add rgN)r§rAr—r¯rŒr5Úadd_output_macro)rrBr r r rDÿs   zCondorDAGNode.add_output_filecCsB||jkr>|j |¡t| ¡tƒs>| ¡ ¡dkr>| |¡dS)zm Add filename as a checkpoint file for this DAG node @param filename: checkpoint filename to add rgN)r©rAr—r¯rŒr5Úadd_checkpoint_macro)rrBr r r rE s   z!CondorDAGNode.add_checkpoint_filecCs,t|jƒ}t| ¡tƒr(|| ¡ ¡}|S)zC Return list of input files for this DAG node and its job. )rlr¨r—r¯rqrF)rZ input_filesr r r rFs zCondorDAGNode.get_input_filescCs,t|jƒ}t| ¡tƒr(|| ¡ ¡}|S)zD Return list of output files for this DAG node and its job. )rlr§r—r¯rqrG)rZ output_filesr r r rGs zCondorDAGNode.get_output_filescCs,t|jƒ}t| ¡tƒr(|| ¡ ¡}|S)zJ Return a list of checkpoint files for this DAG node and its job. )rlr©r—r¯rqrH)rZcheckpoint_filesr r r rH(s z"CondorDAGNode.get_checkpoint_filescCs|j d|¡}||j|<dS)a… Add a variable (macro) for this node. This can be different for each node in the DAG, even if they use the same CondorJob. Within the CondorJob, the value of the macro can be referenced as '$(name)' -- for instance, to define a unique output or error file for each node. @param name: macro name. @param value: value of the macro for this node in the DAG rN)r¦rr¡)rrºr>rƒr r r Ú add_macro1s zCondorDAGNode.add_macrocCsL|j d|¡}||jkr$||j|<n$||j|krH|j|d|7<dS)z¾ Add a variable (macro) for storing the input/output files associated with this node. @param io: macroinput or macrooutput @param filename: filename of input/output file rz,%sN)r¦rr¡)rÚiorBr r r Ú add_io_macro>s   zCondorDAGNode.add_io_macrocCs| d|¡dS)z‡ Add a variable (macro) for storing the input files associated with this node. @param filename: filename of input file Z macroinputN)rÈ)rrBr r r rÃLszCondorDAGNode.add_input_macrocCs| d|¡dS)z‰ Add a variable (macro) for storing the output files associated with this node. @param filename: filename of output file Z macrooutputN)rÈ)rrBr r r rÄTszCondorDAGNode.add_output_macrocCs| d|¡dS)NZmacrocheckpoint)rÈ)rrBr r r rÅ\sz"CondorDAGNode.add_checkpoint_macrocCs|jS)z± Return the opts for this node. Note that this returns only the options for this instance of the node and not those associated with the underlying job template. )r¡)rr r r rQ_szCondorDAGNode.get_optscCs,|j d|¡}||jd|<|j |¡dS)a Add a variable (macro) condor command for this node. If the command specified does not exist in the CondorJob, it is added so the submit file will be correct. PLEASE NOTE: AS with other add_var commands, the variable must be set for all nodes that use the CondorJob instance. @param command: command name @param value: Value of the command for this node in the DAG. rrƒN)r¦rr r™r†)rr…r>rƒr r r r†gs z CondorDAGNode.add_var_condor_cmdFcCs.|j d|¡}||jd|<|j ||¡dS)a Add a variable (macro) option for this node. If the option specified does not exist in the CondorJob, it is added so the submit file will be correct when written. @param opt: option name. @param value: value of the option for this node in the DAG. rrƒN)r¦rr¡r™r„)rrMr>r‚rƒr r r r„uszCondorDAGNode.add_var_optcCs*| ||¡|r| |¡n | |¡dS)aê Add a variable (macro) option for this node. If the option specified does not exist in the CondorJob, it is added so the submit file will be correct when written. The value of the option is also added to the list of input files. @param opt: option name. @param value: value of the option for this node in the DAG. @param file_is_output_file: A boolean if the file will be an output file instead of an input file. The default is to have it be an input. N)r„rDrC)rrMrBZfile_is_output_filer r r rPs  zCondorDAGNode.add_file_optcCs0|j |¡|jj|j|d|jd7_dS)zâ Add a variable (or macro) argument to the condor job. The argument is added to the submit file and a different value of the argument can be set for each node in the DAG. @param arg: name of option to add. )r‰rrN)r¢rAr™rŠr£)rrIr‰r r r rŠs zCondorDAGNode.add_var_argcCs| |¡| |¡dS)a- Add a variable (or macro) file name argument to the condor job. The argument is added to the submit file and a different value of the argument can be set for each node in the DAG. The file name is also added to the list of input files. @param filename: name of option to add. N)rCrŠ)rrBr r r rK›s zCondorDAGNode.add_file_argcCs|jS)z¸ Return the arguments for this node. Note that this returns only the arguments for this instance of the node and not those associated with the underlying job template. )r¢)rr r r rL¦szCondorDAGNode.get_argscCs ||_dS)zz Set the number of times that this node in the DAG should retry. @param retry: number of times to retry node. N)r¤)rÚretryr r r Ú set_retry®szCondorDAGNode.set_retrycCs|jS)z} Return the number of times that this node in the DAG should retry. @param retry: number of times to retry node. )r¤)rr r r Ú get_retryµszCondorDAGNode.get_retrycCsžt| ¡tƒrP| d d|j|j ¡g¡¡| ¡ ¡rn| d| ¡ ¡¡n| d|jd|j ¡¡| d¡| d|jdt |j ƒd¡dS)zy Write the DAG entry for this node's job to the DAG file descriptor. @param fh: descriptor of open DAG file. rizSUBDAG EXTERNALz DIR zJOB rhzRETRY N) r—r¯rŒrkÚjoinr˜r™rer“rUr¤)rÚfhr r r Ú write_job¼s  zCondorDAGNode.write_jobcCs"| d|jd|jd¡dS)z~ Write the DAG entry for this node's category to the DAG file descriptor. @param fh: descriptor of open DAG file. z CATEGORY rirhN)rkr˜rš)rrÍr r r Úwrite_categoryÎszCondorDAGNode.write_categorycCs"| d|jd|jd¡dS)z~ Write the DAG entry for this node's priority to the DAG file descriptor. @param fh: descriptor of open DAG file. z PRIORITY rirhN)rkr˜r›)rrÍr r r Úwrite_priorityÕszCondorDAGNode.write_prioritycCsöt|j ¡ƒs"t|j ¡ƒs"|jr2| d|j¡x:|j ¡D],}| dt|ƒdt|j|ƒd¡q>Wx:|j ¡D],}| dt|ƒdt|j|ƒd¡qzW|jrèx6t|j ƒD](}| dt|ƒd|j|d¡q¼W| d¡dS)z† Write the variable (macro) options and arguments to the DAG file descriptor. @param fh: descriptor of open DAG file. zVARS riz="ú"z macroargumentrhN) rlr rmr¡r¢rkr˜rUÚranger£)rrÍÚkÚir r r Ú write_varsÜs",,(zCondorDAGNode.write_varscCs6x0|jD]&}| dt|ƒdt|ƒd¡qWdS)z Write the parent/child relations for this job to the DAG file descriptor. @param fh: descriptor of open DAG file. zPARENT z CHILD rhN)r¥rkrU)rrÍÚparentr r r Ú write_parentsís zCondorDAGNode.write_parentscCs:|jr6| dt|ƒd|jdd |j¡d¡dS)zg Write the pre script for the job, if there is one @param fh: descriptor of open DAG file. z SCRIPT PRE rirhN)rœrkrUrÌr)rrÍr r r Úwrite_pre_scriptõszCondorDAGNode.write_pre_scriptcCs:|jr6| dt|ƒd|jdd |j¡d¡dS)zh Write the post script for the job, if there is one @param fh: descriptor of open DAG file. z SCRIPT POST rirhN)ržrkrUrÌrŸ)rrÍr r r Úwrite_post_scriptþszCondorDAGNode.write_post_scriptcCs(x"|jD]}| d|j|f¡qWdS)zŠ Write as a comment into the DAG file the list of input files for this DAG node. @param fh: descriptor of open DAG file. z!## Job %s requires input file %s N)r¨rkr˜)rrÍÚfr r r Úwrite_input_filess zCondorDAGNode.write_input_filescCs(x"|jD]}| d|j|f¡qWdS)z‹ Write as a comment into the DAG file the list of output files for this DAG node. @param fh: descriptor of open DAG file. z### Job %s generates output file %s N)r§rkr˜)rrÍrÚr r r Úwrite_output_filess z CondorDAGNode.write_output_filescCs|j |¡dS)zh Set the Condor log file to be used by this CondorJob. @param log: path of Condor log file. N)r™r])rÚlogr r r r]szCondorDAGNode.set_log_filecCs&t|ttfƒstdƒ‚|j |¡dS)z¥ Add a parent to this node. This node will not be executed until the parent node has run sucessfully. @param node: CondorDAGNode to add as a parent. z4Parent must be a CondorDAGNode or a CondorDAGManNodeN)r—r}r‘rr¥rA)rÚnoder r r Ú add_parent"szCondorDAGNode.add_parentc Csšt d¡}t d¡}| ¡ ¡}| ¡}g}xb|D]Z}||}| |¡}|rv| d¡} || } | d|t| ƒf¡q2| d|t|ƒf¡q2W| ¡ ¡}xb|D]Z}||}| |¡}|ræ| d¡} || } | d|t| ƒf¡q¢| d|t|ƒf¡q¢W| ¡  ¡} |  ¡}x€| D]x} |  | ¡}|r~t |  | ¡dƒ} y| d|| df¡Wn t k rz| d¡YnXn| d| df¡qW|S) zE Return a list of tuples containg the command line arguments z \$\((.+)\)z\d+rrz--%sz-%srz%sr)ryrzr¯rQÚmatchÚgrouprArUrSrLÚsearchrªÚfindallr‡)rÚpatZargpatrTÚmacrosÚcmd_listrÓÚvalÚmÚkeyr>rr±rˆr r r Úget_cmd_tuple_list,sD             z CondorDAGNode.get_cmd_tuple_listcCs0d}| ¡}x|D]}|d |¡d7}qW|S)z] Return the full command line that will be used when this node is run by DAGman. rri)rêrÌ)rr=ræÚargumentr r r Ú get_cmd_linees  zCondorDAGNode.get_cmd_linecCsdS)zÉ The finalize method of a node is called before the node is finally added to the DAG and can be overridden to do any last minute clean up (such as setting extra command line arguments) Nr )rr r r ÚfinalizerszCondorDAGNode.finalizeN)F)F)F)5rrrrrr²r¯r´rµr¶r·r¸r¹r»r¼r¾r¿rÁrÂrCrDrErFrGrHrÆrÈrÃrÄrÅrQr†r„rPrŠrKrLrÊrËrÎrÏrÐrÕr×rØrÙrÛrÜr]rßrêrìrír r r r r}ksd'               9 r}cs@eZdZdZ‡fdd„Zdd„Zdd„Zdd „Zd d „Z‡Z S) r‘zæ Condor DAGMan node class. Appropriate for setting up DAGs to run within a DAG. Adds the user-tag functionality to condor_dagman processes running in the DAG. May also be used to extend dagman-node specific functionality. cs&tt|ƒ |¡d|_g|_d|_dS)z" @job: a CondorDAGNodeJob N)rtr‘rÚ_CondorDAGManNode__user_tagÚ%_CondorDAGManNode__maxjobs_categoriesZ_CondorDAGManNode__cluster_jobs)rr¯)r|r r rszCondorDAGManNode.__init__cCst|ƒ|_dS)zu Set the user tag that is passed to the analysis code. @param user_tag: the user tag to identify the job N)rUrî)rÚusertagr r r Ú set_user_tagŠszCondorDAGManNode.set_user_tagcCs|jS)z$ Returns the usertag string )rî)rr r r Ú get_user_tag‘szCondorDAGManNode.get_user_tagcCs|j t|ƒt|ƒf¡dS)z® Add a category to this DAG called categoryName with a maxjobs of maxJobsNum. @param node: Add (categoryName,maxJobsNum) tuple to CondorDAG.__maxjobs_categories. N)rïrArU)rÚ categoryNameÚ maxJobsNumr r r Úadd_maxjobs_category—sz%CondorDAGManNode.add_maxjobs_categorycCs|jS)zH Return an array of tuples containing (categoryName,maxJobsNum) )rï)rr r r Úget_maxjobs_categoriesžsz'CondorDAGManNode.get_maxjobs_categories) rrrrrrñròrõrör‹r r )r|r r‘{s  r‘c@s€eZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd d „Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„ZdS)Ú CondorDAGa[ A CondorDAG is a Condor Directed Acyclic Graph that describes a collection of Condor jobs and the order in which to run them. All Condor jobs in the DAG must write their Codor logs to the same file. NOTE: The log file must not be on an NFS mounted system as the Condor jobs must be able to get an exclusive file lock on the log file. cCs4||_d|_g|_g|_g|_d|_d|_d|_dS)zW @param log: path to log file which must not be on an NFS mounted file system. Nr)Ú_CondorDAG__log_file_pathÚ_CondorDAG__dag_file_pathÚ_CondorDAG__jobsÚ_CondorDAG__nodesÚ_CondorDAG__maxjobs_categoriesÚ_CondorDAG__integer_node_namesÚ_CondorDAG__node_countÚ_CondorDAG__nodes_finalized)rrÝr r r r­szCondorDAG.__init__cCs|jS)z; Return a list containing all the nodes in the DAG )rû)rr r r Ú get_nodesºszCondorDAG.get_nodescCs|jS)z: Return a list containing all the jobs in the DAG )rú)rr r r Úget_jobsÀszCondorDAG.get_jobscCs d|_dS)z, Use integer node names for the DAG rrN)rý)rr r r Úset_integer_node_namesÆsz CondorDAG.set_integer_node_namescCs|d|_dS)zd Set the name of the file into which the DAG is written. @param path: path to DAG file. z.dagN)rù)rr\r r r Ú set_dag_fileÌszCondorDAG.set_dag_filecCs|jstdƒ‚n|jSdS)z* Return the path to the DAG file. zNo path for DAG fileN)rørrù)rr r r Ú get_dag_fileÓs zCondorDAG.get_dag_filecCs~t|tƒstdƒ‚t| ¡tƒs,| |j¡|j |¡|j rN|  t |j ƒ¡|j d7_ | ¡|j krz|j  | ¡¡dS)ae Add a CondorDAGNode to this DAG. The CondorJob that the node uses is also added to the list of Condor jobs in the DAG so that a list of the submit files needed by the DAG can be maintained. Each unique CondorJob will be added once to prevent duplicate submit files being written. @param node: CondorDAGNode to add to the CondorDAG. z-Nodes must be class CondorDAGNode or subclassrrN)r—r}rr¯rŒr]rørûrArýr»rUrþrú)rrÞr r r Úadd_nodeÜs   zCondorDAG.add_nodecCs|j t|ƒt|ƒf¡dS)z® Add a category to this DAG called categoryName with a maxjobs of maxJobsNum. @param node: Add (categoryName,maxJobsNum) tuple to CondorDAG.__maxjobs_categories. N)rürArU)rrórôr r r rõïszCondorDAG.add_maxjobs_categorycCs|jS)zH Return an array of tuples containing (categoryName,maxJobsNum) )rü)rr r r röösz CondorDAG.get_maxjobs_categoriescCs.| dt|dƒdt|dƒd¡dS)a Write the DAG entry for this category's maxjobs to the DAG file descriptor. @param fh: descriptor of open DAG file. @param category: tuple containing type of jobs to set a maxjobs limit for and the maximum number of jobs of that type to run at once. zMAXJOBS rrirrrhN)rkrU)rrÍr½r r r Ú write_maxjobsüszCondorDAG.write_maxjobscCs:|jsx|jD] }| ¡qWx|jD] }| ¡q&WdS)z„ Write all the submit files used by the dag to disk. Each submit file is written to the file name set in the CondorJob. N)rÿrûrírúrp)rrÞr¯r r r Úwrite_sub_filess    zCondorDAG.write_sub_filescCsì|jstdƒ‚yt|jdƒ}Wntd|jƒ‚YnXxn|jD]d}| |¡| |¡| ¡rl| |¡| ¡r~|  |¡|  |¡|  |¡|  |¡|  |¡qBWx|jD]}| |¡q²Wx|jD]}| ||¡qÌW| ¡dS)z9 Write all the nodes in the DAG to the DAG file. zNo path for DAG filerfzCannot open file N)rùrrjrûrÎrÕr¿rÏrÂrÐrØrÙrÛrÜr×rürrn)rZdagfilerÞr½r r r Úwrite_concrete_dags,          zCondorDAG.write_concrete_dagcCs*|jsx|jD] }| ¡qW| ¡dS)z Write a dag. N)rÿrûrír)rrÞr r r Ú write_dag+s  zCondorDAG.write_dagcCsä|jstdƒ‚y0|j}d | d¡dd…¡d}t|dƒ}Wntd|jƒ‚YnXx`|jD]V}| d| ¡¡t|t ƒrš| d |  ¡  ¡¡qb| d |  ¡  ¡|  ¡f¡qbW| ¡t |t |¡d tjB¡dS) zµ Write the workflow to a script (.sh instead of .dag). Assuming that parents were added to the DAG before their children, dependencies should be handled correctly. zNo path for DAG fileÚ.Néÿÿÿÿz.shrfzCannot open file z # Job %s zcondor_submit_dag %s z%s %s r)rùrrÌÚsplitrjrûrkr¼r—r‘r¯r”r3rìrnÚosÚchmodÚstatÚS_IEXEC)rZdfpZ outfilenameÚoutfilerÞr r r Ú write_script4s   zCondorDAG.write_scriptN)rrrrrrrrrrrrõrörrrr rr r r r r÷¥s     r÷c@s0eZdZdZdd„Zdd„Zdd„Zdd „Zd S) Ú AnalysisJobz] Describes a generic analysis job that filters LIGO data as configured by an ini file. cCs:||_yt|j dd¡ƒ ¡|_Wnd|_YnXdS)zV @param cp: ConfigParser object that contains the configuration for this job. ÚinputÚchannelN)Ú_AnalysisJob__cprUrVrWÚ_AnalysisJob__channel)rrXr r r rVs zAnalysisJob.__init__cCst|j ||¡ƒ ¡S)z¥ Get the configration variable in a particular section of this jobs ini file. @param sec: ini file section. @param opt: option from section sec. )rUrrVrW)rÚsecrMr r r Ú get_config`szAnalysisJob.get_configcCs ||_dS)z€ Set the name of the channel that this job is filtering. This will overwrite the value obtained at initialization. N)r)rrr r r Ú set_channeliszAnalysisJob.set_channelcCs|jS)zß Returns the name of the channel that this job is filtering. Note that channel is defined to be IFO independent, so this may be LSC-AS_Q or IOO-MC_F. The IFO is set on a per node basis, not a per job basis. )r)rr r r rpszAnalysisJob.channelN)rrrrrrrrr r r r rQs   rc@seZdZdZdd„Zd>dd„Zdd„Zd?d d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dd„Zd@dd„Zdd„ZdAdd„Zdd „ZdBd!d"„Zd#d$„ZdCd%d&„Zd'd(„Zd)d*„Zd+d,„ZdDd-d.„Zd/d0„ZdEd1d2„Zd3d4„Zd5d6„Zd7d8„Zd9d:„Zd;d<„Z d=S)FÚ AnalysisNodezc Contains the methods that allow an object to be built to analyse LIGO data in a Condor DAG. cCstd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ t  d¡|_| ¡ ¡ dd¡|_dS)NrZH2zuser-tag)Ú_AnalysisNode__startÚ_AnalysisNode__endÚ_AnalysisNode__data_startÚ_AnalysisNode__pad_dataÚ_AnalysisNode__data_endÚ_AnalysisNode__trig_startÚ_AnalysisNode__trig_endÚ_AnalysisNode__ifoÚ_AnalysisNode__ifo_tagÚ_AnalysisNode__inputÚ_AnalysisNode__outputÚ_AnalysisNode__calibrationÚ _AnalysisNode__calibration_cacheryrzÚ_AnalysisNode__LHO2kr¯rQrVÚ_AnalysisNode__user_tag)rr r r r~s zAnalysisNode.__init__TcCs |r| d|¡||_||_dS)zî Set the GPS start time of the analysis node by setting a --gps-start-time option to the node when it is executed. @param time: GPS start time of job. @bool pass_to_command_line: add gps-start-time as variable option. zgps-start-timeN)r„rr)rr«Úpass_to_command_liner r r Ú set_starts zAnalysisNode.set_startcCs|jS)z- Get the GPS start time of the node. )r)rr r r Ú get_startszAnalysisNode.get_startcCs |r| d|¡||_||_dS)zæ Set the GPS end time of the analysis node by setting a --gps-end-time option to the node when it is executed. @param time: GPS end time of job. @bool pass_to_command_line: add gps-end-time as variable option. z gps-end-timeN)r„rr )rr«r+r r r Úset_end£s zAnalysisNode.set_endcCs|jS)z+ Get the GPS end time of the node. )r)rr r r Úget_end¯szAnalysisNode.get_endcCs ||_dS)zr Set the GPS start time of the data needed by this analysis node. @param time: GPS start time of job. N)r)rr«r r r Úset_data_startµszAnalysisNode.set_data_startcCs|jS)zA Get the GPS start time of the data needed by this node. )r)rr r r Úget_data_start¼szAnalysisNode.get_data_startcCs ||_dS)zr Set the GPS start time of the data needed by this analysis node. @param time: GPS start time of job. N)r)rÚpadr r r Ú set_pad_dataÂszAnalysisNode.set_pad_datacCs|jS)zA Get the GPS start time of the data needed by this node. )r)rr r r Ú get_pad_dataÉszAnalysisNode.get_pad_datacCs ||_dS)zn Set the GPS end time of the data needed by this analysis node. @param time: GPS end time of job. N)r )rr«r r r Ú set_data_endÏszAnalysisNode.set_data_endcCs|jS)z? Get the GPS end time of the data needed by this node. )r )rr r r Ú get_data_endÖszAnalysisNode.get_data_endcCs|r| d|¡||_dS)zô Set the trig start time of the analysis node by setting a --trig-start-time option to the node when it is executed. @param time: trig start time of job. @bool pass_to_command_line: add trig-start-time as a variable option. ztrig-start-timeN)r„r!)rr«r+r r r Úset_trig_startÜs zAnalysisNode.set_trig_startcCs|jS)z. Get the trig start time of the node. )r!)rr r r Úget_trig_startçszAnalysisNode.get_trig_startcCs|r| d|¡||_dS)zì Set the trig end time of the analysis node by setting a --trig-end-time option to the node when it is executed. @param time: trig end time of job. @bool pass_to_command_line: add trig-end-time as a variable option. z trig-end-timeN)r„r")rr«r+r r r Ú set_trig_endís zAnalysisNode.set_trig_endcCs|jS)z, Get the trig end time of the node. )r")rr r r Ú get_trig_endøszAnalysisNode.get_trig_endcCs$||_|r| d|¡| |¡dS)zµ Add an input to the node by adding a --input option. @param filename: option argument to pass as input. @bool pass_to_command_line: add input as a variable option. rN)r%r„rC)rrBr+r r r Ú set_inputþs zAnalysisNode.set_inputcCs|jS)z4 Get the file that will be passed as input. )r%)rr r r Ú get_input szAnalysisNode.get_inputcCs$||_|r| d|¡| |¡dS)z¹ Add an output to the node by adding a --output option. @param filename: option argument to pass as output. @bool pass_to_command_line: add output as a variable option. ÚoutputN)r&r„rD)rrBr+r r r Ú set_outputs zAnalysisNode.set_outputcCs|jS)z5 Get the file that will be passed as output. )r&)rr r r Ú get_outputszAnalysisNode.get_outputcCs2||_| ¡ ¡r.| d|d| ¡ ¡¡dS)a Set the ifo name to analyze. If the channel name for the job is defined, then the name of the ifo is prepended to the channel name obtained from the job configuration file and passed with a --channel-name option. @param ifo: two letter ifo code (e.g. L1, H1 or H2). z channel-nameú:N)r#r¯rr„)rZifor r r Úset_ifo s zAnalysisNode.set_ifocCs|jS)z8 Returns the two letter IFO code for this node. )r#)rr r r Úget_ifo+szAnalysisNode.get_ifocCs||_|r| d|¡dS)zº Set the ifo tag that is passed to the analysis code. @param ifo_tag: a string to identify one or more IFOs @bool pass_to_command_line: add ifo-tag as a variable option. zifo-tagN)r$r„)rZifo_tagr+r r r Ú set_ifo_tag1szAnalysisNode.set_ifo_tagcCs|jS)z$ Returns the IFO tag string )r$)rr r r Ú get_ifo_tag;szAnalysisNode.get_ifo_tagcCs||_|r| d|¡dS)z¸ Set the user tag that is passed to the analysis code. @param user_tag: the user tag to identify the job @bool pass_to_command_line: add user-tag as a variable option. zuser-tagN)r*r„)rrðr+r r r rñAszAnalysisNode.set_user_tagcCs|jS)z$ Returns the usertag string )r*)rr r r ròKszAnalysisNode.get_user_tagc Csòt|tƒr"| d|¡| |¡nÌt|tƒræ| dd¡t|ƒdkrLtdƒ‚xŠ|D]‚}| d¡d d¡\}}}}t|ƒ}t|ƒt|ƒ}||  ¡|  ¡t|ƒdkrR||  ¡|  ¡t|ƒdkrR| |¡qRW| d |¡ntd ƒ‚d S) z¤ Set the LAL frame cache to to use. The frame cache is passed to the job with the --frame-cache argument. @param filename: calibration file to use. z frame-cachezglob-frame-datarirz?LDR did not return any LFNs for query: check ifo and frame typer ú-rrz frame-typezUnknown LFN cache formatN) r—rUr„rCrlÚlenrr rªr6r4r1) rrBÚlfnr±ÚbroÚdZt_startZt_endr r r Ú set_cacheQs"         zAnalysisNode.set_cachecCs¸|jr¨|jdkr¨| ¡ dd¡}|j |j¡r€|jdkr€|jdkr€|jt| ¡ dd¡ƒkrn| ¡ dd¡}q’| ¡ dd¡}n| ¡ d|j¡}tj  ||¡}||_ n d }t |ƒ‚d S) zJ Determine the path to the correct calibration cache file to use. rÚ calibrationr\i Õw+i}Å+zH2-cal-epoch-boundaryzH2-1zH2-2z$IFO and start-time must be set firstN) r#rr¯rr)ràrªr r\rÌr(r)rZcal_pathZcal_fileÚcalÚmsgr r r Úcalibration_cache_pathosz#AnalysisNode.calibration_cache_pathcCs.| ¡| d|j¡|j|_| |j¡dS)zË Set the path to the calibration cache file for the given IFO. During S2 the Hanford 2km IFO had two calibration epochs, so if the start time is during S2, we use the correct cache file. zcalibration-cacheN)rNr„r(r'rC)rr r r rKˆszAnalysisNode.calibrationcCs|jS)zF Return the calibration cache file to be used by the DAG. )r()rr r r Úget_calibration–szAnalysisNode.get_calibrationN)T)T)T)T)T)T)T)T)!rrrrrr,r-r.r/r0r1r3r4r5r6r7r8r9r:r;r<r>r?rArBrCrDrñròrJrNrKrOr r r r rys<         rc@sbeZdZdZddd„Zdd„Zdd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dS)Ú AnalysisChunkzm An AnalysisChunk is the unit of data that a node works with, usually some subset of a ScienceSegment. rcCs&||_||_|||_||_||_dS)zè @param start: GPS start time of the chunk. @param end: GPS end time of the chunk. @param trig_start: GPS time at which to start generating triggers @param trig_end: GPS time at which to stop generating triggers N)Ú_AnalysisChunk__startÚ_AnalysisChunk__endÚ_AnalysisChunk__lengthÚ_AnalysisChunk__trig_startÚ_AnalysisChunk__trig_end)rÚstartÚendÚ trig_startÚtrig_endr r r r¤s  zAnalysisChunk.__init__cCsx|jr$|jr$d|j|j|j|jfS|jrD|jsDd|j|j|jfS|jsd|jrdd|j|j|jfSd|j|jfSdS)Nz=z0z.z!)rTrUrQrR)rr r r r²±s   zAnalysisChunk.__repr__cCsx|jr|jr|j|j}n@|jr4|js4|j|j}n&|jsN|jrN|j|j}n |j|j}|dkrpt|dƒ‚n|SdS)zi Returns the length of data for which this AnalysisChunk will produce triggers (in seconds). rzhas negative lengthN)rTrUrRrQr)rÚxr r r Ú__len__¾s    zAnalysisChunk.__len__cCs|jS)z2 Returns the GPS start time of the chunk. )rQ)rr r r rVÑszAnalysisChunk.startcCs|jS)z0 Returns the GPS end time of the chunk. )rR)rr r r rW×szAnalysisChunk.endcCs|jS)z@ Returns the length (duration) of the chunk in seconds. )rS)rr r r ÚdurÝszAnalysisChunk.durcCs|jS)z] Return the first GPS time at which triggers for this chunk should be generated. )rT)rr r r rXãszAnalysisChunk.trig_startcCs|jS)z\ Return the last GPS time at which triggers for this chunk should be generated. )rU)rr r r rYêszAnalysisChunk.trig_endcCs ||_dS)zZ Set the first GPS time at which triggers for this chunk should be generated. N)rT)rrVr r r r7ñszAnalysisChunk.set_trig_startcCs ||_dS)zY Set the last GPS time at which triggers for this chunk should be generated. N)rU)rrWr r r r9øszAnalysisChunk.set_trig_endN)rr)rrrrrr²r[rVrWr\rXrYr7r9r r r r rPŸs  rPc@sœeZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd&d d„Z d'dd„Z dd„Z dd„Z dd„Z dd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%S)(ÚScienceSegmentaP A ScienceSegment is a period of time where the experimenters determine that the inteferometer is in a state where the data is suitable for scientific analysis. A science segment can have a list of AnalysisChunks asscociated with it that break the segment up into (possibly overlapping) smaller time intervals for analysis. cCsH|d|_|d|_|d|_|d|_g|_| ¡|_d|_d|_dS)zy @param segment: a tuple containing the (segment id, gps start time, gps end time, duration) of the segment. rrrééN) Ú_ScienceSegment__idÚ_ScienceSegment__startÚ_ScienceSegment__endÚ_ScienceSegment__durÚ_ScienceSegment__chunksr\Ú_ScienceSegment__unusedZ_ScienceSegment__ifoÚ_ScienceSegment__df_node)rÚsegmentr r r r s     zScienceSegment.__init__cCs|dkrtdƒ‚|j|S)zm Allows iteration over and direct access to the AnalysisChunks contained in this ScienceSegment. rzlist index out of range)r‡rd)rrÔr r r Ú __getitem__szScienceSegment.__getitem__cCs t|jƒS)zP Returns the number of AnalysisChunks contained in this ScienceSegment. )rFrd)rr r r r[szScienceSegment.__len__cCs$d| ¡| ¡| ¡| ¡|jfS)Nz<)ÚidrVrWr\re)rr r r r²%szScienceSegment.__repr__cCst| ¡| ¡ƒS)zL ScienceSegments are compared by the GPS start time of the segment. )ÚcmprV)rÚotherr r r Ú__cmp__)szScienceSegment.__cmp__rcCs>| ¡d|}| ¡|}||} x||kr.||} |rj|r| ||ddd|d|kr|dkr ddt | ||dd¡} | d} d} d}| d|krÒtdddtd ƒt d ¡n | |krât| ƒ} | | kròt| ƒ}|j  t || | |ƒ¡n|j  t || ƒ¡|| 7}|| 8}q(W|||_ d S) a| Divides the science segment into chunks of length seconds overlapped by overlap seconds. If the play option is set, only chunks that contain S2 playground data are generated. If the user has a more complicated way of generating chunks, this method should be overriden in a sub-class. Any data at the end of the ScienceSegment that is too short to contain a chunk is ignored. The length of this unused data is stored and can be retrieved with the unused() method. @param length: length of chunk in seconds. @param overlap: overlap between chunks in seconds. @param play: 1 : only generate chunks that overlap with S2 playground data. 2 : as play = 1 plus compute trig start and end times to coincide with the start/end of the playground @param sl: slide by sl seconds before determining playground data. @param excl_play: exclude the first excl_play second from the start and end of the chunk when computing if the chunk overlaps with playground. @param pad_data: exclude the first and last pad_data seconds of the segment when generating chunks r^i Õw+iâiXrz&Two playground segments in this chunk:ri)rWz3 Code to handle this case has not been implementedrrN) r\rVÚmathÚfloorÚprintÚsysÚexitrªrdrArPre)rÚlengthÚoverlapÚplayÚslÚ excl_playÚpad_dataZ time_leftrVÚ incrementrWÚ play_startÚplay_endrXrYr r r Ú make_chunks/s4      zScienceSegment.make_chunkscCs|j t||||ƒ¡dS)zß Add an AnalysisChunk to the list associated with this ScienceSegment. @param start: GPS start time of chunk. @param end: GPS end time of chunk. @param trig_start: GPS start time for triggers from chunk N)rdrArP)rrVrWrXrYr r r Ú add_chunkaszScienceSegment.add_chunkcCs|jS)zT Returns the length of data in the science segment not used to make chunks. )re)rr r r ÚunusedjszScienceSegment.unusedcCs ||_dS)zP Set the length of data in the science segment not used to make chunks. N)re)rr}r r r Ú set_unusedpszScienceSegment.set_unusedcCs|jS)z0 Returns the ID of this ScienceSegment. )r`)rr r r rivszScienceSegment.idcCs|jS)z< Returns the GPS start time of this ScienceSegment. )ra)rr r r rV|szScienceSegment.startcCs|jS)z: Returns the GPS end time of this ScienceSegment. )rb)rr r r rW‚szScienceSegment.endcCs|j|j|7_||_dS)zv Override the GPS start time (and set the duration) of this ScienceSegment. @param t: new GPS start time. N)rcra)rr r r r r,ˆszScienceSegment.set_startcCs|j|j|8_||_dS)zr Override the GPS end time (and set the duration) of this ScienceSegment. @param t: new GPS end time. N)rcrb)rr r r r r.szScienceSegment.set_endcCs|jS)zJ Returns the length (duration) in seconds of this ScienceSegment. )rc)rr r r r\˜szScienceSegment.durcCs ||_dS)zŽ Set the DataFind node associated with this ScienceSegment to df_node. @param df_node: the DataFind node for this ScienceSegment. N)rf)rZdf_noder r r Ú set_df_nodežszScienceSegment.set_df_nodecCs|jS)z< Returns the DataFind node for this ScienceSegment. )rf)rr r r Ú get_df_node¥szScienceSegment.get_df_nodeN)rrrrrr)rr)rrrrrrhr[r²rlr{r|r}r~rirVrWr,r.r\rr€r r r r r]s$ 2 r]c@s²eZdZdZdd„Zdd„Zdd„Zdd „Zd*d d „Zd d„Z dd„Z d+dd„Z d,dd„Z d-dd„Z d.dd„Zdd„Zdd„Zdd„Zdd „Zd!d"„Zd#d$„Zd%d&„Zd'd(„Zd)S)/Ú ScienceDatazÔ An object that can contain all the science data used in an analysis. Can contain multiple ScienceSegments and has a method to generate these from a text file produces by the LIGOtools segwizard program. cCsg|_d|_dS)N)Ú_ScienceData__sci_segsÚ_ScienceData__filename)rr r r r²szScienceData.__init__cCs |j|S)zl Allows direct access to or iteration over the ScienceSegments associated with the ScienceData. )r‚)rrÔr r r rh¶szScienceData.__getitem__cCs d|jS)Nz)rƒ)rr r r r²½szScienceData.__repr__cCs t|jƒS)zP Returns the number of ScienceSegments associated with the ScienceData. )rFr‚)rr r r r[ÀszScienceData.__len__rc CsÔ||_t d¡}x¾t|ƒD]²}| |¡st| ¡dƒ|krttt| ¡ƒƒ\}}} } |dkrh||7}n|dkrx| |7} | t |ƒ8} |dkr¬||7}| |8} | dt |ƒ8} t t ||| | gƒƒ} |j   | ¡qWdS)aõ Parse the science segments from the segwizard output contained in file. @param filename: input text file containing a list of science segments generated by segwizard. @param min_length: only append science segments that are longer than min_length. @param slide_sec: Slide each ScienceSegment by:: delta > 0: [s,e] -> [s+delta,e]. delta < 0: [s,e] -> [s,e-delta]. @param buffer: shrink the ScienceSegment:: [s,e] -> [s+buffer,e-buffer] z\A#r_rr^N)rƒryrzrjràrªr rlÚmapÚabsr]Útupler‚rA) rrBÚ min_lengthZ slide_secÚbufferZ octothorpeÚlineriÚstÚenZdurZr r r ÚreadÆs    zScienceData.readcCst|ƒ}|j |¡dS)N)r]r‚rA)rZ seg_tuplerZr r r Úappend_from_tupleíszScienceData.append_from_tuplec Cs„||_xxt|ƒD]l}| ¡}t|dƒ}tt t|dƒ¡ƒ}tt t|dƒ¡ƒ}||}tt ||||gƒƒ}|j   |¡qWdS)z´ Parse the science segments from a tama list of locked segments contained in file. @param filename: input text file containing a list of tama segments. rr_éN) rƒrjr rªrmÚceilÚfloatrnr]r†r‚rA) rrBr‰ÚcolumnsrirVrWr\rZr r r Ú tama_readñs zScienceData.tama_readc Cs(x"|jD]}| ||||||¡qWdS)aæ Divide each ScienceSegment contained in this object into AnalysisChunks. @param length: length of chunk in seconds. @param overlap: overlap between segments. @param play: if true, only generate chunks that overlap with S2 playground data. @param sl: slide by sl seconds before determining playground data. @param excl_play: exclude the first excl_play second from the start and end of the chunk when computing if the chunk overlaps with playground. N)r‚r{)rrrrsrtrurvrwÚsegr r r r{s zScienceData.make_chunkscCs6x.|jD]"}| ¡|kr | ¡|} | |} |r`|r$| ||ddd|d|kr$| | ¡|} |dkrddt | ||dd¡} | d} d}| d| krÐtdƒtdƒt d¡nD| | kràt| ƒ} | | krðt| ƒ}|dks|| kr$|  | | | |¡n|  | | | ¡|  d¡q Wd S) a€ Create an extra chunk that uses up the unused data in the science segment. @param length: length of chunk in seconds. @param trig_overlap: length of time start generating triggers before the start of the unused data. @param play: - 1 : only generate chunks that overlap with S2 playground data. - 2 : as 1 plus compute trig start and end times to coincide with the start/end of the playground @param min_length: the unused data must be greater than min_length to make a chunk. @param sl: slide by sl seconds before determining playground data. @param excl_play: exclude the first excl_play second from the start and end of the chunk when computing if the chunk overlaps with playground. @param pad_data: exclude the first and last pad_data seconds of the segment when generating chunks i Õw+iâiXr^rz%Two playground segments in this chunkz3 Code to handle this case has not been implementedrrN) r‚r}rWrmrnrorprqrªr|r~)rrrZ trig_overlaprtr‡rurvrwr“rWrVrXryrzrYr r r Úmake_chunks_from_unuseds0     z#ScienceData.make_chunks_from_unusedc Csˆx‚|jD]x}| ¡|kr| ¡| ¡|}| ¡}||} |rh|rv|||ddd| d|krv| |||¡| d¡qWdS)a Create a chunk that uses up the unused data in the science segment @param min_length: the unused data must be greater than min_length to make a chunk. @param overlap: overlap between chunks in seconds. @param play: if true, only generate chunks that overlap with S2 playground data. @param sl: slide by sl seconds before determining playground data. @param excl_play: exclude the first excl_play second from the start and end of the chunk when computing if the chunk overlaps with playground. i Õw+iâiXr^rN)r‚r}rWr|r~) rr‡rsrtrurvr“rVrWrrr r r Úmake_short_chunks_from_unusedBs  z)ScienceData.make_short_chunks_from_unusedc CsÞxØ|jD]Î}| ¡|}| ¡|}| ¡|kr²|||}x4t|dƒD]$}|||} | |} | | | ¡qJW||d|} | |d} | | | ¡| | |¡| d¡q| ¡|krÌ| ||¡q| d¡qWdS)ap Splits ScienceSegments up into chunks, of a given maximum length. The length of the last two chunks are chosen so that the data utilisation is optimised. @param min_length: minimum chunk length. @param max_length: maximum chunk length. @param pad_data: exclude the first and last pad_data seconds of the segment when generating chunks rrr^rN)r‚rVrWr}rÒr|r~) rr‡Ú max_lengthrwr“Z seg_startZseg_endÚNrÔrVÚstopÚmiddler r r Úmake_optimised_chunksXs"           z!ScienceData.make_optimised_chunkscCsôt|ƒ}t|ƒ}d}g}d}d}d}xÀ|D]¸} |  ¡} |  ¡} |  ¡} xš|| krà|| kr¨| |krf|}n| }| |krx|} n| } tt| || | |gƒƒ}| |¡|| kr¨P|d7}|t|ƒkrÖ||}| ¡}| ¡}qHd}d}qHWq*W||_t|ƒS)a Replaces the ScienceSegments contained in this instance of ScienceData with the intersection of those in the instance other. Returns the number of segments in the intersection. @param other: ScienceData to use to generate the intersection r rri”5w)rFrVrWrir]r†rAr‚)rrkÚlength1Úlength2ÚostartÚoutlistZiseg2Ústart2Ústop2Zseg1Ústart1Ústop1riÚostoprZZseg2r r r Ú intersection~s@       zScienceData.intersectioncCsŠt|ƒ}t|ƒ}d}g}d}d}d}d} d} x |dkrx|d7}||krn|| ¡}|| ¡} || ¡} n ||krxP| dkr´|d7}||krª|| ¡} || ¡} n ||kr´P|dkrÚ| dksÌ|| krÚ|} | }d}n| dkrð| } | }d} nP|dkr| }|}q0| |kr"||krL|}nq0tt| ||||gƒƒ}| |¡| }|}q0W|dkr|tt| ||||gƒƒ}| |¡||_t|ƒS)zý Replaces the ScienceSegments contained in this instance of ScienceData with the union of those in the instance other. Returns the number of ScienceSegments in the union. @param other: ScienceData to use to generate the intersection r rr)rFrVrWrir]r†rAr‚)rrkr›rœrÚseglistÚi1Úi2r¡rŸrir¢r ZustartZustopr£rZr r r Úunion¼sd          zScienceData.unionc CsÈt|ƒdkrdS|j ¡g}d}xl|D]d}| ¡}| ¡}| ¡}||kr€|dkrvtt|||||gƒƒ}| |¡|}|}q(||kr(|}q(W|dkrºtt|||||gƒƒ}| |¡||_t|ƒS)zr Coalesces any adjacent ScienceSegments. Returns the number of ScienceSegments in the coalesced list. rr ) rFr‚ÚsortrVrWrir]r†rA) rržr£r“rVr˜rirrZr r r Úcoalesce s,     zScienceData.coalescecCsÎt|ƒdkr"ttddddgƒƒ|_g}d}xj|D]b}| ¡}| ¡}|dks\||ks\||krdtdƒ‚|dkrŽttd||||gƒƒ}| |¡|}q0W|dkrÀttd|dd|gƒƒ}| |¡||_t|ƒS)z~ Inverts the ScienceSegments in the class (i.e. set NOT). Returns the number of ScienceSegments after inversion. riÿ“5wz Invalid list)rFr]r†r‚rVrWrrA)rržrr“rVr˜rZr r r Úinvert: s$    zScienceData.invertcCsÔt|ƒ}d}g}d}d}d}x¨|D] }| ¡}| ¡} | ¡} ||dt||||ƒ} xb| | krÀ| |krv| }n|}| |} | | kr| } n| } tt| || | |gƒƒ}| |¡| |} q`Wq"W||_t|ƒS)zH Keep only times in ScienceSegments which are in the playground r i Õw+iâiXrr) rFrVrWrirªr]r†rAr‚)rrrrržZbegin_s2Z play_spaceZplay_lenr“rVr˜riryZ play_stopr£rZr r r rtZ s2   zScienceData.playcCs$| |¡| |¡| ¡t|ƒS)zl Intersection routine for three inputs. Built out of the intersect, coalesce and play routines )r¤rªrF)rÚsecondÚthirdr r r Ú intersect_3Š s  zScienceData.intersect_3cCs.| |¡| |¡| |¡| ¡t|ƒS)z0 Intersection routine for four inputs. )r¤rªrF)rr¬r­Zfourthr r r Ú intersect_4” s    zScienceData.intersect_4c Csžg}xŠ|D]‚}| ¡}| ¡}| ¡}xd||krŠ||}||krF|}n|||krbt||dƒ}tt|||||gƒƒ}| |¡|}q(Wq W||_t|ƒS)zP Split the segments in the list is subsegments at least as long as dt r^) rVrWrirªr]r†rAr‚rF) rÚdtržr“rVr˜riZtmpstoprZr r r r ž s      zScienceData.splitN)rr)rrrrr)rrrrr)rrrr)r)rrrrrrhr²r[rŒrr’r{r”r•ršr¤r¨rªr«rtr®r¯r r r r r r¬s* '  1  &>U) 0  rc@s.eZdZdd„Zdd„Zd dd„Zdd „ZdS) Ú LsyncCachecCs6||_ddddœ|_x|j ¡D]}i|j|<q WdS)N)ÚgwfZsftÚxml)Ú_LsyncCache__pathÚcacherm)rr\Útyper r r r¹ szLsyncCache.__init__cstj‡‡fdd„tˆƒDƒŽS)zZ Group an iterable into an n-tuples iterable. Incomplete tuples are discarded csg|]}t ˆ|dˆ¡‘qS)N)Ú itertoolsÚislice)Ú.0rÔ)ÚlstÚnr r ú Ë sz$LsyncCache.group..)r·ÚiziprÒ)rrºr»r )rºr»r ráÆ szLsyncCache.groupNcCs8|j}|j}|rt |¡}nd}t|dƒ}i}xô|D]ì}|rL| |¡dkrLq4| ¡ dd¡\}} } } | d¡\} } }}}t|ƒ}dd„| dd … d¡Dƒ} d d„|  | d ¡Dƒ}| |krÂi|| <||| krÚi|| |<| |f}||| |krd t |ƒ}t |ƒ‚t j  |¡|| ||<q4W| ¡||d <dS)ay Each line of the frame cache file is like the following: /frames/E13/LHO/frames/hoftMon_H1/H-H1_DMT_C00_L2-9246,H,H1_DMT_C00_L2,1,16 1240664820 6231 {924600000 924646720 924646784 924647472 924647712 924700000} The description is as follows: 1.1) Directory path of files 1.2) Site 1.3) Type 1.4) Number of frames in the files (assumed to be 1) 1.5) Duration of the frame files. 2) UNIX timestamp for directory modification time. 3) Number of files that that match the above pattern in the directory. 4) List of time range or segments [start, stop) We store the cache for each site and frameType combination as a dictionary where the keys are (directory, duration) tuples and the values are segment lists. Since the cache file is already coalesced we do not have to call the coalesce method on the segment lists. Nr°rir_ú,cSsg|] }t|ƒ‘qSr )rª)r¹Úsr r r r¼ sz$LsyncCache.parse..rrr cSsg|]}tj |¡‘qSr )ÚligoÚsegmentsrg)r¹r±r r r r¼ sr^z8The combination %s is not unique in the frame cache filer²)r´rµryrzrjrârWr rªrárUÚ RuntimeErrorrÀrÁÚ segmentlistrn)rZ type_regexr\rµZ type_filterrÚZgwfDictr‰ÚheaderZmodTimeZ fileCountÚtimesrÚsiteÚ frameTypeZ frameCountÚdurationrÁrérMr r r ÚparseÍ s6      zLsyncCache.parsec Cs|j}||dkrgS||d|kr*gStj ||¡}tj |g¡}i}x¢|d|| ¡D]Š\} } | \} } |  |¡} | s€q`xh| D]`}| |¡r†|\}}t||| ƒ}x<|D]4}| tj ||| ¡¡r®d|||| f}d||<q®Wq†Wq`Wt|  ¡ƒ}|  ¡|S)z r²z%s-%s-%d-%d.gwfN) rµrÀrÁrgrÃÚitemsZ intersectsrÒrlrmr©)rrÆrÇZgpsStartZgpsEndrµrâZ searchlistZlfnDictrér¥rr\rsr¿Út1Út2rÅr rGZlfnsr r r Úget_lfns s0       zLsyncCache.get_lfns)N)rrrrrárÉrÍr r r r r±¸ s  Or±c@s2eZdZdZd dd„Zdd„Zdd„Zd d „ZdS) ÚLSCDataFindJoba· An LSCdataFind job used to locate data. The static options are read from the section [datafind] in the ini file. The stdout from LSCdataFind contains the paths to the frame files and is directed to a file in the cache directory named by site and GPS start and end times. The stderr is directed to the logs directory. The job always runs in the scheduler universe. The path to the executable is determined from the ini file. Nc Cs,| dd¡|_d|_t ||j|j¡t ||¡||_||_d|_|  dd¡rd|  d| dd¡¡|r~t |ƒ|_|j  |¡xR|j  d¡D]B}t|ƒ ¡}|dd…dkrŒt|j d|¡ƒ ¡}| ||¡qŒW| dd ¡| d d ¡|  d d ¡| tj |d¡¡| tj |d¡¡| d¡dS)aT @param cache_dir: the directory to write the output lal cache files to. @param log_dir: the directory to write the stderr file to. @param config_file: ConfigParser object containing the path to the LSCdataFind executable in the [condor] section and a [datafind] section from which the LSCdataFind options are read. ÚcondorÚdatafindÚlocalNÚaccounting_grouprŽr¶z lal-cacherzurl-typeÚfileÚgetenvÚTruezkdatafind-$(macroobservatory)-$(macrotype)-$(macrogpsstarttime)-$(macrogpsendtime)-$(cluster)-$(process).errzkdatafind-$(macroobservatory)-$(macrotype)-$(macrogpsstarttime)-$(macrogpsendtime)-$(cluster)-$(process).outz datafind.sub)rVZ_LSCDataFindJob__executableZ_LSCDataFindJob__universerqrrÚ_LSCDataFindJob__cache_dirÚ_LSCDataFindJob__config_fileÚ_LSCDataFindJob__lsync_cacheÚ has_optionr?r±rÉrTrUrWrNr`r r\rÌrbrd) rÚ cache_dirÚlog_dirÚ config_fileZlsync_cache_fileZlsync_type_regexÚorMrIr r r rY s.        zLSCDataFindJob.__init__cCs|jS)zD returns the directroy that the cache files are written to. )rÖ)rr r r Ú get_cache_dir szLSCDataFindJob.get_cache_dircCs|jS)z. return the configuration file object )r×)rr r r Úget_config_file… szLSCDataFindJob.get_config_filecCs|jS)N)rØ)rr r r Ú lsync_cache‹ szLSCDataFindJob.lsync_cache)NN)rrrrrrÞrßràr r r r rÎP s  &rÎc@sreZdZdZdd„Zdd„Zddd„Zd d „Zd d „Zd d„Z dd„Z dd„Z dd„Z dd„Z dd„Zdd„ZdS)ÚLSCDataFindNodezE A DataFindNode runs an instance of LSCdataFind in a Condor DAG. cCspt ||¡t |¡d|_d|_d|_d|_||_d|_y|  |  ¡  ¡  dd¡¡Wnd|_ YnXdS)zM @param job: A CondorDAGJob that can run an instance of LALdataFind. rNrÐr¶)r}rrÚ_LSCDataFindNode__startÚ_LSCDataFindNode__endÚ_LSCDataFindNode__observatoryÚ_LSCDataFindNode__outputÚ_LSCDataFindNode__jobZ_LSCDataFindNode__lfn_listÚset_typer¯rßrVÚ_LSCDataFindNode__type)rr¯r r r r“ s  zLSCDataFindNode.__init__cCst|jrp|jrp|jrp|jrptj |j ¡|jd|jddt |jƒdt |j|jƒd¡|_ |  |j ¡dS)z… Private method to set the file to write the cache to. Automaticaly set once the ifo, start and end times have been set. rEZ_CACHEz.lcfN) rârãrärèr r\rÌrærÞrUrår>)rr r r Z __set_output¦ sLzLSCDataFindNode.__set_outputNcCs@|r| dt|ƒt|ƒ¡n| dt|ƒ¡||_| ¡dS)z] Set the start time of the datafind query. @param time: GPS start time of query. zgps-start-timeN)r„rªrâÚ_LSCDataFindNode__set_output)rr«r2r r r r,¯ s zLSCDataFindNode.set_startcCs|jS)z5 Return the start time of the datafind query )râ)rr r r r-» szLSCDataFindNode.get_startcCs| d|¡||_| ¡dS)zY Set the end time of the datafind query. @param time: GPS end time of query. z gps-end-timeN)r„rãré)rr«r r r r.Á s zLSCDataFindNode.set_endcCs|jS)z5 Return the start time of the datafind query )rã)rr r r r/Ê szLSCDataFindNode.get_endcCs"| d|¡t|ƒ|_| ¡dS)a! Set the IFO to retrieve data for. Since the data from both Hanford interferometers is stored in the same frame file, this takes the first letter of the IFO (e.g. L or H) and passes it to the --observatory option of LSCdataFind. @param obs: IFO to obtain data for. Z observatoryN)r„rUräré)rZobsr r r Úset_observatoryÐ s  zLSCDataFindNode.set_observatorycCs|jS)z5 Return the start time of the datafind query )rä)rr r r Úget_observatoryÜ szLSCDataFindNode.get_observatorycCs&| dt|ƒ¡t|ƒ|_| ¡dS)z2 sets the frame type that we are querying r¶N)r„rUrèré)rr¶r r r rçâ s zLSCDataFindNode.set_typecCs|jS)z2 gets the frame type that we are querying )rè)rr r r Úget_typeê szLSCDataFindNode.get_typecCs|jS)N)rå)rr r r Úget_output_cacheð sz LSCDataFindNode.get_output_cachecCs|jS)zP Return the output file, i.e. the file containing the frame cache data. )rå)rr r r r?ó szLSCDataFindNode.get_output)N)rrrrrrér,r-r.r/rêrërçrìrír?r r r r rá s   rác@seZdZdZdd„ZdS)Ú LigolwAddJobzG A ligolw_add job can be used to concatenate several ligo lw files cCs | dd¡|_d|_t ||j|j¡t ||¡| |d¡| dd¡| dd¡rj| d| dd¡¡|  t j   |d¡¡|  t j   |d¡¡| d ¡d S) z? cp = ConfigParser object from which options are read. rÏZ ligolw_addÚvanillarÔrÕrÒz$ligolw_add-$(cluster)-$(process).outz$ligolw_add-$(cluster)-$(process).errzligolw_add.subN)rVZ_LigolwAddJob__executableZ_LigolwAddJob__universerqrrrZr?rÙrbr r\rÌr`rd)rrÛrXr r r rþ s    zLigolwAddJob.__init__N)rrrrrr r r r rîú srîc@seZdZdZdd„ZdS)Ú LigolwAddNodez5 Runs an instance of ligolw_add in a Condor DAG. cCst ||¡t |¡dS)zK @param job: A CondorDAGJob that can run an instance of ligolw_add N)r}rr)rr¯r r r r s zLigolwAddNode.__init__N)rrrrrr r r r rð srðc@seZdZdZdd„ZdS)Ú LigolwCutJobzD A ligolw_cut job can be used to remove parts of a ligo lw file cCst| dd¡|_d|_t ||j|j¡t ||¡| dd¡| tj   |d¡¡|  tj   |d¡¡|  d¡d S) z? cp = ConfigParser object from which options are read. rÏZ ligolw_cutrïrÔrÕz$ligolw_cut-$(cluster)-$(process).outz$ligolw_cut-$(cluster)-$(process).errzligolw_cut.subN) rVZ_LigolwCutJob__executableZ_LigolwCutJob__universerqrrr?rbr r\rÌr`rd)rrÛrXr r r r! s  zLigolwCutJob.__init__N)rrrrrr r r r rñ srñc@seZdZdZdd„ZdS)Ú LigolwCutNodez5 Runs an instance of ligolw_cut in a Condor DAG. cCst ||¡t |¡dS)zK @param job: A CondorDAGJob that can run an instance of ligolw_cut N)r}rr)rr¯r r r r5 s zLigolwCutNode.__init__N)rrrrrr r r r rò1 sròc@seZdZdZdd„ZdS)ÚNoopJobz A Noop Job does nothing. cCs˜d|_d|_t ||j|j¡t ||¡| dd¡| dd¡| dd¡rb| d| dd¡¡| t j   |d¡¡|  t j   |d ¡¡|  d ¡d S) z? cp = ConfigParser object from which options are read. ÚtruerÑrÔrÕZnoop_jobrÏrÒznoop-$(cluster)-$(process).outznoop-$(cluster)-$(process).errznoop.subN)Z_NoopJob__executableZ_NoopJob__universerqrrr?rÙrVrbr r\rÌr`rd)rrÛrXr r r rA s    zNoopJob.__init__N)rrrrrr r r r ró= sróc@seZdZdZdd„ZdS)ÚNoopNodez& Run an noop job in a Condor DAG. cCs8t ||¡t |¡d|_d|_d|_d|_d|_dS)z7 @param job: A CondorDAGJob that does nothing. N)r}rrZ_NoopNode__serverZ_NoopNode__identityZ_NoopNode__insertZ_NoopNode__pfnZ_NoopNode__query)rr¯r r r rX s  zNoopNode.__init__N)rrrrrr r r r rõT srõc@s(eZdZdZdd„Zdd„Zdd„ZdS) Ú SqliteJoba~ A cbc sqlite job adds to CondorDAGJob and AnalysisJob features common to jobs which read or write to a sqlite database. Of note, the universe is always set to local regardless of what's in the cp file, the extension is set to None so that it may be set by individual SqliteNodes, log files do not have macrogpsstarttime and endtime in them, and get_env is set to True. cCs¾||_| d|¡}d}t |||¡t ||¡x8|D]0}| |¡rR| ||¡q6tj  d|d¡q6W|  dd¡|  dd¡r–|  d| dd¡¡|  d|d ¡|  d|d ¡d S) zª @cp: a ConfigParser object from which options are read @sections: list of sections in cp to get added options @exec_name: the name of the sql executable rÏrïz)warning: config file is missing section [z] rÔrÕrÒzlogs/z-$(cluster)-$(process).outz-$(cluster)-$(process).errN)Ú_SqliteJob__exec_namerVrqrrÚ has_sectionrZrpÚstderrrkr?rÙrbr`)rrXÚsectionsÚ exec_namer1r0rr r r rm s      zSqliteJob.__init__cCs ||_dS)z Set the exec_name name N)r÷)rrûr r r Ú set_exec_name… szSqliteJob.set_exec_namecCs|jS)z Get the exec_name name )r÷)rr r r Ú get_exec_name‹ szSqliteJob.get_exec_nameN)rrrrrrürýr r r r röe sröc@s8eZdZdZdd„Zdd„Zdd„Zdd „Zd d „Zd S) Ú SqliteNodezÇ A cbc sqlite node adds to the standard AnalysisNode features common to nodes which read or write to a sqlite database. Specifically, it adds the set_tmp_space_path and set_database methods. cCs&t ||¡t |¡d|_d|_dS)z @job: an Sqlite job N)r}rrÚ_SqliteNode__tmp_spaceÚ_SqliteNode__database)rr¯r r r r˜ s  zSqliteNode.__init__cCs| d|¡||_dS)z? Sets temp-space path. This should be on a local disk. z tmp-spaceN)r„rÿ)rZ tmp_spacer r r Ú set_tmp_space¡ s zSqliteNode.set_tmp_spacecCs|jS)z Gets tmp-space path. )rÿ)rr r r Ú get_tmp_space¨ szSqliteNode.get_tmp_spacecCs| d|¡||_dS)z Sets database option. ÚdatabaseN)rPr)rrr r r Ú set_database® s zSqliteNode.set_databasecCs|jS)z Gets database option. )r)rr r r Ú get_databaseµ szSqliteNode.get_databaseN) rrrrrrrrrr r r r rþ’ s  rþcs(eZdZdZ‡fdd„Zdd„Z‡ZS)ÚLigolwSqliteJobzi A LigolwSqlite job. The static options are read from the section [ligolw_sqlite] in the ini file. cs"d}dg}tt|ƒ |||¡dS)z? @cp: ConfigParser object from which options are read. Z ligolw_sqliteN)rtrr)rrXrûrú)r|r r rÁ szLigolwSqliteJob.__init__cCs| dd¡dS)zy Sets the --replace option. This will cause the job to overwrite existing databases rather than add to them. ÚreplacerN)rN)rr r r Ú set_replaceÉ szLigolwSqliteJob.set_replace)rrrrrrr‹r r )r|r r¼ s rcsHeZdZdZ‡fdd„Zdd„Zdd„Zdd „Zd d „Zd d „Z ‡Z S)ÚLigolwSqliteNodez A LigolwSqlite node. cs&tt|ƒ |¡d|_d|_d|_dS)z! @job: a LigolwSqliteJob N)rtr rÚ_LigolwSqliteNode__input_cacheÚ_LigolwSqliteNode__xml_outputZ_LigolwSqliteNode__xml_input)rr¯)r|r r rÕ szLigolwSqliteNode.__init__cCs| d|¡||_dS)z Sets input cache. z input-cacheN)rPr )rZ input_cacher r r Úset_input_cacheÞ s z LigolwSqliteNode.set_input_cachecCs|jS)z Gets input cache. )r )rr r r Úget_input_cacheå sz LigolwSqliteNode.get_input_cachecCs| |¡dS)z. Sets xml input file instead of cache N)rŠ)rÚxml_filer r r Ú set_xml_inputë szLigolwSqliteNode.set_xml_inputcCs*| ¡dkrtdƒ‚| d|¡||_dS)zL Tell ligolw_sqlite to dump the contents of the database to a file. Nzno database specifiedÚextract)rÚ ValueErrorrPr )rrr r r Úset_xml_outputñ s  zLigolwSqliteNode.set_xml_outputcCs(|jr |jS| ¡r| ¡Stdƒ‚dS)zx Override standard get_output to return xml-file if xml-file is specified. Otherwise, will return database. z(no output xml file or database specifiedN)r rr)rr r r r?ú s zLigolwSqliteNode.get_output) rrrrrr r rrr?r‹r r )r|r r Ñ s  r c@seZdZdZdd„ZdS)ÚDeepCopyableConfigParserz² The standard SafeConfigParser no longer supports deepcopy() as of python 2.7 (see http://bugs.python.org/issue16058). This subclass restores that functionality. cCs0tƒ}| |¡| d¡| ¡}| |¡|S)Nr)r rkÚseekr|Úreadfp)rÚmemoZ config_stringZ new_configr r r Ú __deepcopy__ s    z%DeepCopyableConfigParser.__deepcopy__N)rrrrrr r r r r sr)FrÚ __future__rÚ __author__ZgluerÚdateÚ__date__riÚ __version__r rpryr«r¬rmZ six.movesrrÚsocketr·Z ligo.segmentsrÀÚhashlibrZcjsonrÚ ImportErrorÚjsonrZsix.moves.http_clientÚsixrr r Ú ExceptionrrrrrrrÚobjectrrqrŒr}r‘r÷rrrPr]rr±rÎrárîrðrñròrórõrörþrr ÚSafeConfigParserrr r r r Ús–     HA*-((b,?k  -*5