2015, 2016, 2017, Alexander Nitz, Ian Harry, Christopher M. Biwer, Duncan A.  Brown, Josh Willis, and Tito Dal Canton

Calculating the Effectualness (Fitting Factor) of Template Banks

Pycbc’s workflow module configuration file(s) and command line interface

Pycbc's workflow module configuration file(s) and command line interface

Antenna Patterns and Projecting a Signal into the Detector Frame

Options to the coincidence clustering + statistic asignment executable

Supplying the config file on the command line and overriding options

Supported inspiral trigger generators and instructions for using them

Configuration files - Are you running from production configuration (.ini) files?

Supported splittable executables and instructions for using them

Workflow: the inspiral analysis workflow generator (pycbc.workflow)

xi2_from_mass1_mass2_spin2x_spin2y() (in module pycbc.conversions)

spin1z_from_mass1_mass2_chi_eff_chi_a() (in module pycbc.conversions)

spin2z_from_mass1_mass2_chi_eff_chi_a() (in module pycbc.conversions)

spin2x_from_mass1_mass2_xi2_phi_a_phi_s() (in module pycbc.conversions)

spin2y_from_mass1_mass2_xi2_phi_a_phi_s() (in module pycbc.conversions)

cartesian_to_spherical_azimuthal() (in module pycbc.coordinates)

cosmological_quantity_from_redshift() (in module pycbc.cosmology)

light_travel_time_to_detector() (pycbc.detector.Detector method)

mass1_mass2_spin1z_spin2z_to_beta_sigma_gamma() (in module pycbc.pnutils)

compute_search_efficiency_in_bins() (in module pycbc.sensitivity)

inverse (pycbc.transforms.AlignedMassSpinToCartesianSpin attribute)

inverse_transform() (pycbc.transforms.AlignedMassSpinToCartesianSpin method)

name (pycbc.transforms.AlignedMassSpinToCartesianSpin attribute)

transform() (pycbc.transforms.AlignedMassSpinToCartesianSpin method)

name (pycbc.transforms.CartesianSpin1ToSphericalSpin1 attribute)

name (pycbc.transforms.CartesianSpin2ToSphericalSpin2 attribute)

inverse (pycbc.transforms.CartesianSpinToAlignedMassSpin attribute)

inverse_jacobian() (pycbc.transforms.CartesianSpinToAlignedMassSpin method)

inverse_transform() (pycbc.transforms.CartesianSpinToAlignedMassSpin method)

jacobian() (pycbc.transforms.CartesianSpinToAlignedMassSpin method)

name (pycbc.transforms.CartesianSpinToAlignedMassSpin attribute)

transform() (pycbc.transforms.CartesianSpinToAlignedMassSpin method)

inverse (pycbc.transforms.CartesianSpinToPrecessionMassSpin attribute)

inverse_jacobian() (pycbc.transforms.CartesianSpinToPrecessionMassSpin method)

inverse_transform() (pycbc.transforms.CartesianSpinToPrecessionMassSpin method)

jacobian() (pycbc.transforms.CartesianSpinToPrecessionMassSpin method)

name (pycbc.transforms.CartesianSpinToPrecessionMassSpin attribute)

transform() (pycbc.transforms.CartesianSpinToPrecessionMassSpin method)

inverse_jacobian() (pycbc.transforms.CartesianToSpherical method)

inverse_transform() (pycbc.transforms.CartesianToSpherical method)

inverse_jacobian() (pycbc.transforms.ChiPToCartesianSpin method)

inverse_transform() (pycbc.transforms.ChiPToCartesianSpin method)

inverse_jacobian() (pycbc.transforms.ChirpDistanceToDistance method)

inverse_transform() (pycbc.transforms.ChirpDistanceToDistance method)

inverse_jacobian() (pycbc.transforms.DistanceToChirpDistance method)

inverse_transform() (pycbc.transforms.DistanceToChirpDistance method)

distance (pycbc.transforms.LambdaFromMultipleTOVFiles attribute)

from_config() (pycbc.transforms.LambdaFromMultipleTOVFiles class method)

lambda_param (pycbc.transforms.LambdaFromMultipleTOVFiles attribute)

map_file (pycbc.transforms.LambdaFromMultipleTOVFiles attribute)

mass_param (pycbc.transforms.LambdaFromMultipleTOVFiles attribute)

transform() (pycbc.transforms.LambdaFromMultipleTOVFiles method)

lambda_from_tov_data() (pycbc.transforms.LambdaFromTOVFile static method)

inverse_jacobian() (pycbc.transforms.Mass1Mass2ToMchirpEta method)

inverse_transform() (pycbc.transforms.Mass1Mass2ToMchirpEta method)

inverse_jacobian() (pycbc.transforms.Mass1Mass2ToMchirpQ method)

inverse_transform() (pycbc.transforms.Mass1Mass2ToMchirpQ method)

inverse_jacobian() (pycbc.transforms.MchirpEtaToMass1Mass2 method)

inverse_transform() (pycbc.transforms.MchirpEtaToMass1Mass2 method)

inverse_jacobian() (pycbc.transforms.MchirpQToMass1Mass2 method)

inverse_transform() (pycbc.transforms.MchirpQToMass1Mass2 method)

inverse (pycbc.transforms.PrecessionMassSpinToCartesianSpin attribute)

inverse_transform() (pycbc.transforms.PrecessionMassSpinToCartesianSpin method)

name (pycbc.transforms.PrecessionMassSpinToCartesianSpin attribute)

transform() (pycbc.transforms.PrecessionMassSpinToCartesianSpin method)

inverse (pycbc.transforms.SphericalSpin1ToCartesianSpin1 attribute)

name (pycbc.transforms.SphericalSpin1ToCartesianSpin1 attribute)

inverse (pycbc.transforms.SphericalSpin2ToCartesianSpin2 attribute)

name (pycbc.transforms.SphericalSpin2ToCartesianSpin2 attribute)

inverse_transform() (pycbc.transforms.SphericalToCartesian method)

apply_boundary_conditions() (pycbc.distributions.angular.UniformAngle method)

from_config() (pycbc.distributions.angular.UniformAngle class method)

bounds (pycbc.distributions.angular.UniformSolidAngle attribute)

params (pycbc.distributions.angular.UniformSolidAngle attribute)

polar_angle (pycbc.distributions.angular.UniformSolidAngle attribute)

azimuthal_angle (pycbc.distributions.angular.UniformSolidAngle attribute)

from_config() (pycbc.distributions.angular.UniformSolidAngle class method)

from_config() (pycbc.distributions.arbitrary.Arbitrary class method)

get_kde_from_arrays() (pycbc.distributions.arbitrary.Arbitrary static method)

set_bandwidth() (pycbc.distributions.arbitrary.Arbitrary method)

from_config() (pycbc.distributions.arbitrary.FromFile class method)

get_arrays_from_file() (pycbc.distributions.arbitrary.FromFile method)

apply_boundary_conditions() (pycbc.distributions.bounded.BoundedDist method)

from_config() (pycbc.distributions.bounded.BoundedDist class method)

get_param_bounds_from_config() (in module pycbc.distributions.bounded)

name (pycbc.distributions.constraints.SupernovaeConvexHull attribute)

apply_boundary_conditions() (pycbc.distributions.external.External method)

from_config() (pycbc.distributions.external.External class method)

from_config() (pycbc.distributions.fixedsamples.FixedSamples class method)

from_config() (pycbc.distributions.gaussian.Gaussian class method)

variable_args (pycbc.distributions.joint.JointDistribution attribute)

distributions (pycbc.distributions.joint.JointDistribution attribute)

constraints (pycbc.distributions.joint.JointDistribution attribute)

apply_boundary_conditions() (pycbc.distributions.joint.JointDistribution method)

well_reflected (pycbc.distributions.joint.JointDistribution attribute)

name (pycbc.distributions.mass.MchirpfromUniformMass1Mass2 attribute)

params (pycbc.distributions.mass.MchirpfromUniformMass1Mass2 attribute)

bounds (pycbc.distributions.mass.MchirpfromUniformMass1Mass2 attribute)

norm (pycbc.distributions.mass.MchirpfromUniformMass1Mass2 attribute)

lognorm (pycbc.distributions.mass.MchirpfromUniformMass1Mass2 attribute)

name (pycbc.distributions.mass.QfromUniformMass1Mass2 attribute)

params (pycbc.distributions.mass.QfromUniformMass1Mass2 attribute)

bounds (pycbc.distributions.mass.QfromUniformMass1Mass2 attribute)

norm (pycbc.distributions.mass.QfromUniformMass1Mass2 attribute)

lognorm (pycbc.distributions.mass.QfromUniformMass1Mass2 attribute)

from_config() (pycbc.distributions.mass.QfromUniformMass1Mass2 class method)

params (pycbc.distributions.power_law.UniformPowerLaw attribute)

bounds (pycbc.distributions.power_law.UniformPowerLaw attribute)

lognorm (pycbc.distributions.power_law.UniformPowerLaw attribute)

from_config() (pycbc.distributions.power_law.UniformPowerLaw class method)

from_config() (pycbc.distributions.qnm.UniformF0Tau class method)

from_config() (pycbc.distributions.spins.IndependentChiPChiEff class method)

name (pycbc.distributions.spins.IndependentChiPChiEff attribute)

from_config() (pycbc.distributions.uniform.Uniform class method)

read_distributions_from_config() (in module pycbc.distributions)

LiveCoincTimeslideBackgroundEstimator (class in pycbc.events.coinc)

add_singles() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator method)

backout_last() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator method)

from_cli() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator class method)

ifar() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator method)

insert_args() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator static method)

pick_best_coinc() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator class method)

restore_state() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator static method)

save_state() (pycbc.events.coinc.LiveCoincTimeslideBackgroundEstimator method)

multiifo_noise_coincident_area() (in module pycbc.events.coinc_rate)

multiifo_signal_coincident_area() (in module pycbc.events.coinc_rate)

findchirp_cluster_over_window() (in module pycbc.events.eventmgr)

add_template_events() (pycbc.events.eventmgr.EventManager method)

add_template_params() (pycbc.events.eventmgr.EventManager method)

cluster_template_events() (pycbc.events.eventmgr.EventManager method)

consolidate_events() (pycbc.events.eventmgr.EventManager method)

finalize_template_events() (pycbc.events.eventmgr.EventManager method)

from_multi_ifo_interface() (pycbc.events.eventmgr.EventManager class method)

keep_loudest_in_interval() (pycbc.events.eventmgr.EventManager method)

keep_near_injection() (pycbc.events.eventmgr.EventManager method)

restore_state() (pycbc.events.eventmgr.EventManager static method)

finalize_template_events() (pycbc.events.eventmgr.EventManagerMultiDet method)

write_events() (pycbc.events.eventmgr.EventManagerMultiDet method)

write_to_hdf() (pycbc.events.eventmgr.EventManagerMultiDet method)

finalize_template_events() (pycbc.events.eventmgr.EventManagerCoherent method)

write_to_hdf() (pycbc.events.eventmgr.EventManagerCoherent method)

findchirp_cluster_over_window_cython (in module pycbc.events.eventmgr_cython)

get_newsnr_sgveto_psdvar_scaled() (in module pycbc.events.ranking)

get_newsnr_sgveto_psdvar_scaled_threshold() (in module pycbc.events.ranking)

get_newsnr_sgveto_psdvar_threshold() (in module pycbc.events.ranking)

newsnr_sgveto_psdvar_scaled_threshold() (in module pycbc.events.ranking)

newsnr_sgveto_psdvar_threshold() (in module pycbc.events.ranking)

parallel_thresh_cluster() (in module pycbc.events.simd_threshold_cython)

parallel_threshold() (in module pycbc.events.simd_threshold_cython)

coinc_lim_for_thresh() (pycbc.events.stat.ExpFitCombinedSNR method)

coinc_lim_for_thresh() (pycbc.events.stat.ExpFitSGBgRateStatistic method)

rank_stat_coinc() (pycbc.events.stat.ExpFitSGBgRateStatistic method)

reassign_rate() (pycbc.events.stat.ExpFitSGBgRateStatistic method)

assign_median_sigma() (pycbc.events.stat.ExpFitSGFgBgNormStatistic method)

coinc_lim_for_thresh() (pycbc.events.stat.ExpFitSGFgBgNormStatistic method)

lognoiserate() (pycbc.events.stat.ExpFitSGFgBgNormStatistic method)

rank_stat_coinc() (pycbc.events.stat.ExpFitSGFgBgNormStatistic method)

rank_stat_single() (pycbc.events.stat.ExpFitSGFgBgNormStatistic method)

logsignalrate() (pycbc.events.stat.ExpFitSGPSDFgBgNormBBHStatistic method)

single() (pycbc.events.stat.ExpFitSGPSDFgBgNormBBHStatistic method)

coinc_lim_for_thresh() (pycbc.events.stat.ExpFitStatistic method)

coinc_lim_for_thresh_OLD() (pycbc.events.stat.ExpFitStatistic method)

coinc_lim_for_thresh() (pycbc.events.stat.PhaseTDExpFitStatistic method)

coinc_lim_for_thresh_OLD() (pycbc.events.stat.PhaseTDExpFitStatistic method)

rank_stat_coinc() (pycbc.events.stat.PhaseTDExpFitStatistic method)

rank_stat_single() (pycbc.events.stat.PhaseTDExpFitStatistic method)

coinc_lim_for_thresh() (pycbc.events.stat.PhaseTDStatistic method)

coinc_lim_for_thresh() (pycbc.events.stat.QuadratureSumStatistic method)

rank_stat_coinc() (pycbc.events.stat.QuadratureSumStatistic method)

rank_stat_single() (pycbc.events.stat.QuadratureSumStatistic method)

threshold_and_cluster() (pycbc.events.threshold_cpu.CPUThresholdCluster method)

insert_bank_bins_option_group() (in module pycbc.events.triggers)

combine_results() (pycbc.filter.matchedfilter.LiveBatchMatchedFilter method)

process_all() (pycbc.filter.matchedfilter.LiveBatchMatchedFilter method)

process_data() (pycbc.filter.matchedfilter.LiveBatchMatchedFilter method)

set_data() (pycbc.filter.matchedfilter.LiveBatchMatchedFilter method)

MatchedFilterSkyMaxControl (class in pycbc.filter.matchedfilter)

MatchedFilterSkyMaxControlNoPhase (class in pycbc.filter.matchedfilter)

compute_max_snr_over_sky_loc_stat_no_phase() (in module pycbc.filter.matchedfilter)

compute_max_snr_over_sky_loc_stat() (in module pycbc.filter.matchedfilter)

compute_followup_snr_series() (in module pycbc.filter.matchedfilter)

compute_u_val_for_sky_loc_stat_no_phase() (in module pycbc.filter.matchedfilter)

compute_u_val_for_sky_loc_stat() (in module pycbc.filter.matchedfilter)

followup_event_significance() (in module pycbc.filter.matchedfilter)

correlate (pycbc.filter.matchedfilter_cpu.CPUCorrelator attribute)

batch_correlate_execute() (in module pycbc.filter.matchedfilter_cpu)

interpolate_complex_frequency() (in module pycbc.filter.resample)

ccorrf_parallel() (in module pycbc.filter.simd_correlate_cython)

available_tests (pycbc.inference.burn_in.BaseBurnInTests attribute)

burn_in_index() (pycbc.inference.burn_in.BaseBurnInTests method)

from_config() (pycbc.inference.burn_in.BaseBurnInTests class method)

max_posterior() (pycbc.inference.burn_in.BaseBurnInTests method)

min_iterations() (pycbc.inference.burn_in.BaseBurnInTests method)

posterior_step() (pycbc.inference.burn_in.BaseBurnInTests method)

available_tests (pycbc.inference.burn_in.EnsembleMCMCBurnInTests attribute)

burn_in_index() (pycbc.inference.burn_in.EnsembleMCMCBurnInTests method)

evaluate() (pycbc.inference.burn_in.EnsembleMCMCBurnInTests method)

ks_test() (pycbc.inference.burn_in.EnsembleMCMCBurnInTests method)

max_posterior() (pycbc.inference.burn_in.EnsembleMCMCBurnInTests method)

posterior_step() (pycbc.inference.burn_in.EnsembleMCMCBurnInTests method)

EnsembleMultiTemperedMCMCBurnInTests (class in pycbc.inference.burn_in)

burn_in_index() (pycbc.inference.burn_in.MCMCBurnInTests method)

max_posterior() (pycbc.inference.burn_in.MCMCBurnInTests method)

posterior_step() (pycbc.inference.burn_in.MCMCBurnInTests method)

arithmetic_mean_estimator() (in module pycbc.inference.evidence)

thermodynamic_integration() (in module pycbc.inference.evidence)

add_density_option_group() (in module pycbc.inference.option_utils)

add_injsamples_map_opt() (in module pycbc.inference.option_utils)

add_plot_posterior_option_group() (in module pycbc.inference.option_utils)

add_scatter_option_group() (in module pycbc.inference.option_utils)

expected_parameters_from_cli() (in module pycbc.inference.option_utils)

config_group (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

copy_info() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

copy_metadata() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

copy_samples() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

data_group (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

effective_nsamples (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

extra_args_parser() (pycbc.inference.io.base_hdf.BaseInferenceFile static method)

get_slice() (pycbc.inference.io.base_hdf.BaseInferenceFile static method)

injections_group (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

log_evidence (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

parse_parameters() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_config_file() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_data() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_injections() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_psds() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_random_state() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_raw_samples() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

read_samples() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

sampler_group (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

samples_from_cli() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

samples_group (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

static_params (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

thin_end (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

thin_interval (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

thin_start (pycbc.inference.io.base_hdf.BaseInferenceFile attribute)

write_command_line() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_config_file() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_data() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_injections() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_kwargs_to_attrs() (pycbc.inference.io.base_hdf.BaseInferenceFile class method)

write_logevidence() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_psd() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_random_state() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_samples() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_stilde() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

write_strain() (pycbc.inference.io.base_hdf.BaseInferenceFile method)

acl (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

act (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

burn_in_index (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

burn_in_iteration (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

extra_args_parser() (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO static method)

is_burned_in (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

iterations() (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO method)

last_iteration() (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO method)

nchains (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

niterations (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

nwalkers (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

raw_acls (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

raw_acts (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

thin() (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO method)

thinned_by (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO attribute)

write_niterations() (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO method)

write_resume_point() (pycbc.inference.io.base_mcmc.CommonMCMCMetadataIO method)

thin_interval (pycbc.inference.io.base_mcmc.EnsembleMCMCMetadataIO attribute)

thin_start (pycbc.inference.io.base_mcmc.EnsembleMCMCMetadataIO attribute)

thin_interval (pycbc.inference.io.base_mcmc.MCMCMetadataIO attribute)

thin_start (pycbc.inference.io.base_mcmc.MCMCMetadataIO attribute)

ensemble_read_raw_samples() (in module pycbc.inference.io.base_mcmc)

thin_samples_for_writing() (in module pycbc.inference.io.base_mcmc)

CommonMultiTemperedMetadataIO (class in pycbc.inference.io.base_multitemper)

extra_args_parser() (pycbc.inference.io.base_multitemper.CommonMultiTemperedMetadataIO static method)

ensemble_read_raw_samples() (in module pycbc.inference.io.base_multitemper)

read_raw_samples() (in module pycbc.inference.io.base_multitemper)

BaseNestedSamplerFile (class in pycbc.inference.io.base_nested_sampler)

name (pycbc.inference.io.base_nested_sampler.BaseNestedSamplerFile attribute)

run_end_time (pycbc.inference.io.base_sampler.BaseSamplerFile attribute)

run_start_time (pycbc.inference.io.base_sampler.BaseSamplerFile attribute)

validate() (pycbc.inference.io.base_sampler.BaseSamplerFile method)

write_resume_point() (pycbc.inference.io.base_sampler.BaseSamplerFile method)

write_run_end_time() (pycbc.inference.io.base_sampler.BaseSamplerFile method)

write_run_start_time() (pycbc.inference.io.base_sampler.BaseSamplerFile method)

extra_args_parser() (pycbc.inference.io.dynesty.CommonNestedMetadataIO static method)

read_raw_samples() (pycbc.inference.io.dynesty.DynestyFile method)

write_raw_samples() (pycbc.inference.io.dynesty.DynestyFile method)

read_acceptance_fraction() (pycbc.inference.io.emcee.EmceeFile method)

write_acceptance_fraction() (pycbc.inference.io.emcee.EmceeFile method)

read_acceptance_fraction() (pycbc.inference.io.emcee_pt.EmceePTFile method)

read_raw_samples() (pycbc.inference.io.emcee_pt.EmceePTFile method)

write_acceptance_fraction() (pycbc.inference.io.emcee_pt.EmceePTFile method)

write_sampler_metadata() (pycbc.inference.io.emcee_pt.EmceePTFile method)

write_samples() (pycbc.inference.io.emcee_pt.EmceePTFile method)

read_acceptance_fraction() (pycbc.inference.io.epsie.EpsieFile method)

read_acceptance_rate() (pycbc.inference.io.epsie.EpsieFile method)

read_acceptance_ratio() (pycbc.inference.io.epsie.EpsieFile method)

write_acceptance_ratio() (pycbc.inference.io.epsie.EpsieFile method)

write_sampler_metadata() (pycbc.inference.io.epsie.EpsieFile method)

write_temperature_data() (pycbc.inference.io.epsie.EpsieFile method)

niterations (pycbc.inference.io.multinest.MultinestFile attribute)

read_raw_samples() (pycbc.inference.io.multinest.MultinestFile method)

write_logevidence() (pycbc.inference.io.multinest.MultinestFile method)

write_niterations() (pycbc.inference.io.multinest.MultinestFile method)

write_resume_point() (pycbc.inference.io.multinest.MultinestFile method)

write_sampler_metadata() (pycbc.inference.io.multinest.MultinestFile method)

write_samples() (pycbc.inference.io.multinest.MultinestFile method)

read_raw_samples() (pycbc.inference.io.posterior.PosteriorFile method)

write_samples() (pycbc.inference.io.posterior.PosteriorFile method)

read_raw_samples_from_file() (in module pycbc.inference.io.posterior)

write_samples_to_file() (in module pycbc.inference.io.posterior)

read_ensemble_attrs() (pycbc.inference.io.ptemcee.PTEmceeFile method)

read_raw_samples() (pycbc.inference.io.ptemcee.PTEmceeFile method)

starting_betas (pycbc.inference.io.ptemcee.PTEmceeFile attribute)

write_ensemble_attrs() (pycbc.inference.io.ptemcee.PTEmceeFile method)

write_sampler_metadata() (pycbc.inference.io.ptemcee.PTEmceeFile method)

add_results_option_group() (pycbc.inference.io.ResultsArgumentParser method)

parse_known_args() (pycbc.inference.io.ResultsArgumentParser method)

from_config() (pycbc.inference.jump.angular.EpsieATAdaptiveAngular class method)

from_config() (pycbc.inference.jump.angular.EpsieAdaptiveAngular class method)

from_config() (pycbc.inference.jump.angular.EpsieAngular class method)

EpsieATAdaptiveBoundedNormal (class in pycbc.inference.jump.bounded_normal)

from_config() (pycbc.inference.jump.bounded_normal.EpsieATAdaptiveBoundedNormal class method)

EpsieAdaptiveBoundedNormal (class in pycbc.inference.jump.bounded_normal)

from_config() (pycbc.inference.jump.bounded_normal.EpsieAdaptiveBoundedNormal class method)

EpsieBoundedNormal (class in pycbc.inference.jump.bounded_normal)

from_config() (pycbc.inference.jump.bounded_normal.EpsieBoundedNormal class method)

EpsieAdaptiveBoundedDiscrete (class in pycbc.inference.jump.discrete)

from_config() (pycbc.inference.jump.discrete.EpsieAdaptiveBoundedDiscrete class method)

EpsieAdaptiveNormalDiscrete (class in pycbc.inference.jump.discrete)

from_config() (pycbc.inference.jump.discrete.EpsieAdaptiveNormalDiscrete class method)

from_config() (pycbc.inference.jump.discrete.EpsieBoundedDiscrete class method)

from_config() (pycbc.inference.jump.discrete.EpsieNormalDiscrete class method)

from_config() (pycbc.inference.jump.normal.EpsieATAdaptiveNormal class method)

from_config() (pycbc.inference.jump.normal.EpsieAdaptiveNormal class method)

from_config() (pycbc.inference.jump.normal.EpsieNormal class method)

epsie_adaptive_from_config() (in module pycbc.inference.jump.normal)

epsie_at_adaptive_from_config() (in module pycbc.inference.jump.normal)

get_epsie_adaptation_settings() (in module pycbc.inference.jump.normal)

get_epsie_discrete_successive_settings() (in module pycbc.inference.jump.normal)

current_params (pycbc.inference.models.base.BaseModel attribute)

extra_args_from_config() (pycbc.inference.models.base.BaseModel static method)

from_config() (pycbc.inference.models.base.BaseModel class method)

get_current_stats() (pycbc.inference.models.base.BaseModel method)

prior_from_config() (pycbc.inference.models.base.BaseModel static method)

sampling_params (pycbc.inference.models.base.BaseModel attribute)

variable_params (pycbc.inference.models.base.BaseModel attribute)

from_config() (pycbc.inference.models.base.SamplingTransforms class method)

logjacobian() (pycbc.inference.models.base.SamplingTransforms method)

read_sampling_params_from_config() (in module pycbc.inference.models.base)

detectors (pycbc.inference.models.base_data.BaseDataModel attribute)

lognl (pycbc.inference.models.base_data.BaseDataModel attribute)

loglr (pycbc.inference.models.base_data.BaseDataModel attribute)

logplr (pycbc.inference.models.base_data.BaseDataModel attribute)

write_metadata() (pycbc.inference.models.base_data.BaseDataModel method)

BruteParallelGaussianMarginalize (class in pycbc.inference.models.brute_marg)

check_validtimes() (in module pycbc.inference.models.data_utils)

create_data_parser() (in module pycbc.inference.models.data_utils)

data_opts_from_config() (in module pycbc.inference.models.data_utils)

detectors_with_valid_data() (in module pycbc.inference.models.data_utils)

fd_data_from_strain_dict() (in module pycbc.inference.models.data_utils)

gate_overwhitened_data() (in module pycbc.inference.models.data_utils)

strain_from_cli_multi_ifos() (in module pycbc.inference.models.data_utils)

BaseGaussianNoise (class in pycbc.inference.models.gaussian_noise)

data (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

kmin (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

kmax (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

psds (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

psd_segments (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

weight (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

normalize (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

lognorm (pycbc.inference.models.gaussian_noise.BaseGaussianNoise attribute)

det_lognl() (pycbc.inference.models.gaussian_noise.BaseGaussianNoise method)

det_lognorm() (pycbc.inference.models.gaussian_noise.BaseGaussianNoise method)

from_config() (pycbc.inference.models.gaussian_noise.BaseGaussianNoise class method)

det_cplx_loglr() (pycbc.inference.models.gaussian_noise.GaussianNoise method)

det_optimal_snrsq() (pycbc.inference.models.gaussian_noise.GaussianNoise method)

name (pycbc.inference.models.gaussian_noise.GaussianNoise attribute)

create_waveform_generator() (in module pycbc.inference.models.gaussian_noise)

get_values_from_injection() (in module pycbc.inference.models.gaussian_noise)

MarginalizedHMPolPhase (class in pycbc.inference.models.marginalized_gaussian_noise)

MarginalizedPhaseGaussianNoise (class in pycbc.inference.models.marginalized_gaussian_noise)

MarginalizedPolarization (class in pycbc.inference.models.marginalized_gaussian_noise)

extra_args_from_config() (pycbc.inference.models.relbin.Relative static method)

write_metadata() (pycbc.inference.models.relbin.Relative method)

likelihood_parts() (in module pycbc.inference.models.relbin_cpu)

likelihood_parts_v() (in module pycbc.inference.models.relbin_cpu)

SingleTemplate (class in pycbc.inference.models.single_template)

name (pycbc.inference.models.single_template.SingleTemplate attribute)

model_stats (pycbc.inference.sampler.base.BaseSampler attribute)

resume_from_checkpoint() (pycbc.inference.sampler.base.BaseSampler method)

sampling_params (pycbc.inference.sampler.base.BaseSampler attribute)

static_params (pycbc.inference.sampler.base.BaseSampler attribute)

variable_params (pycbc.inference.sampler.base.BaseSampler attribute)

create_new_output_file() (in module pycbc.inference.sampler.base)

initial_dist_from_config() (in module pycbc.inference.sampler.base)

log_likelihood() (pycbc.inference.sampler.base_cube.CubeModel method)

prior_transform() (pycbc.inference.sampler.base_cube.CubeModel method)

call_global_loglikelihood() (in module pycbc.inference.sampler.base_cube)

call_global_logprior() (in module pycbc.inference.sampler.base_cube)

niterations (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

checkpoint_interval (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

checkpoint_signal (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

target_niterations (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

target_eff_nsamples (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

thin_interval (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

max_samples_per_chain (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

thin_safety_factor (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

effective_nsamples (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

base_shape (pycbc.inference.sampler.base_mcmc.BaseMCMC attribute)

checkpoint() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

checkpoint_from_config() (pycbc.inference.sampler.base_mcmc.BaseMCMC static method)

ckpt_signal_from_config() (pycbc.inference.sampler.base_mcmc.BaseMCMC static method)

clear_samples() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

compute_acf() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

compute_acl() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

effective_nsamples() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

get_thin_interval() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

resume_from_checkpoint() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

set_burn_in() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

set_burn_in_from_config() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

set_start_from_config() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

set_state_from_file() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

set_target() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

set_target_from_config() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

write_results() (pycbc.inference.sampler.base_mcmc.BaseMCMC method)

acl (pycbc.inference.sampler.base_mcmc.EnsembleSupport attribute)

effective_nsamples (pycbc.inference.sampler.base_mcmc.EnsembleSupport attribute)

nwalkers (pycbc.inference.sampler.base_mcmc.EnsembleSupport attribute)

blob_data_to_dict() (in module pycbc.inference.sampler.base_mcmc)

ensemble_compute_acf() (in module pycbc.inference.sampler.base_mcmc)

ensemble_compute_acl() (in module pycbc.inference.sampler.base_mcmc)

get_optional_arg_from_config() (in module pycbc.inference.sampler.base_mcmc)

raw_samples_to_dict() (in module pycbc.inference.sampler.base_mcmc)

MultiTemperedSupport (class in pycbc.inference.sampler.base_multitemper)

betas_from_config() (pycbc.inference.sampler.base_multitemper.MultiTemperedSupport static method)

ntemps (pycbc.inference.sampler.base_multitemper.MultiTemperedSupport attribute)

acl_from_raw_acls() (in module pycbc.inference.sampler.base_multitemper)

compute_acf() (in module pycbc.inference.sampler.base_multitemper)

compute_acl() (in module pycbc.inference.sampler.base_multitemper)

ensemble_compute_acf() (in module pycbc.inference.sampler.base_multitemper)

ensemble_compute_acl() (in module pycbc.inference.sampler.base_multitemper)

read_betas_from_hdf() (in module pycbc.inference.sampler.base_multitemper)

checkpoint() (pycbc.inference.sampler.dynesty.DynestySampler method)

finalize() (pycbc.inference.sampler.dynesty.DynestySampler method)

from_config() (pycbc.inference.sampler.dynesty.DynestySampler class method)

logz_err (pycbc.inference.sampler.dynesty.DynestySampler attribute)

model_stats (pycbc.inference.sampler.dynesty.DynestySampler attribute)

niterations (pycbc.inference.sampler.dynesty.DynestySampler attribute)

resume_from_checkpoint() (pycbc.inference.sampler.dynesty.DynestySampler method)

samples (pycbc.inference.sampler.dynesty.DynestySampler attribute)

set_initial_conditions() (pycbc.inference.sampler.dynesty.DynestySampler method)

set_state_from_file() (pycbc.inference.sampler.dynesty.DynestySampler method)

write_results() (pycbc.inference.sampler.dynesty.DynestySampler method)

base_shape (pycbc.inference.sampler.emcee.EmceeEnsembleSampler attribute)

burn_in_class (pycbc.inference.sampler.emcee.EmceeEnsembleSampler attribute)

clear_samples() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler method)

compute_acf() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler static method)

compute_acl() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler static method)

finalize() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler method)

from_config() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler class method)

io (pycbc.inference.sampler.emcee.EmceeEnsembleSampler attribute)

model_stats (pycbc.inference.sampler.emcee.EmceeEnsembleSampler attribute)

name (pycbc.inference.sampler.emcee.EmceeEnsembleSampler attribute)

run_mcmc() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler method)

samples (pycbc.inference.sampler.emcee.EmceeEnsembleSampler attribute)

write_results() (pycbc.inference.sampler.emcee.EmceeEnsembleSampler method)

base_shape (pycbc.inference.sampler.emcee_pt.EmceePTSampler attribute)

betas (pycbc.inference.sampler.emcee_pt.EmceePTSampler attribute)

burn_in_class (pycbc.inference.sampler.emcee_pt.EmceePTSampler attribute)

calculate_logevidence() (pycbc.inference.sampler.emcee_pt.EmceePTSampler class method)

clear_samples() (pycbc.inference.sampler.emcee_pt.EmceePTSampler method)

compute_acf() (pycbc.inference.sampler.emcee_pt.EmceePTSampler static method)

compute_acl() (pycbc.inference.sampler.emcee_pt.EmceePTSampler static method)

finalize() (pycbc.inference.sampler.emcee_pt.EmceePTSampler method)

from_config() (pycbc.inference.sampler.emcee_pt.EmceePTSampler class method)

model_stats (pycbc.inference.sampler.emcee_pt.EmceePTSampler attribute)

name (pycbc.inference.sampler.emcee_pt.EmceePTSampler attribute)

run_mcmc() (pycbc.inference.sampler.emcee_pt.EmceePTSampler method)

samples (pycbc.inference.sampler.emcee_pt.EmceePTSampler attribute)

set_state_from_file() (pycbc.inference.sampler.emcee_pt.EmceePTSampler method)

write_results() (pycbc.inference.sampler.emcee_pt.EmceePTSampler method)

base_shape (pycbc.inference.sampler.epsie.EpsieSampler attribute)

burn_in_class (pycbc.inference.sampler.epsie.EpsieSampler attribute)

clear_samples() (pycbc.inference.sampler.epsie.EpsieSampler method)

compute_acf() (pycbc.inference.sampler.epsie.EpsieSampler static method)

compute_acl() (pycbc.inference.sampler.epsie.EpsieSampler static method)

effective_nsamples (pycbc.inference.sampler.epsie.EpsieSampler attribute)

from_config() (pycbc.inference.sampler.epsie.EpsieSampler class method)

model_stats (pycbc.inference.sampler.epsie.EpsieSampler attribute)

set_state_from_file() (pycbc.inference.sampler.epsie.EpsieSampler method)

swap_interval (pycbc.inference.sampler.epsie.EpsieSampler attribute)

write_results() (pycbc.inference.sampler.epsie.EpsieSampler method)

check_if_finished() (pycbc.inference.sampler.multinest.MultinestSampler method)

checkpoint() (pycbc.inference.sampler.multinest.MultinestSampler method)

dlogz (pycbc.inference.sampler.multinest.MultinestSampler attribute)

finalize() (pycbc.inference.sampler.multinest.MultinestSampler method)

from_config() (pycbc.inference.sampler.multinest.MultinestSampler class method)

importance_dlogz (pycbc.inference.sampler.multinest.MultinestSampler attribute)

importance_logz (pycbc.inference.sampler.multinest.MultinestSampler attribute)

io (pycbc.inference.sampler.multinest.MultinestSampler attribute)

loglikelihood() (pycbc.inference.sampler.multinest.MultinestSampler method)

logz (pycbc.inference.sampler.multinest.MultinestSampler attribute)

model_stats (pycbc.inference.sampler.multinest.MultinestSampler attribute)

name (pycbc.inference.sampler.multinest.MultinestSampler attribute)

niterations (pycbc.inference.sampler.multinest.MultinestSampler attribute)

nlivepoints (pycbc.inference.sampler.multinest.MultinestSampler attribute)

run() (pycbc.inference.sampler.multinest.MultinestSampler method)

samples (pycbc.inference.sampler.multinest.MultinestSampler attribute)

setup_output() (pycbc.inference.sampler.multinest.MultinestSampler method)

transform_prior() (pycbc.inference.sampler.multinest.MultinestSampler method)

write_results() (pycbc.inference.sampler.multinest.MultinestSampler method)

adaptation_lag (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

adaptation_time (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

adaptive (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

base_shape (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

betas (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

burn_in_class (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

calculate_logevidence() (pycbc.inference.sampler.ptemcee.PTEmceeSampler class method)

chain (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

clear_samples() (pycbc.inference.sampler.ptemcee.PTEmceeSampler method)

compute_acf() (pycbc.inference.sampler.ptemcee.PTEmceeSampler static method)

compute_acl() (pycbc.inference.sampler.ptemcee.PTEmceeSampler static method)

ensemble (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

finalize() (pycbc.inference.sampler.ptemcee.PTEmceeSampler method)

from_config() (pycbc.inference.sampler.ptemcee.PTEmceeSampler class method)

model_stats (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

ntemps (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

run_mcmc() (pycbc.inference.sampler.ptemcee.PTEmceeSampler method)

samples (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

scale_factor (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

set_state_from_file() (pycbc.inference.sampler.ptemcee.PTEmceeSampler method)

starting_betas (pycbc.inference.sampler.ptemcee.PTEmceeSampler attribute)

write_results() (pycbc.inference.sampler.ptemcee.PTEmceeSampler method)

checkpoint() (pycbc.inference.sampler.ultranest.UltranestSampler method)

finalize() (pycbc.inference.sampler.ultranest.UltranestSampler method)

from_config() (pycbc.inference.sampler.ultranest.UltranestSampler class method)

io (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

logz (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

logz_err (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

model_stats (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

name (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

niterations (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

run() (pycbc.inference.sampler.ultranest.UltranestSampler method)

samples (pycbc.inference.sampler.ultranest.UltranestSampler attribute)

write_results() (pycbc.inference.sampler.ultranest.UltranestSampler method)

make_strain_from_inj_object() (pycbc.inject.inject.CBCHDFInjectionSet method)

required_params (pycbc.inject.inject.CBCHDFInjectionSet attribute)

supported_approximants() (pycbc.inject.inject.CBCHDFInjectionSet static method)

IncoherentFromFileHDFInjectionSet (class in pycbc.inject.inject)

apply() (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet method)

end_times() (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet method)

injtype (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet attribute)

loadts() (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet method)

set_ref_time() (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet method)

slice_and_taper() (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet static method)

supported_approximants() (pycbc.inject.inject.IncoherentFromFileHDFInjectionSet static method)

end_times() (pycbc.inject.inject.RingdownHDFInjectionSet method)

required_params (pycbc.inject.inject.RingdownHDFInjectionSet attribute)

supported_approximants() (pycbc.inject.inject.RingdownHDFInjectionSet static method)

from_cli() (pycbc.inject.injfilterrejector.InjFilterRejector class method)

from_cli_multi_ifos() (pycbc.inject.injfilterrejector.InjFilterRejector class method)

from_cli_single_ifo() (pycbc.inject.injfilterrejector.InjFilterRejector class method)

insert_injfilterrejector_option_group() (in module pycbc.inject.injfilterrejector)

insert_injfilterrejector_option_group_multi_ifo() (in module pycbc.inject.injfilterrejector)

get_snglfile_array_dict() (pycbc.io.hdf.ForegroundTriggers method)

mask_to_n_loudest_clustered_events() (pycbc.io.hdf.SingleDetTriggers method)

newsnr_sgveto_psdvar_threshold (pycbc.io.hdf.SingleDetTriggers attribute)

recursively_save_dict_contents_to_group() (in module pycbc.io.hdf)

draw_flat_samples() (in module pycbc.population.rates_functions)

mchirp_sampler_flat() (in module pycbc.population.rates_functions)

mchirp_sampler_imf() (in module pycbc.population.rates_functions)

mchirp_sampler_lnm() (in module pycbc.population.rates_functions)

process_full_data() (in module pycbc.population.rates_functions)

skew_lognormal_samples() (in module pycbc.population.rates_functions)

inj_distance_pdf() (in module pycbc.population.scale_injections)

process_injections() (in module pycbc.population.scale_injections)

AdVBNSOptimizedSensitivityP1200087() (in module pycbc.psd.analytical)

AdVEarlyHighSensitivityP1200087() (in module pycbc.psd.analytical)

AdVEarlyLowSensitivityP1200087() (in module pycbc.psd.analytical)

AdVLateHighSensitivityP1200087() (in module pycbc.psd.analytical)

AdVLateLowSensitivityP1200087() (in module pycbc.psd.analytical)

AdVMidHighSensitivityP1200087() (in module pycbc.psd.analytical)

CosmicExplorerPessimisticP1600143() (in module pycbc.psd.analytical)

CosmicExplorerWidebandP1600143() (in module pycbc.psd.analytical)

KAGRADesignSensitivityT1600593() (in module pycbc.psd.analytical)

KAGRAEarlySensitivityT1600593() (in module pycbc.psd.analytical)

KAGRAOpeningSensitivityT1600593() (in module pycbc.psd.analytical)

aLIGOAPlusDesignSensitivityT1800042() (in module pycbc.psd.analytical)

aLIGOAdVO4IntermediateT1800545() (in module pycbc.psd.analytical)

aLIGOBNSOptimizedSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGODesignSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOEarlyHighSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOEarlyLowSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOLateHighSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOLateLowSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOMidHighSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOMidLowSensitivityP1200087() (in module pycbc.psd.analytical)

aLIGOaLIGODesignSensitivityT1800044() (in module pycbc.psd.analytical)

write_exclusion_distances() (in module pycbc.results.legacy_grb)

calculate_contours() (in module pycbc.results.pygrb_plotting_utils)

contour_plotter() (in module pycbc.results.pygrb_plotting_utils)

format_single_chisqs() (in module pycbc.results.pygrb_plotting_utils)

load_injections() (in module pycbc.results.pygrb_plotting_utils)

pygrb_plot_opts_parser() (in module pycbc.results.pygrb_plotting_utils)

pygrb_shared_plot_setups() (in module pycbc.results.pygrb_plotting_utils)

render_workflow_html_template() (in module pycbc.results.render)

create_density_plot() (in module pycbc.results.scatter_histograms)

create_marginalized_hist() (in module pycbc.results.scatter_histograms)

create_multidim_plot() (in module pycbc.results.scatter_histograms)

remove_common_offset() (in module pycbc.results.scatter_histograms)

set_marginal_histogram_title() (in module pycbc.results.scatter_histograms)

write_library_information() (in module pycbc.results.versioning)

apply_calibration() (pycbc.strain.calibration.CubicSpline method)

apply_calibration() (pycbc.strain.calibration.Recalibrate method)

from_config() (pycbc.strain.calibration.Recalibrate class method)

apply_calibration() (pycbc.strain.recalibrate.CubicSpline method)

from_config() (pycbc.strain.recalibrate.PhysicalModel class method)

tf_from_file() (pycbc.strain.recalibrate.PhysicalModel class method)

apply_calibration() (pycbc.strain.recalibrate.Recalibrate method)

from_config() (pycbc.strain.recalibrate.Recalibrate class method)

from_cli_multi_ifos() (pycbc.strain.strain.StrainSegments class method)

from_cli_single_ifo() (pycbc.strain.strain.StrainSegments class method)

insert_segment_option_group() (pycbc.strain.strain.StrainSegments class method)

insert_segment_option_group_multi_ifo() (pycbc.strain.strain.StrainSegments class method)

required_opts_list (pycbc.strain.strain.StrainSegments attribute)

verify_segment_options() (pycbc.strain.strain.StrainSegments class method)

verify_segment_options_multi_ifo() (pycbc.strain.strain.StrainSegments class method)

insert_strain_option_group_multi_ifo() (in module pycbc.strain.strain)

verify_strain_options_multi_ifo() (in module pycbc.strain.strain)

calculate_ethinca_metric_comps() (in module pycbc.tmpltbank.bank_output_utils)

convert_to_sngl_inspiral_table() (in module pycbc.tmpltbank.bank_output_utils)

output_sngl_inspiral_table() (in module pycbc.tmpltbank.bank_output_utils)

return_empty_sngl() (in module pycbc.tmpltbank.bank_output_utils)

return_search_summary() (in module pycbc.tmpltbank.bank_output_utils)

find_xi_extrema_brute() (in module pycbc.tmpltbank.brute_force_methods)

get_mass_distribution() (in module pycbc.tmpltbank.brute_force_methods)

get_physical_covaried_masses() (in module pycbc.tmpltbank.brute_force_methods)

stack_xi_direction_brute() (in module pycbc.tmpltbank.brute_force_methods)

calculate_metric_comp() (in module pycbc.tmpltbank.calc_moments)

determine_eigen_directions() (in module pycbc.tmpltbank.calc_moments)

find_closest_calculated_frequencies() (in module pycbc.tmpltbank.coord_utils)

find_max_and_min_frequencies() (in module pycbc.tmpltbank.coord_utils)

get_random_mass_point_particles() (in module pycbc.tmpltbank.coord_utils)

ns_g_mass_to_ns_b_mass() (in module pycbc.tmpltbank.em_progenitors)

ns_g_mass_to_ns_compactness() (in module pycbc.tmpltbank.em_progenitors)

ethinca_order_from_string() (in module pycbc.tmpltbank.lambda_mapping)

generate_inverse_mapping() (in module pycbc.tmpltbank.lambda_mapping)

generate_anstar_3d_lattice() (in module pycbc.tmpltbank.lattice_utils)

generate_hexagonal_lattice() (in module pycbc.tmpltbank.lattice_utils)

IndentedHelpFormatterWithNL (class in pycbc.tmpltbank.option_utils)

check_ethinca_against_bank_params() (in module pycbc.tmpltbank.option_utils)

from_argparse() (pycbc.tmpltbank.option_utils.ethincaParameters class method)

get_options_from_group() (in module pycbc.tmpltbank.option_utils)

insert_base_bank_options() (in module pycbc.tmpltbank.option_utils)

insert_ethinca_metric_options() (in module pycbc.tmpltbank.option_utils)

insert_mass_range_option_group() (in module pycbc.tmpltbank.option_utils)

insert_metric_calculation_options() (in module pycbc.tmpltbank.option_utils)

default_ns_eos (pycbc.tmpltbank.option_utils.massRangeParameters attribute)

from_argparse() (pycbc.tmpltbank.option_utils.massRangeParameters class method)

is_outside_range() (pycbc.tmpltbank.option_utils.massRangeParameters method)

evecsCV (pycbc.tmpltbank.option_utils.metricParameters attribute)

from_argparse() (pycbc.tmpltbank.option_utils.metricParameters class method)

metric (pycbc.tmpltbank.option_utils.metricParameters attribute)

moments (pycbc.tmpltbank.option_utils.metricParameters attribute)

verify_ethinca_metric_options() (in module pycbc.tmpltbank.option_utils)

verify_mass_range_options() (in module pycbc.tmpltbank.option_utils)

verify_metric_calculation_options() (in module pycbc.tmpltbank.option_utils)

PartitionedTmpltbank (class in pycbc.tmpltbank.partitioned_bank)

find_point_bin() (pycbc.tmpltbank.partitioned_bank.PartitionedTmpltbank method)

add_options_to_section() (pycbc.types.config.InterpolatingConfigParser method)

check_duplicate_options() (pycbc.types.config.InterpolatingConfigParser method)

from_cli() (pycbc.types.config.InterpolatingConfigParser class method)

get_opt_tag() (pycbc.types.config.InterpolatingConfigParser method)

get_opt_tags() (pycbc.types.config.InterpolatingConfigParser method)

get_subsections() (pycbc.types.config.InterpolatingConfigParser method)

has_option_tag() (pycbc.types.config.InterpolatingConfigParser method)

has_option_tags() (pycbc.types.config.InterpolatingConfigParser method)

interpolate_string() (pycbc.types.config.InterpolatingConfigParser method)

populate_shared_sections() (pycbc.types.config.InterpolatingConfigParser method)

read_ini_file() (pycbc.types.config.InterpolatingConfigParser method)

sanity_check_subsections() (pycbc.types.config.InterpolatingConfigParser method)

split_multi_sections() (pycbc.types.config.InterpolatingConfigParser method)

sample_frequencies (pycbc.types.frequencyseries.FrequencySeries attribute)

almost_equal_elem() (pycbc.types.frequencyseries.FrequencySeries method)

almost_equal_norm() (pycbc.types.frequencyseries.FrequencySeries method)

at_frequency() (pycbc.types.frequencyseries.FrequencySeries method)

cyclic_time_shift() (pycbc.types.frequencyseries.FrequencySeries method)

duration (pycbc.types.frequencyseries.FrequencySeries attribute)

end_time (pycbc.types.frequencyseries.FrequencySeries attribute)

get_delta_f() (pycbc.types.frequencyseries.FrequencySeries method)

get_epoch() (pycbc.types.frequencyseries.FrequencySeries method)

get_sample_frequencies() (pycbc.types.frequencyseries.FrequencySeries method)

sample_rate (pycbc.types.frequencyseries.FrequencySeries attribute)

start_time (pycbc.types.frequencyseries.FrequencySeries attribute)

to_frequencyseries() (pycbc.types.frequencyseries.FrequencySeries method)

to_timeseries() (pycbc.types.frequencyseries.FrequencySeries method)

default_set (pycbc.types.optparse.DictWithDefaultReturn attribute)

convert_to_process_params_dict() (in module pycbc.types.optparse)

values() (pycbc.vetoes.autochisq.SingleDetSkyMaxAutoChisq method)

cache_overlaps() (pycbc.vetoes.bank_chisq.SingleDetBankVeto method)

cache_segment_snrs() (pycbc.vetoes.bank_chisq.SingleDetBankVeto method)

values() (pycbc.vetoes.bank_chisq.SingleDetSkyMaxBankVeto method)

cached_chisq_bins() (pycbc.vetoes.chisq.SingleDetPowerChisq method)

parse_option() (pycbc.vetoes.chisq.SingleDetPowerChisq static method)

calculate_chisq_bins() (pycbc.vetoes.chisq.SingleDetSkyMaxPowerChisq method)

power_chisq_at_points_from_precomputed() (in module pycbc.vetoes.chisq)

power_chisq_bins_from_sigmasq_series() (in module pycbc.vetoes.chisq)

insert_option_group() (pycbc.vetoes.sgchisq.SingleDetSGChisq static method)

generate_with_delta_f_and_max_freq() (pycbc.waveform.bank.FilterBank method)

get_decompressed_waveform() (pycbc.waveform.bank.FilterBank method)

has_compressed_waveforms (pycbc.waveform.bank.TemplateBank attribute)

ensure_standard_filter_columns() (pycbc.waveform.bank.TemplateBank method)

sample_points (pycbc.waveform.compress.CompressedWaveform attribute)

amplitude (pycbc.waveform.compress.CompressedWaveform attribute)

load_to_memory (pycbc.waveform.compress.CompressedWaveform attribute)

interpolation (pycbc.waveform.compress.CompressedWaveform attribute)

tolerance (pycbc.waveform.compress.CompressedWaveform attribute)

precision (pycbc.waveform.compress.CompressedWaveform attribute)

clear_cache() (pycbc.waveform.compress.CompressedWaveform method)

decompress() (pycbc.waveform.compress.CompressedWaveform method)

from_hdf() (pycbc.waveform.compress.CompressedWaveform class method)

write_to_hdf() (pycbc.waveform.compress.CompressedWaveform method)

inline_linear_interp() (in module pycbc.waveform.decompress_cpu)

decomp_ccode_double() (in module pycbc.waveform.decompress_cpu_cython)

decomp_ccode_float() (in module pycbc.waveform.decompress_cpu_cython)

possible_args (pycbc.waveform.generator.BaseCBCGenerator attribute)

BaseFDomainDetFrameGenerator (class in pycbc.waveform.generator)

location_args (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

detectors (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

detector_names (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

epoch (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

current_params (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

variable_args (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

generate() (pycbc.waveform.generator.BaseFDomainDetFrameGenerator method)

set_epoch() (pycbc.waveform.generator.BaseFDomainDetFrameGenerator method)

static_args (pycbc.waveform.generator.BaseFDomainDetFrameGenerator attribute)

variable_args (pycbc.waveform.generator.BaseGenerator attribute)

frozen_params (pycbc.waveform.generator.BaseGenerator attribute)

current_params (pycbc.waveform.generator.BaseGenerator attribute)

location_args (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

detectors (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

detector_names (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

epoch (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

current_params (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

rframe_generator (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

variable_args (pycbc.waveform.generator.FDomainDetFrameGenerator attribute)

generate() (pycbc.waveform.generator.FDomainDetFrameGenerator method)

select_rframe_generator() (pycbc.waveform.generator.FDomainDetFrameGenerator static method)

FDomainDetFrameModesGenerator (class in pycbc.waveform.generator)

location_args (pycbc.waveform.generator.FDomainDetFrameModesGenerator attribute)

detectors (pycbc.waveform.generator.FDomainDetFrameModesGenerator attribute)

epoch (pycbc.waveform.generator.FDomainDetFrameModesGenerator attribute)

variable_args (pycbc.waveform.generator.FDomainDetFrameModesGenerator attribute)

generate() (pycbc.waveform.generator.FDomainDetFrameModesGenerator method)

select_rframe_generator() (pycbc.waveform.generator.FDomainDetFrameModesGenerator static method)

FDomainDetFrameTwoPolGenerator (class in pycbc.waveform.generator)

detectors (pycbc.waveform.generator.FDomainDetFrameTwoPolGenerator attribute)

epoch (pycbc.waveform.generator.FDomainDetFrameTwoPolGenerator attribute)

generate() (pycbc.waveform.generator.FDomainDetFrameTwoPolGenerator method)

select_rframe_generator() (pycbc.waveform.generator.FDomainDetFrameTwoPolGenerator static method)

FDomainFreqTauRingdownGenerator (class in pycbc.waveform.generator)

FDomainMassSpinRingdownGenerator (class in pycbc.waveform.generator)

TDomainFreqTauRingdownGenerator (class in pycbc.waveform.generator)

TDomainMassSpinRingdownGenerator (class in pycbc.waveform.generator)

select_waveform_generator() (in module pycbc.waveform.generator)

select_waveform_modes_generator() (in module pycbc.waveform.generator)

nltides_fourier_phase_difference() (in module pycbc.waveform.nltides)

description_dict (pycbc.waveform.parameters.ParameterList attribute)

descriptions (pycbc.waveform.parameters.ParameterList attribute)

get_fd_from_final_mass_spin() (in module pycbc.waveform.ringdown)

get_td_from_final_mass_spin() (in module pycbc.waveform.ringdown)

spa_tmplt_inline_sequence() (in module pycbc.waveform.spa_tmplt_cpu)

amplitude_from_frequencyseries() (in module pycbc.waveform.utils)

get_waveform_end_frequency() (in module pycbc.waveform.waveform)

get_template_amplitude_norm() (in module pycbc.waveform.waveform)

get_waveform_filter_length_in_time() (in module pycbc.waveform.waveform)

print_sgburst_approximants() (in module pycbc.waveform.waveform)

td_waveform_to_fd_waveform() (in module pycbc.waveform.waveform)

get_two_pol_waveform_filter() (in module pycbc.waveform.waveform)

fd_waveform_mode_approximants() (in module pycbc.waveform.waveform_modes)

get_fd_waveform_modes() (in module pycbc.waveform.waveform_modes)

get_td_waveform_modes() (in module pycbc.waveform.waveform_modes)

td_waveform_mode_approximants() (in module pycbc.waveform.waveform_modes)

current_retention_level (pycbc.workflow.coincidence.CensorForeground attribute)

current_retention_level (pycbc.workflow.coincidence.MergeExecutable attribute)

create_node() (pycbc.workflow.coincidence.PyCBCAddStatmap method)

current_retention_level (pycbc.workflow.coincidence.PyCBCAddStatmap attribute)

create_node() (pycbc.workflow.coincidence.PyCBCBank2HDFExecutable method)

create_node() (pycbc.workflow.coincidence.PyCBCCombineStatmap method)

PyCBCDistributeBackgroundBins (class in pycbc.workflow.coincidence)

create_node() (pycbc.workflow.coincidence.PyCBCDistributeBackgroundBins method)

create_node() (pycbc.workflow.coincidence.PyCBCExcludeZerolag method)

create_node() (pycbc.workflow.coincidence.PyCBCFindCoincExecutable method)

PyCBCFitByTemplateExecutable (class in pycbc.workflow.coincidence)

create_node() (pycbc.workflow.coincidence.PyCBCFitByTemplateExecutable method)

PyCBCFitOverParamExecutable (class in pycbc.workflow.coincidence)

create_node() (pycbc.workflow.coincidence.PyCBCFitOverParamExecutable method)

create_node() (pycbc.workflow.coincidence.PyCBCHDFInjFindExecutable method)

create_node() (pycbc.workflow.coincidence.PyCBCStatMapExecutable method)

create_node() (pycbc.workflow.coincidence.PyCBCStatMapInjExecutable method)

create_node() (pycbc.workflow.coincidence.PyCBCTrig2HDFExecutable method)

find_injections_in_hdf_coinc() (in module pycbc.workflow.coincidence)

make_foreground_censored_veto() (in module pycbc.workflow.coincidence)

merge_single_detector_hdf_files() (in module pycbc.workflow.coincidence)

select_files_by_ifo_combination() (in module pycbc.workflow.coincidence)

setup_interval_coinc_inj() (in module pycbc.workflow.coincidence)

add_options_to_section() (in module pycbc.workflow.configparser_test)

check_duplicate_options() (in module pycbc.workflow.configparser_test)

interpolate_string() (in module pycbc.workflow.configparser_test)

parse_workflow_ini_file() (in module pycbc.workflow.configparser_test)

perform_extended_interpolation() (in module pycbc.workflow.configparser_test)

sanity_check_subsections() (in module pycbc.workflow.configparser_test)

split_multi_sections() (in module pycbc.workflow.configparser_test)

get_cli_option() (pycbc.workflow.configuration.WorkflowConfigParser method)

interpolate_exe() (pycbc.workflow.configuration.WorkflowConfigParser method)

resolve_file_url() (pycbc.workflow.configuration.WorkflowConfigParser method)

resolve_urls() (pycbc.workflow.configuration.WorkflowConfigParser method)

section_to_cli() (pycbc.workflow.configuration.WorkflowConfigParser method)

add_workflow_command_line_group() (in module pycbc.workflow.configuration)

KEEP_BUT_RAISE_WARNING (pycbc.workflow.core.Executable attribute)

current_retention_level (pycbc.workflow.core.Executable attribute)

time_dependent_options (pycbc.workflow.core.Executable attribute)

update_current_retention_level() (pycbc.workflow.core.Executable method)

update_output_directory() (pycbc.workflow.core.Executable method)

find_all_output_in_range() (pycbc.workflow.core.FileList method)

get_times_covered_by_files() (pycbc.workflow.core.FileList method)

add_multiifo_output_list_opt() (pycbc.workflow.core.Node method)

new_multiifo_output_list_opt() (pycbc.workflow.core.Node method)

from_multi_segment_list() (pycbc.workflow.core.SegFile class method)

from_segment_list_dict() (pycbc.workflow.core.SegFile class method)

set_job_properties() (pycbc.workflow.core.Workflow static method)

check_output_error_and_retcode() (in module pycbc.workflow.core)

convert_cachelist_to_filelist() (in module pycbc.workflow.datafind)

datafind_keep_unique_backups() (in module pycbc.workflow.datafind)

get_missing_segs_from_frame_file_cache() (in module pycbc.workflow.datafind)

get_science_segs_from_datafind_outs() (in module pycbc.workflow.datafind)

setup_datafind_from_pregenerated_lcf_files() (in module pycbc.workflow.datafind)

setup_datafind_runtime_cache_multi_calls_perifo() (in module pycbc.workflow.datafind)

setup_datafind_runtime_cache_single_call_perifo() (in module pycbc.workflow.datafind)

setup_datafind_runtime_frames_multi_calls_perifo() (in module pycbc.workflow.datafind)

setup_datafind_runtime_frames_single_call_perifo() (in module pycbc.workflow.datafind)

setup_datafind_server_connection() (in module pycbc.workflow.datafind)

create_fits_file() (in module pycbc.workflow.inference_followups)

create_posterior_files() (in module pycbc.workflow.inference_followups)

get_diagnostic_plots() (in module pycbc.workflow.inference_followups)

make_diagnostic_plots() (in module pycbc.workflow.inference_followups)

make_inference_acceptance_rate_plot() (in module pycbc.workflow.inference_followups)

make_inference_dynesty_run_plot() (in module pycbc.workflow.inference_followups)

make_inference_dynesty_trace_plot() (in module pycbc.workflow.inference_followups)

make_inference_inj_recovery_plot() (in module pycbc.workflow.inference_followups)

make_inference_plot() (in module pycbc.workflow.inference_followups)

make_inference_plot_mcmc_history() (in module pycbc.workflow.inference_followups)

make_inference_posterior_plot() (in module pycbc.workflow.inference_followups)

make_inference_pp_plot() (in module pycbc.workflow.inference_followups)

make_inference_pp_table() (in module pycbc.workflow.inference_followups)

make_inference_prior_plot() (in module pycbc.workflow.inference_followups)

make_inference_samples_plot() (in module pycbc.workflow.inference_followups)

make_inference_skymap() (in module pycbc.workflow.inference_followups)

make_inference_summary_table() (in module pycbc.workflow.inference_followups)

make_posterior_workflow() (in module pycbc.workflow.inference_followups)

create_node() (pycbc.workflow.injection.PyCBCOptimalSNRExecutable method)

create_node() (pycbc.workflow.jobsetup.ComputeDurationsExecutable method)

create_node() (pycbc.workflow.jobsetup.ExtractToXMLExecutable method)

GstlalFarfromsnrchisqhistExecutable (class in pycbc.workflow.jobsetup)

GstlalMarginalizeLikelihoodExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.GstlalPlotBackground method)

current_retention_level (pycbc.workflow.jobsetup.GstlalPlotBackground attribute)

create_node() (pycbc.workflow.jobsetup.GstlalPlotSensitivity method)

create_node() (pycbc.workflow.jobsetup.GstlalPlotSummary method)

current_retention_level (pycbc.workflow.jobsetup.GstlalPlotSummary attribute)

create_and_add_node() (pycbc.workflow.jobsetup.GstlalSummaryPage method)

current_retention_level (pycbc.workflow.jobsetup.GstlalSummaryPage attribute)

create_node() (pycbc.workflow.jobsetup.InspinjfindExecutable method)

get_data_times_for_job() (pycbc.workflow.jobsetup.JobSegmenter method)

get_data_times_for_job_legacy() (pycbc.workflow.jobsetup.JobSegmenter method)

get_data_times_for_job_workflow() (pycbc.workflow.jobsetup.JobSegmenter method)

get_valid_times_for_job() (pycbc.workflow.jobsetup.JobSegmenter method)

get_valid_times_for_job_legacy() (pycbc.workflow.jobsetup.JobSegmenter method)

get_valid_times_for_job_workflow() (pycbc.workflow.jobsetup.JobSegmenter method)

create_node() (pycbc.workflow.jobsetup.LalappsInspinjExecutable method)

create_node() (pycbc.workflow.jobsetup.LigoLWCombineSegsExecutable method)

create_node() (pycbc.workflow.jobsetup.LigolwAddExecutable method)

current_retention_level (pycbc.workflow.jobsetup.LigolwAddExecutable attribute)

LigolwCBCAlignTotalSpinExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.LigolwCBCAlignTotalSpinExecutable method)

LigolwCBCJitterSkylocExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.LigolwCBCJitterSkylocExecutable method)

create_node() (pycbc.workflow.jobsetup.LigolwSSthincaExecutable method)

PyCBCCreateInjectionsExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PyCBCCreateInjectionsExecutable method)

create_node() (pycbc.workflow.jobsetup.PyCBCInspiralExecutable method)

file_input_options (pycbc.workflow.jobsetup.PyCBCInspiralExecutable attribute)

get_valid_times() (pycbc.workflow.jobsetup.PyCBCInspiralExecutable method)

zero_pad_data_extend() (pycbc.workflow.jobsetup.PyCBCInspiralExecutable method)

create_node() (pycbc.workflow.jobsetup.PyCBCMultiInspiralExecutable method)

get_valid_times() (pycbc.workflow.jobsetup.PyCBCMultiInspiralExecutable method)

create_nodata_node() (pycbc.workflow.jobsetup.PyCBCTmpltbankExecutable method)

create_node() (pycbc.workflow.jobsetup.PyCBCTmpltbankExecutable method)

get_valid_times() (pycbc.workflow.jobsetup.PyCBCTmpltbankExecutable method)

PycbcCalculateLikelihoodExecutable (class in pycbc.workflow.jobsetup)

PycbcCombineLikelihoodExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PycbcCombineLikelihoodExecutable method)

PycbcConditionStrainExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PycbcConditionStrainExecutable method)

PycbcCreateInjectionsExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PycbcCreateInjectionsExecutable method)

PycbcDarkVsBrightInjectionsExecutable (class in pycbc.workflow.jobsetup)

PycbcGenerateRankingDataExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PycbcInferenceExecutable method)

PycbcPickleHorizonDistsExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PycbcPickleHorizonDistsExecutable method)

create_node() (pycbc.workflow.jobsetup.PycbcSplitBankExecutable method)

extension (pycbc.workflow.jobsetup.PycbcSplitBankExecutable attribute)

extension (pycbc.workflow.jobsetup.PycbcSplitBankXmlExecutable attribute)

create_node() (pycbc.workflow.jobsetup.PycbcSplitInspinjExecutable method)

PycbcSqliteSimplifyExecutable (class in pycbc.workflow.jobsetup)

create_node() (pycbc.workflow.jobsetup.PycbcSqliteSimplifyExecutable method)

create_node() (pycbc.workflow.jobsetup.PycbcTimeslidesExecutable method)

create_node() (pycbc.workflow.jobsetup.SQLInOutExecutable method)

current_retention_level (pycbc.workflow.jobsetup.SQLInOutExecutable attribute)

multi_ifo_coherent_job_setup() (in module pycbc.workflow.jobsetup)

select_matchedfilter_class() (in module pycbc.workflow.jobsetup)

setup_matchedfltr_dax_generated() (in module pycbc.workflow.matched_filter)

setup_matchedfltr_dax_generated_multi() (in module pycbc.workflow.matched_filter)

setup_matchedfltr_workflow() (in module pycbc.workflow.matched_filter)

file_input_options (pycbc.workflow.minifollowups.PlotQScanExecutable attribute)

SingleTemplateExecutable (class in pycbc.workflow.minifollowups)

SingleTimeFreqExecutable (class in pycbc.workflow.minifollowups)

make_plot_waveform_plot() (in module pycbc.workflow.minifollowups)

make_single_template_plots() (in module pycbc.workflow.minifollowups)

make_singles_timefreq() (in module pycbc.workflow.minifollowups)

make_trigger_timeseries() (in module pycbc.workflow.minifollowups)

setup_foreground_minifollowups() (in module pycbc.workflow.minifollowups)

setup_injection_minifollowups() (in module pycbc.workflow.minifollowups)

setup_single_det_minifollowups() (in module pycbc.workflow.minifollowups)

add_profile() (pycbc.workflow.pegasus_workflow.Executable method)

clear_pfns() (pycbc.workflow.pegasus_workflow.Executable method)

insert_into_dax() (pycbc.workflow.pegasus_workflow.Executable method)

is_same_as() (pycbc.workflow.pegasus_workflow.Executable method)

add_input_list_opt() (pycbc.workflow.pegasus_workflow.Node method)

add_output_list_opt() (pycbc.workflow.pegasus_workflow.Node method)

new_output_file_opt() (pycbc.workflow.pegasus_workflow.Node method)

set_category() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

set_memory() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

set_num_cpus() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

set_num_retries() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

set_priority() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

set_storage() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

set_universe() (pycbc.workflow.pegasus_workflow.ProfileShortcuts method)

add_workflow() (pycbc.workflow.pegasus_workflow.Workflow method)

current_retention_level (pycbc.workflow.plotting.PlotExecutable attribute)

get_cumulative_veto_group_files() (in module pycbc.workflow.segment)

get_triggered_coherent_segment() (in module pycbc.workflow.segment)

select_splitfilejob_instance() (in module pycbc.workflow.splittable)

setup_splittable_dax_generated() (in module pycbc.workflow.splittable)

setup_splittable_workflow() (in module pycbc.workflow.splittable)

setup_tmpltbank_dax_generated() (in module pycbc.workflow.tmpltbank)

setup_tmpltbank_pregenerated() (in module pycbc.workflow.tmpltbank)

setup_tmpltbank_without_frames() (in module pycbc.workflow.tmpltbank)

ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON 3.9.2 DOCUMENTATION

email.message.Message: Representing an email message using the compat32 API

What is the most efficient way to concatenate many strings together?

Why does a_tuple[i] += [‘item’] raise an exception when the addition works?

What does the slash(/) in the parameter list of a function mean?

PEP 3118: New memoryview implementation and buffer protocol documentation

PEP 476: Enabling certificate verification by default for stdlib http clients

PEP 471 - os.scandir() function – a better and faster directory iterator

Why must ‘self’ be used explicitly in method definitions and calls?

argparse — Parser for command-line options, arguments and sub-commands

functools — Higher-order functions and operations on callable objects

multiprocessing.shared_memory —  Provides shared memory for direct access across processes

sysconfig — Provide access to Python’s configuration information

../tools/einsteinathome/pycbc_build_eah.sh --force-debian4 --help

>> [Fri Mar 26 23:26:38 UTC 2021] Start ../tools/einsteinathome/pycbc_build_eah.sh --force-debian4 --help

>> [Fri Mar 26 23:26:38 UTC 2021] Using Debian 4.0 (etch) settings
Options:

    --force-debian4                 force Debian 4.0 build, must be first option in command-line

    --print-env                     dump environment at beginning, must be first option in command-line

    --help                          print this messge and exit

    --clean                         perform a clean build (takes quite some time); delete ~/.cache and
                                    tarballs containing precompiled libraries (lalsuite, scipy etc.)

    --clean-lalsuite                checkout and build lalsuite from scratch

    --clean-sundays                 perform a clean-lalsuite build on sundays

    --clean-pycbc                   check out pycbc git repo from scratch

    --lalsuite-commit=<commit>      specify a commit (or tag or branch) of lalsuite to build from

    --blessed-lalsuite              get the lalsuite commit to build from
                                    https://github.com/ligo-cbc/pycbc/releases/latest

    --pycbc-commit=<commit>         specify a commit or tag of pycbc to build from (specifying a
                                    branch will only work reliably in conjunction with --clean-pycbc)

    --pycbc-remote=<username>       add pycbc repository github.com/username as remote

    --pycbc-branch=<branch>         checkout branch before building

    --no-pycbc-update               don't update local pycbc repo

    --no-lalsuite-update            don't update local lalsuite repo

    --bema-testing                  use einsteinathome_testing branch of bema-ligo/pycbc repo

    --no-cleanup                    keep build directories after successful build for later inspection

    --download-url=<url>            download test frame, template bank, and roms from <url>

    --with-extra-libs=<url>         add extra files from a tar file at <url> to the bundles

    --with-extra-bank=<file>        run pycbc_inspiral again with an extra template bank

    --with-extra-approximant=<file> run pycbc_inspiral again with an extra approximant

    --with-lal-data-path=<path>     run test job using ROM data from <path>

    --processing-scheme=<scheme>    run test job using processing scheme <scheme>

    --verbose-python                run PyInstalled Python in verbose mode, showing imports

    --no-analysis                   for testing, don't run analysis

    --silent-build                  do not brint build messages unless there is an error

    --pycbc-fetch-ref               fetch and use a specific reference for pycbc

dict_keys(['GW190408_181802-v1', 'GW190412-v3', 'GW190413_052954-v1', 'GW190413_134308-v1', 'GW190421_213856-v1', 'GW190424_180648-v1', 'GW190425-v2', 'GW190426_152155-v1', 'GW190503_185404-v1', 'GW190512_180714-v1', 'GW190513_205428-v1', 'GW190514_065416-v1', 'GW190517_055101-v1', 'GW190519_153544-v1', 'GW190521-v3', 'GW190521_074359-v1', 'GW190527_092055-v1', 'GW190602_175927-v1', 'GW190620_030421-v1', 'GW190630_185205-v1', 'GW190701_203306-v1', 'GW190706_222641-v1', 'GW190707_093326-v1', 'GW190708_232457-v1', 'GW190719_215514-v1', 'GW190720_000836-v1', 'GW190727_060333-v1', 'GW190728_064510-v1', 'GW190731_140936-v1', 'GW190803_022701-v1', 'GW190814-v2', 'GW190828_063405-v1', 'GW190828_065509-v1', 'GW190909_114149-v1', 'GW190910_112807-v1', 'GW190915_235702-v1', 'GW190924_021846-v1', 'GW190929_012149-v1', 'GW190930_133541-v1'])
[1238782700.3, 1239082262.2, 1239168612.5, 1239198206.7, 1239917954.3, 1240164426.1, 1240215503.0, 1240327333.3, 1240944862.3, 1241719652.4, 1241816086.8, 1241852074.8, 1242107479.8, 1242315362.4, 1242442967.4, 1242459857.5, 1242984073.8, 1243533585.1, 1245035079.3, 1245955943.2, 1246048404.6, 1246487219.3, 1246527224.2, 1246663515.4, 1247608532.9, 1247616534.7, 1248242632.0, 1248331528.5, 1248617394.6, 1248834439.9, 1249852257.0, 1251009263.8, 1251010527.9, 1252064527.7, 1252150105.3, 1252627040.7, 1253326744.8, 1253755327.5, 1253885759.2]

TAMA_300 T1 Latitude 0.6226733601781139 Longitude 2.435363594690946
VIRGO_CITF V0 Latitude 0.7615118398044829 Longitude 0.1833380521285067
VIRGO V1 Latitude 0.7615118398044829 Longitude 0.1833380521285067
GEO_600 G1 Latitude 0.9118498274833728 Longitude 0.17116780434996115
LHO_2k H2 Latitude 0.810795263791696 Longitude -2.0840567691659397
LHO_4k H1 Latitude 0.810795263791696 Longitude -2.0840567691659397
LLO_4k L1 Latitude 0.5334231350225018 Longitude -1.5843093707829259
LIO_4k I1 Latitude 0.24841853018214574 Longitude 1.3340133249409996
CIT_40 C1 Latitude 0.5963790053711457 Longitude -2.0617574453805614
ET1_T1400308 E1 Latitude 0.7615118398044829 Longitude 0.1833380521285067
ET2_T1400308 E2 Latitude 0.7629930799052169 Longitude 0.18405858870223898
ET3_T1400308 E3 Latitude 0.7627046325725655 Longitude 0.18192996730110464
ET0_T1400308 E0 Latitude 0.7627046325725655 Longitude 0.18192996730110464
KAGRA K1 Latitude 0.6355068496865413 Longitude 2.396441015339088
ACIGA U1 Latitude 1.5707963267948966 Longitude 0.0
ALLEGRO_320 A1 Latitude 0.5307987920242562 Longitude -1.591370684957212
AURIGA O1 Latitude 0.7915649933828905 Longitude 0.2085377567924929
EXPLORER X1 Latitude 0.8107054375131121 Longitude 0.10821041362369214
Nautilus N1 Latitude 0.7299645670603321 Longitude 0.2211768494583896
NIOBE B1 Latitude -0.5573418077694188 Longitude 2.0213821620185053

Direct Time from H1 to H1 is 0.0 seconds
Direct Time from H1 to L1 is 0.010012846152223925 seconds
Direct Time from H1 to V1 is 0.027287979933844225 seconds
Direct Time from L1 to H1 is 0.010012846152223925 seconds
Direct Time from L1 to L1 is 0.0 seconds
Direct Time from L1 to V1 is 0.026448341016726495 seconds
Direct Time from V1 to H1 is 0.027287979933844225 seconds
Direct Time from V1 to L1 is 0.026448341016726495 seconds
Direct Time from V1 to V1 is 0.0 seconds

GW passed through H1 0.0 seconds relative to passing by Hanford
GW passed through L1 0.002444164315694857 seconds relative to passing by Hanford
GW passed through V1 -0.014733669723508454 seconds relative to passing by Hanford

usage: pycbc_geom_nonspinbank [-h] [--version] [-V]
                              [--random-seed RANDOM_SEED] -m MIN_MATCH -O
                              OUTPUT_FILE [--f-low-column NAME] --pn-order
                              PN_ORDER [--f0 F0] --f-low F_LOW --f-upper
                              F_UPPER --delta-f DELTA_F [--write-metric]
                              --min-mass1 MIN_MASS1 --max-mass1 MAX_MASS1
                              --min-mass2 MIN_MASS2 --max-mass2 MAX_MASS2
                              [--max-total-mass MAX_TOTAL_MASS]
                              [--min-total-mass MIN_TOTAL_MASS]
                              [--max-chirp-mass MAX_CHIRP_MASS]
                              [--min-chirp-mass MIN_CHIRP_MASS]
                              [--max-eta MAX_ETA] [--min-eta MIN_ETA]
                              [--ns-eos NS_EOS]
                              [--remnant-mass-threshold REMNANT_MASS_THRESHOLD]
                              [--use-eos-max-ns-mass]
                              [--delta-bh-spin DELTA_BH_SPIN]
                              [--delta-ns-mass DELTA_NS_MASS]
                              [--psd-model {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,flat_unity}]
                              [--psd-file PSD_FILE] [--asd-file ASD_FILE]
                              [--psd-inverse-length PSD_INVERSE_LENGTH]
                              [--psd-file-xml-ifo-string PSD_FILE_XML_IFO_STRING]
                              [--psd-file-xml-root-name PSD_FILE_XML_ROOT_NAME]
                              [--psdvar-segment SECONDS]
                              [--psdvar-short-segment SECONDS]
                              [--psdvar-long-segment SECONDS]
                              [--psdvar-psd-duration SECONDS]
                              [--psdvar-psd-stride SECONDS]
                              [--psdvar-low-freq HERTZ]
                              [--psdvar-high-freq HERTZ]
                              [--psd-estimation {mean,median,median-mean}]
                              [--psd-segment-length PSD_SEGMENT_LENGTH]
                              [--psd-segment-stride PSD_SEGMENT_STRIDE]
                              [--psd-num-segments PSD_NUM_SEGMENTS]
                              [--psd-output PSD_OUTPUT]
                              [--gps-start-time GPS_START_TIME]
                              [--gps-end-time GPS_END_TIME]
                              [--strain-high-pass STRAIN_HIGH_PASS]
                              [--pad-data PAD_DATA] [--taper-data TAPER_DATA]
                              [--sample-rate SAMPLE_RATE]
                              [--channel-name CHANNEL_NAME]
                              [--frame-cache FRAME_CACHE [FRAME_CACHE ...]]
                              [--frame-files FRAME_FILES [FRAME_FILES ...]]
                              [--hdf-store HDF_STORE]
                              [--frame-type FRAME_TYPE]
                              [--frame-sieve FRAME_SIEVE]
                              [--fake-strain {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,zeroNoise}]
                              [--fake-strain-seed FAKE_STRAIN_SEED]
                              [--fake-strain-from-file FAKE_STRAIN_FROM_FILE]
                              [--fake-strain-flow FAKE_STRAIN_FLOW]
                              [--fake-strain-filter-duration FAKE_STRAIN_FILTER_DURATION]
                              [--fake-strain-sample-rate FAKE_STRAIN_SAMPLE_RATE]
                              [--injection-file INJECTION_FILE]
                              [--sgburst-injection-file SGBURST_INJECTION_FILE]
                              [--injection-scale-factor INJECTION_SCALE_FACTOR]
                              [--injection-sample-rate INJECTION_SAMPLE_RATE]
                              [--injection-f-ref INJECTION_F_REF]
                              [--injection-f-final INJECTION_F_FINAL]
                              [--gating-file GATING_FILE]
                              [--autogating-threshold SIGMA]
                              [--autogating-max-iterations SIGMA]
                              [--autogating-cluster SECONDS]
                              [--autogating-width SECONDS]
                              [--autogating-taper SECONDS]
                              [--autogating-pad SECONDS]
                              [--normalize-strain NORMALIZE_STRAIN]
                              [--zpk-z ZPK_Z [ZPK_Z ...]]
                              [--zpk-p ZPK_P [ZPK_P ...]] [--zpk-k ZPK_K]
                              [--witness-frame-type WITNESS_FRAME_TYPE]
                              [--witness-tf-file WITNESS_TF_FILE]
                              [--witness-filter-length WITNESS_FILTER_LENGTH]
                              [--calculate-time-metric-components | --calculate-ethinca-metric]
                              [--ethinca-pn-order {zeroPN,onePN,onePointFivePN,twoPN,twoPointFivePN,threePN,threePointFivePN}]
                              [--filter-cutoff {SchwarzISCO,LightRing,ERD,BKLISCO,FRD,LRD,MECO,HybridMECO,IMRPhenomBFinal,IMRPhenomCFinal,IMRPhenomDPeak,EOBNRv2RD,EOBNRv2HMRD,SEOBNRv1RD,SEOBNRv1Peak,SEOBNRv2RD,SEOBNRv2Peak,SEOBNRv4RD,SEOBNRv4Peak}]
                              [--ethinca-frequency-step ETHINCA_FREQUENCY_STEP]

Template bank generator for placing a bank of non-spinning templates.

optional arguments:
  -h, --help            show this help message and exit
  --version             show program's version number and exit
  -V, --verbose         verbose output (default: False)
  --random-seed RANDOM_SEED
                        Random seed to use when calling numpy.random functions
                        used in obtaining the principal components and when
                        translating points back to physical space. If given,
                        the code should give the same output when run with the
                        same random seed. (default: None)
  -m MIN_MATCH, --min-match MIN_MATCH
                        Generate bank with specified minimum match. Required.
                        (default: None)
  -O OUTPUT_FILE, --output-file OUTPUT_FILE
                        Output file name. Required. (default: None)
  --f-low-column NAME   If given, store the lower frequency cutoff into column
                        NAME of the single-inspiral table. (default: None)

Options related to calculating the parameter space metric:
  --pn-order PN_ORDER   Determines the PN order to use. For a bank of non-
                        spinning templates, spin-related terms in the metric
                        will be zero. REQUIRED. Choices: * zeroPN: Will only
                        include the dominant term (proportional to chirp mass)
                        * onePN: Will only the leading orbit term and first
                        correction at 1PN * onePointFivePN: Will include orbit
                        and spin terms to 1.5PN. * twoPN: Will include orbit
                        and spin terms to 2PN. * twoPointFivePN: Will include
                        orbit and spin terms to 2.5PN. * threePN: Will include
                        orbit terms to 3PN and spin terms to 2.5PN. *
                        threePointFivePN: Include orbit terms to 3.5PN and
                        spin terms to 2.5PN (default: None)
  --f0 F0               f0 is used as a dynamic scaling factor when
                        calculating integrals used in metric construction.
                        I.e. instead of integrating F(f) we integrate F(f/f0)
                        then rescale by powers of f0. The default value 70Hz
                        should be fine for most applications. OPTIONAL.
                        UNITS=Hz. **WARNING: If the ethinca metric is to be
                        calculated, f0 must be set equal to f-low** (default:
                        70.0)
  --f-low F_LOW         Lower frequency cutoff used in computing the parameter
                        space metric. REQUIRED. UNITS=Hz (default: None)
  --f-upper F_UPPER     Upper frequency cutoff used in computing the parameter
  --delta-f DELTA_F     Frequency spacing used in computing the parameter
                        space metric: integrals of the form \int F(f) df are
                        approximated as \sum F(f) delta_f. REQUIRED. UNITS=Hz
  --write-metric        If given write the metric components to disk as they
                        are calculated. (default: False)

Options related to mass and spin limits for bank generation:
  --min-mass1 MIN_MASS1
                        Minimum mass1: must be >= min-mass2. REQUIRED.
                        UNITS=Solar mass (default: None)
  --max-mass1 MAX_MASS1
                        Maximum mass1: must be >= max-mass2. REQUIRED.
  --min-mass2 MIN_MASS2
                        Minimum mass2. REQUIRED. UNITS=Solar mass (default:
                        None)
  --max-mass2 MAX_MASS2
                        Maximum mass2. REQUIRED. UNITS=Solar mass (default:
                        None)
  --max-total-mass MAX_TOTAL_MASS
                        Maximum total mass. OPTIONAL, if not provided the max
                        total mass is determined by the component masses.
  --min-total-mass MIN_TOTAL_MASS
                        Minimum total mass. OPTIONAL, if not provided the min
  --max-chirp-mass MAX_CHIRP_MASS
                        Maximum chirp mass. OPTIONAL, if not provided the max
                        chirp mass is determined by the component masses.
  --min-chirp-mass MIN_CHIRP_MASS
  --max-eta MAX_ETA     Maximum symmetric mass ratio. OPTIONAL, no upper bound
                        on eta will be imposed if not provided. UNITS=Solar
                        mass. (default: 0.25)
  --min-eta MIN_ETA     Minimum symmetric mass ratio. OPTIONAL, no lower bound
                        mass. (default: 0.0)
  --ns-eos NS_EOS       Select the EOS to be used for the NS when calculating
                        the remnant disk mass. Only 2H is currently supported.
                        OPTIONAL (default: None)
  --remnant-mass-threshold REMNANT_MASS_THRESHOLD
                        Setting this filters EM dim NS-BH binaries: if the
                        remnant disk mass does not exceed this value, the NS-
                        BH binary is dropped from the target parameter space.
                        When it is set to None (default value) the EM dim
                        filter is not activated. OPTIONAL (default: None)
  --use-eos-max-ns-mass
                        Cut the mass range of the smaller object to the
                        maximum mass allowed by EOS. OPTIONAL (default: False)
  --delta-bh-spin DELTA_BH_SPIN
                        Grid spacing used for the BH spin z component when
                        generating the surface of the minumum minimum
                        symmetric mass ratio as a function of BH spin and NS
                        mass required to produce a remnant disk mass that
                        exceeds the threshold specificed in --remnant-mass-
                        threshold. OPTIONAL (0.1 by default) (default: None)
  --delta-ns-mass DELTA_NS_MASS
                        Grid spacing used for the NS mass when generating the
                        surface of the minumum minimum symmetric mass ratio as
                        a function of BH spin and NS mass required to produce
                        a remnant disk mass that exceeds the thrsehold
                        specified in --remnant-mass-threshold. OPTIONAL (0.1
                        by default) (default: None)

Options to select the method of PSD generation:
  The options --psd-model, --psd-file, --asd-file, and --psd-estimation are
  mutually exclusive.

  --psd-model {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,flat_unity}
                        Get PSD from given analytical model. (default: None)
  --psd-file PSD_FILE   Get PSD using given PSD ASCII file (default: None)
  --asd-file ASD_FILE   Get PSD using given ASD ASCII file (default: None)
  --psd-inverse-length PSD_INVERSE_LENGTH
                        (Optional) The maximum length of the impulse response
                        of the overwhitening filter (s) (default: None)
  --psd-file-xml-ifo-string PSD_FILE_XML_IFO_STRING
                        If using an XML PSD file, use the PSD in the file's
                        PSD dictionary with this ifo string. If not given and
                        only one PSD present in the file return that, if not
                        given and multiple (or zero) PSDs present an exception
                        will be raised. (default: None)
  --psd-file-xml-root-name PSD_FILE_XML_ROOT_NAME
                        If given use this as the root name for the PSD XML
                        file. If this means nothing to you, then it is
                        probably safe to ignore this option. (default: psd)
  --psdvar-segment SECONDS
                        Length of segment for mean square calculation of PSD
                        variation. (default: None)
  --psdvar-short-segment SECONDS
                        Length of short segment for outliers removal in PSD
                        variability calculation. (default: None)
  --psdvar-long-segment SECONDS
                        Length of long segment when calculating the PSD
                        variability. (default: None)
  --psdvar-psd-duration SECONDS
                        Duration of short segments for PSD estimation.
  --psdvar-psd-stride SECONDS
                        Separation between PSD estimation segments. (default:
                        None)
  --psdvar-low-freq HERTZ
                        Minimum frequency to consider in strain bandpass.
  --psdvar-high-freq HERTZ
                        Maximum frequency to consider in strain bandpass.
  --psd-estimation {mean,median,median-mean}
                        Measure PSD from the data, using given average method.
  --psd-segment-length PSD_SEGMENT_LENGTH
                        (Required for --psd-estimation) The segment length for
                        PSD estimation (s) (default: None)
  --psd-segment-stride PSD_SEGMENT_STRIDE
                        (Required for --psd-estimation) The separation between
                        consecutive segments (s) (default: None)
  --psd-num-segments PSD_NUM_SEGMENTS
                        (Optional, used only with --psd-estimation). If given,
                        PSDs will be estimated using only this number of
                        segments. If more data is given than needed to make
                        this number of segments then excess data will not be
                        used in the PSD estimate. If not enough data is given,
                        the code will fail. (default: None)
  --psd-output PSD_OUTPUT
                        (Optional) Write PSD to specified file (default: None)

Options for obtaining h(t):
  These options are used for generating h(t) either by reading from a file
  or by generating it. This is only needed if the PSD is to be estimated
  from the data, ie. if the --psd-estimation option is given.

  --gps-start-time GPS_START_TIME
                        The gps start time of the data (integer seconds)
  --gps-end-time GPS_END_TIME
                        The gps end time of the data (integer seconds)
  --strain-high-pass STRAIN_HIGH_PASS
                        High pass frequency (default: None)
  --pad-data PAD_DATA   Extra padding to remove highpass corruption (integer
                        seconds) (default: 8)
  --taper-data TAPER_DATA
                        Taper ends of data to zero using the supplied length
                        as a window (integer seconds) (default: 0)
  --sample-rate SAMPLE_RATE
                        The sample rate to use for h(t) generation (integer
                        Hz) (default: None)
  --channel-name CHANNEL_NAME
                        The channel containing the gravitational strain data
  --frame-cache FRAME_CACHE [FRAME_CACHE ...]
                        Cache file containing the frame locations. (default:
                        None)
  --frame-files FRAME_FILES [FRAME_FILES ...]
                        list of frame files (default: None)
  --hdf-store HDF_STORE
                        Store of time series data in hdf format (default:
                        None)
  --frame-type FRAME_TYPE
                        (optional), replaces frame-files. Use datafind to get
                        the needed frame file(s) of this type. (default: None)
  --frame-sieve FRAME_SIEVE
                        (optional), Only use frame files where the URL matches
                        the regular expression given. (default: None)
  --fake-strain {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,zeroNoise}
                        Name of model PSD for generating fake gaussian noise.
  --fake-strain-seed FAKE_STRAIN_SEED
                        Seed value for the generation of fake colored gaussian
                        noise (default: 0)
  --fake-strain-from-file FAKE_STRAIN_FROM_FILE
                        File containing ASD for generating fake noise from it.
  --fake-strain-flow FAKE_STRAIN_FLOW
                        Low frequency cutoff of the fake strain (default: 1.0)
  --fake-strain-filter-duration FAKE_STRAIN_FILTER_DURATION
                        Duration in seconds of the fake data coloring filter
                        (default: 128.0)
  --fake-strain-sample-rate FAKE_STRAIN_SAMPLE_RATE
                        Sample rate of the fake data generation (default:
                        16384)
  --injection-file INJECTION_FILE
                        (optional) Injection file containing parameters of CBC
                        signals to be added to the strain (default: None)
  --sgburst-injection-file SGBURST_INJECTION_FILE
                        (optional) Injection file containing parametersof
                        sine-Gaussian burst signals to add to the strain
  --injection-scale-factor INJECTION_SCALE_FACTOR
                        Divide injections by this factor before adding to the
                        strain data (default: 1)
  --injection-sample-rate INJECTION_SAMPLE_RATE
                        Sample rate to use for injections (integer Hz).
                        Typically similar to the strain data sample rate.If
                        not provided, the strain sample rate will be used
  --injection-f-ref INJECTION_F_REF
                        Reference frequency in Hz for creating CBC injections
                        from an XML file (default: None)
  --injection-f-final INJECTION_F_FINAL
                        Override the f_final field of a CBC XML injection file
                        (frequency in Hz) (default: None)
  --gating-file GATING_FILE
                        (optional) Text file of gating segments to apply.
                        Format of each line is (all values in seconds):
                        gps_time zeros_half_width pad_half_width (default:
                        None)
  --autogating-threshold SIGMA
                        If given, find and gate glitches producing a deviation
                        larger than SIGMA in the whitened strain time series.
  --autogating-max-iterations SIGMA
                        If given, iteratively apply autogating (default: 1)
  --autogating-cluster SECONDS
                        Length of clustering window for detecting glitches for
                        autogating. (default: 5.0)
  --autogating-width SECONDS
                        Half-width of the gating window. (default: 0.25)
  --autogating-taper SECONDS
                        Taper the strain before and after each gating window
                        over a duration of SECONDS. (default: 0.25)
  --autogating-pad SECONDS
                        Ignore the given length of whitened strain at the ends
                        of a segment, to avoid filters ringing. (default: 16)
  --normalize-strain NORMALIZE_STRAIN
                        (optional) Divide frame data by constant. (default:
                        None)
  --zpk-z ZPK_Z [ZPK_Z ...]
                        (optional) Zero-pole-gain (zpk) filter strain. A list
                        of zeros for transfer function (default: None)
  --zpk-p ZPK_P [ZPK_P ...]
                        of poles for transfer function (default: None)
  --zpk-k ZPK_K         (optional) Zero-pole-gain (zpk) filter strain.
                        Transfer function gain (default: None)
  --witness-frame-type WITNESS_FRAME_TYPE
                        (optional), frame type which will be use to query the
                        witness channel data. (default: None)
  --witness-tf-file WITNESS_TF_FILE
                        an hdf file containing the transfer functions and the
                        associated channel names (default: None)
  --witness-filter-length WITNESS_FILTER_LENGTH
                        filter length in seconds for the transfer function

Ethinca metric options:
  Options used in the calculation of Gamma metric components for the ethinca
  coincidence test and for assigning high-frequency cutoffs to templates.

  --calculate-time-metric-components
                        If given, the ethinca metric will be calculated for
                        only the time component, and stored in the Gamma0
                        entry of the sngl_inspiral table. OPTIONAL,
                        default=False (default: False)
  --calculate-ethinca-metric
                        If given, the ethinca metric will be calculated and
                        stored in the Gamma entries of the sngl_inspiral
                        table. OPTIONAL, default=False (default: False)
  --ethinca-pn-order {zeroPN,onePN,onePointFivePN,twoPN,twoPointFivePN,threePN,threePointFivePN}
                        Specify a PN order to be used in calculating the
                        ethinca metric. OPTIONAL: if not specified, the same
                        order will be used as for the bank metric. (default:
                        None)
  --filter-cutoff {SchwarzISCO,LightRing,ERD,BKLISCO,FRD,LRD,MECO,HybridMECO,IMRPhenomBFinal,IMRPhenomCFinal,IMRPhenomDPeak,EOBNRv2RD,EOBNRv2HMRD,SEOBNRv1RD,SEOBNRv1Peak,SEOBNRv2RD,SEOBNRv2Peak,SEOBNRv4RD,SEOBNRv4Peak}
                        Specify an upper frequency cutoff formula for the
                        ethinca metric calculation, and for the values of
                        f_final assigned to the templates. REQUIRED if the
                        calculate-ethinca-metric option is given. (default:
                        None)
  --ethinca-frequency-step ETHINCA_FREQUENCY_STEP
                        Control the precision of the upper frequency cutoff.
                        For speed, the metric is calculated only for discrete
                        f_max values with a spacing given by this option. Each
                        template is assigned the metric for the f_max closest
                        to its analytical cutoff formula. OPTIONAL,
                        default=10. UNITS=Hz (default: 10.0)

usage: pycbc_geom_aligned_bank [-h] [--version] [-V] [-s STACK_DISTANCE] [-3]
                               [-S SPLIT_BANK_NUM] [-F]
                               [--random-seed RANDOM_SEED]
                               [--print-chi-points FILENAME]
                               [--workflow-name WORKFLOW_NAME]
                               [--dax-filename DAX_FILENAME]
                               [--map-filename MAP_FILENAME]
                               [--transformation-catalog-filename TRANSFORMATION_CATALOG_FILENAME]
                               --intermediate-data-file INTERMEDIATE_DATA_FILE
                               --metadata-file METADATA_FILE
                               [--is-sub-workflow]
                               [--storage-path-base STORAGE_PATH_BASE] -m
                               MIN_MATCH -O OUTPUT_FILE [--f-low-column NAME]
                               --pn-order PN_ORDER [--f0 F0] --f-low F_LOW
                               --f-upper F_UPPER --delta-f DELTA_F
                               [--write-metric] --min-mass1 MIN_MASS1
                               --max-mass1 MAX_MASS1 --min-mass2 MIN_MASS2
                               --max-mass2 MAX_MASS2
                               [--max-total-mass MAX_TOTAL_MASS]
                               [--min-total-mass MIN_TOTAL_MASS]
                               [--max-chirp-mass MAX_CHIRP_MASS]
                               [--min-chirp-mass MIN_CHIRP_MASS]
                               [--max-eta MAX_ETA] [--min-eta MIN_ETA]
                               [--ns-eos NS_EOS]
                               [--remnant-mass-threshold REMNANT_MASS_THRESHOLD]
                               [--use-eos-max-ns-mass]
                               [--delta-bh-spin DELTA_BH_SPIN]
                               [--delta-ns-mass DELTA_NS_MASS]
                               [--max-ns-spin-mag MAX_NS_SPIN_MAG]
                               [--max-bh-spin-mag MAX_BH_SPIN_MAG]
                               [--ns-bh-boundary-mass NS_BH_BOUNDARY_MASS | --nsbh-flag]
                               [--psd-model {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,flat_unity}]
                               [--psd-file PSD_FILE] [--asd-file ASD_FILE]
                               [--psd-inverse-length PSD_INVERSE_LENGTH]
                               [--psd-file-xml-ifo-string PSD_FILE_XML_IFO_STRING]
                               [--psd-file-xml-root-name PSD_FILE_XML_ROOT_NAME]
                               [--psdvar-segment SECONDS]
                               [--psdvar-short-segment SECONDS]
                               [--psdvar-long-segment SECONDS]
                               [--psdvar-psd-duration SECONDS]
                               [--psdvar-psd-stride SECONDS]
                               [--psdvar-low-freq HERTZ]
                               [--psdvar-high-freq HERTZ]
                               [--psd-estimation {mean,median,median-mean}]
                               [--psd-segment-length PSD_SEGMENT_LENGTH]
                               [--psd-segment-stride PSD_SEGMENT_STRIDE]
                               [--psd-num-segments PSD_NUM_SEGMENTS]
                               [--psd-output PSD_OUTPUT]
                               [--gps-start-time GPS_START_TIME]
                               [--gps-end-time GPS_END_TIME]
                               [--strain-high-pass STRAIN_HIGH_PASS]
                               [--pad-data PAD_DATA] [--taper-data TAPER_DATA]
                               [--sample-rate SAMPLE_RATE]
                               [--channel-name CHANNEL_NAME]
                               [--frame-cache FRAME_CACHE [FRAME_CACHE ...]]
                               [--frame-files FRAME_FILES [FRAME_FILES ...]]
                               [--hdf-store HDF_STORE]
                               [--frame-type FRAME_TYPE]
                               [--frame-sieve FRAME_SIEVE]
                               [--fake-strain {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,zeroNoise}]
                               [--fake-strain-seed FAKE_STRAIN_SEED]
                               [--fake-strain-from-file FAKE_STRAIN_FROM_FILE]
                               [--fake-strain-flow FAKE_STRAIN_FLOW]
                               [--fake-strain-filter-duration FAKE_STRAIN_FILTER_DURATION]
                               [--fake-strain-sample-rate FAKE_STRAIN_SAMPLE_RATE]
                               [--injection-file INJECTION_FILE]
                               [--sgburst-injection-file SGBURST_INJECTION_FILE]
                               [--injection-scale-factor INJECTION_SCALE_FACTOR]
                               [--injection-sample-rate INJECTION_SAMPLE_RATE]
                               [--injection-f-ref INJECTION_F_REF]
                               [--injection-f-final INJECTION_F_FINAL]
                               [--gating-file GATING_FILE]
                               [--autogating-threshold SIGMA]
                               [--autogating-max-iterations SIGMA]
                               [--autogating-cluster SECONDS]
                               [--autogating-width SECONDS]
                               [--autogating-taper SECONDS]
                               [--autogating-pad SECONDS]
                               [--normalize-strain NORMALIZE_STRAIN]
                               [--zpk-z ZPK_Z [ZPK_Z ...]]
                               [--zpk-p ZPK_P [ZPK_P ...]] [--zpk-k ZPK_K]
                               [--witness-frame-type WITNESS_FRAME_TYPE]
                               [--witness-tf-file WITNESS_TF_FILE]
                               [--witness-filter-length WITNESS_FILTER_LENGTH]
                               [--calculate-time-metric-components | --calculate-ethinca-metric]
                               [--ethinca-pn-order {zeroPN,onePN,onePointFivePN,twoPN,twoPointFivePN,threePN,threePointFivePN}]
                               [--filter-cutoff {SchwarzISCO,LightRing,ERD,BKLISCO,FRD,LRD,MECO,HybridMECO,IMRPhenomBFinal,IMRPhenomCFinal,IMRPhenomDPeak,EOBNRv2RD,EOBNRv2HMRD,SEOBNRv1RD,SEOBNRv1Peak,SEOBNRv2RD,SEOBNRv2Peak,SEOBNRv4RD,SEOBNRv4Peak}]
                               [--ethinca-frequency-step ETHINCA_FREQUENCY_STEP]

Aligned spin bank generator. Initial placement of template points, and
rejection of "too far away" templates is done in the code, but the final
translation between points in the chirp parameters parameter space is done in
a workflow.

  -s STACK_DISTANCE, --stack-distance STACK_DISTANCE
                        Minimum metric spacing before we stack. (default: 0.2)
  -3, --threed-lattice  Set this to use a 3D lattice. OPTIONAL (default:
                        False)
  -S SPLIT_BANK_NUM, --split-bank-num SPLIT_BANK_NUM
                        Number of points per job in dag. OPTIONAL (default:
                        100)
  -F, --filter-points   Remove nearby points before generating the bank.
                        (default: False)
                        Random seed to use whenever the numpy random functions
                        are called when doing the monte-carlo for obtaining
                        the principal components and when translating all
                        points back to physical space. If this is used the
                        code should give the same output if run with the same
                        random seed. (default: None)
  --print-chi-points FILENAME
                        Add a node to print off an ASCII list of mass
                        parameters and corresponding location in the xi space
                        using pycbc_tmpltbank_to_chi_params. This will be
                        written to FILENAME. If this argument is not given, no
                        chi points file will be written. (default: None)
  --workflow-name WORKFLOW_NAME
                        Name to use for the produced .dax file. (default:
                        bank_gen)
  --dax-filename DAX_FILENAME
                        This can be used if running this job in a sub-workflow
                        to specify the dax filename. (default: None)
  --map-filename MAP_FILENAME
                        to specify the output map filename. WARNING: Giving
                        this if not running as a sub-workflow will cause
                        pycbc_submit_dax to not work. (default: None)
  --transformation-catalog-filename TRANSFORMATION_CATALOG_FILENAME
                        See workflow/core.py (default: None)
  --intermediate-data-file INTERMEDIATE_DATA_FILE
                        The HDF file to be used to store data to pass down to
                        the various worker jobs. (default: None)
  --metadata-file METADATA_FILE
                        Location of the output file containing the metadata
                        that will be added to the final XML file. (default:
                        None)
  --is-sub-workflow     Only give this option if this code is being run as a
                        sub-workflow within pegasus. If this means nothing to
                        you, do not give this option. (default: False)
  --storage-path-base STORAGE_PATH_BASE
                        If running this code as a sub-workflow then this path
                        is pretended to all storage directories. (default:
                        None)

                        70.0)

                        None)
                        None)
  --max-ns-spin-mag MAX_NS_SPIN_MAG
                        Maximum neutron star spin magnitude. Neutron stars are
                        defined as components lighter than the NS-BH boundary
                        (3 Msun by default). REQUIRED if min-mass2 < ns-bh-
                        boundary-mass (default: None)
  --max-bh-spin-mag MAX_BH_SPIN_MAG
                        Maximum black hole spin magnitude. Black holes are
                        defined as components at or above the NS-BH boundary
                        (3 Msun by default). REQUIRED if max-mass1 >= ns-bh-
  --ns-bh-boundary-mass NS_BH_BOUNDARY_MASS
                        Mass boundary between neutron stars and black holes.
                        Components below this mass are considered neutron
                        stars and are subject to the neutron star spin limits.
                        Components at/above are subject to the black hole spin
                        limits. OPTIONAL, default=3.000000. UNITS=Solar mass
  --nsbh-flag           Set this flag if generating a bank that contains only
                        systems with 1 black hole and 1 neutron star. With
                        this flag set the heavier body will always be subject
                        to the black hole spin restriction and the lighter to
                        the neutron star spin restriction, regardless of mass.
                        OPTIONAL. If set, the value of --ns-bh-boundary-mass
                        will be ignored. (default: False)

                        None)

                        None)
                        None)
                        16384)
                        None)
                        None)

                        None)
                        None)

usage: pycbc_aligned_stoch_bank [-h] [--version] [--verbose] [-V]
                                [--bank-fupper-step BANK_FUPPER_STEP]
                                [--bank-fupper-formula {SchwarzISCO,LightRing,ERD,BKLISCO,FRD,LRD,MECO,HybridMECO,IMRPhenomBFinal,IMRPhenomCFinal,IMRPhenomDPeak,EOBNRv2RD,EOBNRv2HMRD,SEOBNRv1RD,SEOBNRv1Peak,SEOBNRv2RD,SEOBNRv2Peak,SEOBNRv4RD,SEOBNRv4Peak}]
                                [-N NUM_SEEDS] [-n NUM_FAILED_CUTOFF]
                                [--random-seed RANDOM_SEED] -m MIN_MATCH -O
                                OUTPUT_FILE [--f-low-column NAME] --pn-order
                                PN_ORDER [--f0 F0] --f-low F_LOW --f-upper
                                F_UPPER --delta-f DELTA_F [--write-metric]
                                --min-mass1 MIN_MASS1 --max-mass1 MAX_MASS1
                                --min-mass2 MIN_MASS2 --max-mass2 MAX_MASS2
                                [--max-total-mass MAX_TOTAL_MASS]
                                [--min-total-mass MIN_TOTAL_MASS]
                                [--max-chirp-mass MAX_CHIRP_MASS]
                                [--min-chirp-mass MIN_CHIRP_MASS]
                                [--max-eta MAX_ETA] [--min-eta MIN_ETA]
                                [--ns-eos NS_EOS]
                                [--remnant-mass-threshold REMNANT_MASS_THRESHOLD]
                                [--use-eos-max-ns-mass]
                                [--delta-bh-spin DELTA_BH_SPIN]
                                [--delta-ns-mass DELTA_NS_MASS]
                                [--max-ns-spin-mag MAX_NS_SPIN_MAG]
                                [--max-bh-spin-mag MAX_BH_SPIN_MAG]
                                [--ns-bh-boundary-mass NS_BH_BOUNDARY_MASS | --nsbh-flag]
                                [--psd-model {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,flat_unity}]
                                [--psd-file PSD_FILE] [--asd-file ASD_FILE]
                                [--psd-inverse-length PSD_INVERSE_LENGTH]
                                [--psd-file-xml-ifo-string PSD_FILE_XML_IFO_STRING]
                                [--psd-file-xml-root-name PSD_FILE_XML_ROOT_NAME]
                                [--psdvar-segment SECONDS]
                                [--psdvar-short-segment SECONDS]
                                [--psdvar-long-segment SECONDS]
                                [--psdvar-psd-duration SECONDS]
                                [--psdvar-psd-stride SECONDS]
                                [--psdvar-low-freq HERTZ]
                                [--psdvar-high-freq HERTZ]
                                [--psd-estimation {mean,median,median-mean}]
                                [--psd-segment-length PSD_SEGMENT_LENGTH]
                                [--psd-segment-stride PSD_SEGMENT_STRIDE]
                                [--psd-num-segments PSD_NUM_SEGMENTS]
                                [--psd-output PSD_OUTPUT]
                                [--gps-start-time GPS_START_TIME]
                                [--gps-end-time GPS_END_TIME]
                                [--strain-high-pass STRAIN_HIGH_PASS]
                                [--pad-data PAD_DATA]
                                [--taper-data TAPER_DATA]
                                [--sample-rate SAMPLE_RATE]
                                [--channel-name CHANNEL_NAME]
                                [--frame-cache FRAME_CACHE [FRAME_CACHE ...]]
                                [--frame-files FRAME_FILES [FRAME_FILES ...]]
                                [--hdf-store HDF_STORE]
                                [--frame-type FRAME_TYPE]
                                [--frame-sieve FRAME_SIEVE]
                                [--fake-strain {AdVBNSOptimizedSensitivityP1200087,AdVDesignSensitivityP1200087,AdVEarlyHighSensitivityP1200087,AdVEarlyLowSensitivityP1200087,AdVLateHighSensitivityP1200087,AdVLateLowSensitivityP1200087,AdVMidHighSensitivityP1200087,AdVMidLowSensitivityP1200087,AdvVirgo,CosmicExplorerP1600143,CosmicExplorerPessimisticP1600143,CosmicExplorerWidebandP1600143,EinsteinTelescopeP1600143,GEO,GEOHF,KAGRA,KAGRADesignSensitivityT1600593,KAGRAEarlySensitivityT1600593,KAGRALateSensitivityT1600593,KAGRAMidSensitivityT1600593,KAGRAOpeningSensitivityT1600593,TAMA,Virgo,aLIGOAPlusDesignSensitivityT1800042,aLIGOAdVO3LowT1800545,aLIGOAdVO4IntermediateT1800545,aLIGOAdVO4T1800545,aLIGOBHBH20Deg,aLIGOBHBH20DegGWINC,aLIGOBNSOptimizedSensitivityP1200087,aLIGODesignSensitivityP1200087,aLIGOEarlyHighSensitivityP1200087,aLIGOEarlyLowSensitivityP1200087,aLIGOHighFrequency,aLIGOHighFrequencyGWINC,aLIGOKAGRA128MpcT1800545,aLIGOKAGRA25MpcT1800545,aLIGOKAGRA80MpcT1800545,aLIGOLateHighSensitivityP1200087,aLIGOLateLowSensitivityP1200087,aLIGOMidHighSensitivityP1200087,aLIGOMidLowSensitivityP1200087,aLIGONSNSOpt,aLIGONSNSOptGWINC,aLIGONoSRMHighPower,aLIGONoSRMLowPower,aLIGONoSRMLowPowerGWINC,aLIGOQuantumBHBH20Deg,aLIGOQuantumHighFrequency,aLIGOQuantumNSNSOpt,aLIGOQuantumNoSRMHighPower,aLIGOQuantumNoSRMLowPower,aLIGOQuantumZeroDetHighPower,aLIGOQuantumZeroDetLowPower,aLIGOThermal,aLIGOZeroDetHighPower,aLIGOZeroDetHighPowerGWINC,aLIGOZeroDetLowPower,aLIGOZeroDetLowPowerGWINC,aLIGOaLIGO140MpcT1800545,aLIGOaLIGO175MpcT1800545,aLIGOaLIGODesignSensitivityT1800044,aLIGOaLIGOO3LowT1800545,eLIGOModel,eLIGOShot,iLIGOModel,iLIGOSRD,iLIGOSeismic,iLIGOShot,iLIGOThermal,zeroNoise}]
                                [--fake-strain-seed FAKE_STRAIN_SEED]
                                [--fake-strain-from-file FAKE_STRAIN_FROM_FILE]
                                [--fake-strain-flow FAKE_STRAIN_FLOW]
                                [--fake-strain-filter-duration FAKE_STRAIN_FILTER_DURATION]
                                [--fake-strain-sample-rate FAKE_STRAIN_SAMPLE_RATE]
                                [--injection-file INJECTION_FILE]
                                [--sgburst-injection-file SGBURST_INJECTION_FILE]
                                [--injection-scale-factor INJECTION_SCALE_FACTOR]
                                [--injection-sample-rate INJECTION_SAMPLE_RATE]
                                [--injection-f-ref INJECTION_F_REF]
                                [--injection-f-final INJECTION_F_FINAL]
                                [--gating-file GATING_FILE]
                                [--autogating-threshold SIGMA]
                                [--autogating-max-iterations SIGMA]
                                [--autogating-cluster SECONDS]
                                [--autogating-width SECONDS]
                                [--autogating-taper SECONDS]
                                [--autogating-pad SECONDS]
                                [--normalize-strain NORMALIZE_STRAIN]
                                [--zpk-z ZPK_Z [ZPK_Z ...]]
                                [--zpk-p ZPK_P [ZPK_P ...]] [--zpk-k ZPK_K]
                                [--witness-frame-type WITNESS_FRAME_TYPE]
                                [--witness-tf-file WITNESS_TF_FILE]
                                [--witness-filter-length WITNESS_FILTER_LENGTH]
                                [--calculate-time-metric-components | --calculate-ethinca-metric]
                                [--ethinca-pn-order {zeroPN,onePN,onePointFivePN,twoPN,twoPointFivePN,threePN,threePointFivePN}]
                                [--filter-cutoff {SchwarzISCO,LightRing,ERD,BKLISCO,FRD,LRD,MECO,HybridMECO,IMRPhenomBFinal,IMRPhenomCFinal,IMRPhenomDPeak,EOBNRv2RD,EOBNRv2HMRD,SEOBNRv1RD,SEOBNRv1Peak,SEOBNRv2RD,SEOBNRv2Peak,SEOBNRv4RD,SEOBNRv4Peak}]
                                [--ethinca-frequency-step ETHINCA_FREQUENCY_STEP]

Stochastic aligned spin bank generator.

  --verbose             verbose output (default: False)
  -V, --vary-fupper     Use a variable upper frequency cutoff in laying out
                        the bank. OPTIONAL. (default: False)
  --bank-fupper-step BANK_FUPPER_STEP
                        Size of discrete frequency steps used when varying the
                        fupper. If --calculate-ethinca-metric and --ethinca-
                        freq-step are also given, the code will use the
                        smaller of the two step values. OPTIONAL. Units=Hz
                        (default: 10.0)
  --bank-fupper-formula {SchwarzISCO,LightRing,ERD,BKLISCO,FRD,LRD,MECO,HybridMECO,IMRPhenomBFinal,IMRPhenomCFinal,IMRPhenomDPeak,EOBNRv2RD,EOBNRv2HMRD,SEOBNRv1RD,SEOBNRv1Peak,SEOBNRv2RD,SEOBNRv2Peak,SEOBNRv4RD,SEOBNRv4Peak}
                        Frequency cutoff formula for varying fupper.
                        Frequencies will be rounded to the nearest discrete
                        step. OPTIONAL. (default: SchwarzISCO)
  -N NUM_SEEDS, --num-seeds NUM_SEEDS
                        Number of seed points used in bank construction.
                        OPTIONAL. (default: 5000000)
  -n NUM_FAILED_CUTOFF, --num-failed-cutoff NUM_FAILED_CUTOFF
                        Maximum number of consecutive, not-accepted test
                        points after which bank generation will be stopped.
                        OPTIONAL. Default value is really large as --num-seeds
                        is intended to provide the termination condition.
                        (default: 1000000000)
                        used in obtaining the principal components in
                        parameter space and when translating points back to
                        physical space. If given, the code should give the
                        same output when run with the same random seed.

                        70.0)

                        None)
                        None)

                        None)

                        None)
                        None)
                        16384)
                        None)
                        None)

                        None)
                        None)

usage: pycbc_make_sbank_workflow [-h] [--version]
                                 [--workflow-name WORKFLOW_NAME]
                                 [-d OUTPUT_DIR] [--output-file OUTPUT_FILE]
                                 [--dax-filename DAX_FILENAME]
                                 [--map-filename MAP_FILENAME]
                                 [--transformation-catalog-filename TRANSFORMATION_CATALOG_FILENAME]
                                 [--is-sub-workflow]
                                 [--tags [TAGS [TAGS ...]]]
                                 [--config-files CONFIGFILE [CONFIGFILE ...]]
                                 [--config-overrides [SECTION:OPTION:VALUE [SECTION:OPTION:VALUE ...]]]
                                 [--config-delete [SECTION:OPTION [SECTION:OPTION ...]]]

Workflow generator for the lalapps_cbc_sbank template bank generation. This is
intended to be standalone, without putting things like the SbankExecutable
class in the pycbc.workflow module, to give an illustration of how a simple
workflow is constructed with pycbc.workflow.

                        Descriptive name of the analysis.
  -d OUTPUT_DIR, --output-dir OUTPUT_DIR
                        Path to output directory.
  --output-file OUTPUT_FILE
                        Specify the output file name. Either a name can be
                        provided or a full path to file. Is this is not given
                        a filename and location is chosen automatically.
                        to specify the dax filename.
                        pycbc_submit_dax to not work.
                        See workflow/core.py
                        you, do not give this option.
  --tags [TAGS [TAGS ...]]
                        If this option is given all jobs, and all workflow
                        configuration options, will use the tags given here.
                        This can be used if running this as a sub-workflow to
                        give two sets of options to two different invocations
                        of the sbank workflow.

Configuration:
  Options needed for parsing config file(s).

  --config-files CONFIGFILE [CONFIGFILE ...]
                        List of config files to be used in analysis.
  --config-overrides [SECTION:OPTION:VALUE [SECTION:OPTION:VALUE ...]]
                        List of section,option,value combinations to add into
                        the configuration file. Normally the gps start and end
                        times might be provided this way, and user specific
                        locations (ie. output directories). This can also be
                        provided as SECTION:OPTION or SECTION:OPTION: both of
                        which indicate that the corresponding value is left
                        blank.
  --config-delete [SECTION:OPTION [SECTION:OPTION ...]]
                        List of section,option combinations to delete from the
                        configuration file. This can also be provided as
                        SECTION which deletes the enture section from the
                        configuration file or SECTION:OPTION which deletes a
                        specific option from a given section.

usage: pycbc_make_uberbank_workflow [-h] [--version]
                                    [--workflow-name WORKFLOW_NAME]
                                    [-d OUTPUT_DIR]
                                    [--config-files CONFIGFILE [CONFIGFILE ...]]
                                    [--config-overrides [SECTION:OPTION:VALUE [SECTION:OPTION:VALUE ...]]]
                                    [--config-delete [SECTION:OPTION [SECTION:OPTION ...]]]

Workflow generator for the 'uberbank' template bank construction. The script
as written runs a configurable example of pycbc_geom_aligned_bank and, unless
instructed otherwise, another idependent one of sbank. Typically, a standard
uberbank uses this round of sbank to cover BBH sources, while a PyGRB uberbank
does not require performing this step: this step can be avoided with the
option skip-coarse-bank = in the section [workflow-bank_structure]. The script
then combines these two banks together, if the BBH sbank was indeed generated.
Finally, it runs a second example of sbank (configuration file tag: [sbank-
final]) on the resulting output. It is foreseen that this script would be
altered to generate workflows in slightly different configurations to the one
described, so hopefully the script is clear enough to allow someone to do
that.

                        blank.

['TaylorT1', 'TaylorT2', 'TaylorT3', 'SpinTaylorT1', 'SpinTaylorT4', 'SpinTaylorT5', 'PhenSpinTaylor', 'PhenSpinTaylorRD', 'EOBNRv2', 'EOBNRv2HM', 'TEOBResum_ROM', 'SEOBNRv1', 'SEOBNRv2', 'SEOBNRv2_opt', 'SEOBNRv3', 'SEOBNRv3_pert', 'SEOBNRv3_opt', 'SEOBNRv3_opt_rk4', 'SEOBNRv4', 'SEOBNRv4_opt', 'SEOBNRv4P', 'SEOBNRv4PHM', 'SEOBNRv2T', 'SEOBNRv4T', 'SEOBNRv4_ROM_NRTidalv2', 'SEOBNRv4_ROM_NRTidalv2_NSBH', 'HGimri', 'IMRPhenomA', 'IMRPhenomB', 'IMRPhenomC', 'IMRPhenomD', 'IMRPhenomD_NRTidalv2', 'IMRPhenomNSBH', 'IMRPhenomHM', 'IMRPhenomPv2', 'IMRPhenomPv2_NRTidal', 'IMRPhenomPv2_NRTidalv2', 'TaylorEt', 'TaylorT4', 'EccentricTD', 'SpinDominatedWf', 'NR_hdf5', 'NRSur7dq2', 'NRSur7dq4', 'SEOBNRv4HM', 'NRHybSur3dq8', 'IMRPhenomXAS', 'IMRPhenomXHM', 'IMRPhenomPv3', 'IMRPhenomPv3HM', 'IMRPhenomXP', 'IMRPhenomXPHM', 'TEOBResumS', 'IMRPhenomT', 'IMRPhenomTHM', 'TaylorF2', 'SEOBNRv1_ROM_EffectiveSpin', 'SEOBNRv1_ROM_DoubleSpin', 'SEOBNRv2_ROM_EffectiveSpin', 'SEOBNRv2_ROM_DoubleSpin', 'EOBNRv2_ROM', 'EOBNRv2HM_ROM', 'SEOBNRv2_ROM_DoubleSpin_HI', 'SEOBNRv4_ROM', 'SEOBNRv4HM_ROM', 'IMRPhenomD_NRTidal', 'SpinTaylorF2', 'TaylorF2NL', 'PreTaylorF2', 'SpinTaylorF2_SWAPPER', 'IMRPhenomDecc']
['EccentricFD', 'TaylorF2', 'TaylorF2Ecc', 'TaylorF2NLTides', 'TaylorF2RedSpin', 'TaylorF2RedSpinTidal', 'SpinTaylorF2', 'EOBNRv2_ROM', 'EOBNRv2HM_ROM', 'SEOBNRv1_ROM_EffectiveSpin', 'SEOBNRv1_ROM_DoubleSpin', 'SEOBNRv2_ROM_EffectiveSpin', 'SEOBNRv2_ROM_DoubleSpin', 'SEOBNRv2_ROM_DoubleSpin_HI', 'Lackey_Tidal_2013_SEOBNRv2_ROM', 'SEOBNRv4_ROM', 'SEOBNRv4HM_ROM', 'SEOBNRv4_ROM_NRTidal', 'SEOBNRv4_ROM_NRTidalv2', 'SEOBNRv4_ROM_NRTidalv2_NSBH', 'SEOBNRv4T_surrogate', 'IMRPhenomA', 'IMRPhenomB', 'IMRPhenomC', 'IMRPhenomD', 'IMRPhenomD_NRTidal', 'IMRPhenomD_NRTidalv2', 'IMRPhenomNSBH', 'IMRPhenomHM', 'IMRPhenomP', 'IMRPhenomPv2', 'IMRPhenomPv2_NRTidal', 'IMRPhenomPv2_NRTidalv2', 'SpinTaylorT4Fourier', 'SpinTaylorT5Fourier', 'NRSur4d2s', 'IMRPhenomXAS', 'IMRPhenomXHM', 'IMRPhenomPv3', 'IMRPhenomPv3HM', 'IMRPhenomXP', 'IMRPhenomXPHM', 'SpinTaylorF2_SWAPPER', 'TaylorF2NL', 'PreTaylorF2', 'multiband', 'IMRPhenomDecc', 'TaylorF2_INTERP', 'SpinTaylorT5', 'SEOBNRv1_ROM_EffectiveSpin_INTERP', 'SEOBNRv1_ROM_DoubleSpin_INTERP', 'SEOBNRv2_ROM_EffectiveSpin_INTERP', 'SEOBNRv2_ROM_DoubleSpin_INTERP', 'EOBNRv2_ROM_INTERP', 'EOBNRv2HM_ROM_INTERP', 'SEOBNRv2_ROM_DoubleSpin_HI_INTERP', 'SEOBNRv4_ROM_INTERP', 'SEOBNRv4HM_ROM_INTERP', 'SEOBNRv4', 'SEOBNRv4P', 'IMRPhenomC_INTERP', 'IMRPhenomD_INTERP', 'IMRPhenomPv2_INTERP', 'IMRPhenomD_NRTidal_INTERP', 'IMRPhenomPv2_NRTidal_INTERP', 'IMRPhenomHM_INTERP', 'IMRPhenomPv3HM_INTERP', 'IMRPhenomXHM_INTERP', 'IMRPhenomXPHM_INTERP', 'SpinTaylorF2_INTERP', 'TaylorF2NL_INTERP', 'PreTaylorF2_INTERP', 'SpinTaylorF2_SWAPPER_INTERP', 'IMRPhenomDecc_INTERP']

usage: 

Find single detector gravitational-wave triggers.

  --version
  -V, --verbose         print extra debugging information
  --update-progress UPDATE_PROGRESS
                        updates a file 'progress.txt' with a value 0 .. 1.0
                        when this amount of (filtering) progress was made
  --update-progress-file UPDATE_PROGRESS_FILE
                        name of the file to write the amount of (filtering)
                        progress to
  --output OUTPUT       FIXME: ADD
  --bank-file BANK_FILE
                        FIXME: ADD
  --snr-threshold SNR_THRESHOLD
                        SNR threshold for trigger generation
  --newsnr-threshold THRESHOLD
                        Cut triggers with NewSNR less than THRESHOLD
  --low-frequency-cutoff LOW_FREQUENCY_CUTOFF
                        The low frequency cutoff to use for filtering (Hz)
  --enable-bank-start-frequency
                        Read the starting frequency of template waveforms from
                        the template bank.
  --max-template-length MAX_TEMPLATE_LENGTH
                        The maximum length of a template is seconds. The
                        starting frequency of the template is modified to
                        ensure the proper length
  --enable-q-transform  compute the q-transform for each segment of a given
                        analysis run. (default = False)
  --approximant APPRX[:COND] [APPRX[:COND] ...]
                        The approximant(s) to use. Multiple approximants to
                        use in different regions may be provided. If multiple
                        approximants are provided, every one but the last must
                        be be followed by a conditional statement defining
                        where that approximant should be used. Conditionals
                        can be any boolean test understood by numpy. For
                        example, 'Apprx:(mtotal > 4) & (mchirp <= 5)' would
                        use approximant 'Apprx' where total mass is > 4 and
                        chirp mass is <= 5. Conditionals are applied in order,
                        with each successive one only applied to regions not
                        covered by previous arguments. For example,
                        `'TaylorF2:mtotal < 4' 'IMRPhenomD:mchirp < 3'` would
                        result in IMRPhenomD being used where chirp mass is <
                        3 and total mass is >= 4. The last approximant given
                        may use 'else' as the conditional or include no
                        conditional. In either case, this will cause the last
                        approximant to be used in any remaning regions after
                        all the previous conditionals have been applied. For
                        the full list of possible parameters to apply
                        conditionals to, see WaveformArray.default_fields().
                        Math operations may also be used on parameters; syntax
                        is python, with any operation recognized by numpy.
  --order {-1,0,1,2,3,4,5,6,7,8}
                        The integer half-PN order at which to generate the
                        approximant. Default is -1 which indicates to use
                        approximant defined default.
  --taper-template {start,end,startend}
                        For time-domain approximants, taper the start and/or
                        end of the waveform before FFTing.
  --cluster-method {template,window}
                        FIXME: ADD
  --cluster-function {findchirp,symmetric}
                        How to cluster together triggers within a window.
                        'findchirp' uses a forward sliding window; 'symmetric'
                        will compare each window to the one before and after,
                        keeping only a local maximum.
  --cluster-window CLUSTER_WINDOW
                        Length of clustering window in seconds. Set to 0 to
                        disable clustering.
  --bank-veto-bank-file BANK_VETO_BANK_FILE
                        FIXME: ADD
  --chisq-snr-threshold CHISQ_SNR_THRESHOLD
                        Minimum SNR to calculate the power chisq
  --chisq-bins CHISQ_BINS
                        Number of frequency bins to use for power chisq.
                        Specify an integer for a constant number of bins, or a
                        function of template attributes. Math functions are
                        allowed, ex.
                        '10./math.sqrt((params.mass1+params.mass2)/100.)'.
                        Non-integer values will be rounded down.
  --chisq-threshold CHISQ_THRESHOLD
                        FIXME: ADD
  --chisq-delta CHISQ_DELTA
                        FIXME: ADD
  --autochi-number-points AUTOCHI_NUMBER_POINTS
                        The number of points to use, in both directions
                        ifdoing a two-sided auto-chisq, to calculate theauto-
                        chisq statistic.
  --autochi-stride AUTOCHI_STRIDE
                        The gap, in sample points, between the points atwhich
                        to calculate auto-chisq.
  --autochi-two-phase   If given auto-chisq will be calculated by testing
                        against both phases of the SNR time-series. If not
                        given, only the phase matching the trigger will be
                        used.
  --autochi-onesided {left,right}
                        Decide whether to calculate auto-chisq usingpoints on
                        both sides of the trigger or only on oneside. If not
                        given points on both sides will beused. If given, with
                        either 'left' or 'right',only points on that side
                        (right = forward in time,left = back in time) will be
                        used.
  --autochi-reverse-template
                        If given, time-reverse the template beforecalculating
                        the auto-chisq statistic. This willcome at additional
                        computational cost as the SNRtime-series will need
                        recomputing for the time-reversed template.
  --autochi-max-valued  If given, store only the maximum value of the auto-
                        chisq over all points tested. A disadvantage of this
                        is that the mean value will not be known analytically.
  --autochi-max-valued-dof INT
                        If using --autochi-max-valued this value denotes the
                        pre-calculated mean value that will be stored as the
                        auto-chisq degrees-of-freedom value.
  --downsample-factor DOWNSAMPLE_FACTOR
                        Factor that determines the interval between the
                        initial SNR sampling. If not set (or 1) no sparse
                        sample is created, and the standard full SNR is
                        calculated.
  --upsample-threshold UPSAMPLE_THRESHOLD
                        The fraction of the SNR threshold to check the sparse
                        SNR sample.
  --upsample-method {pruned_fft}
                        The method to find the SNR points between the sparse
  --user-tag TAG        This is used to identify FULL_DATA jobs for
                        compatibility with pipedown post-processing. Option
                        will be removed when no longer needed.
  --keep-loudest-log-chirp-window KEEP_LOUDEST_LOG_CHIRP_WINDOW
                        Keep loudest triggers within ln chirp mass window
  --keep-loudest-interval KEEP_LOUDEST_INTERVAL
                        Window in seconds to maximize triggers over bank
  --keep-loudest-num KEEP_LOUDEST_NUM
                        Number of triggers to keep from each maximization
                        interval
  --keep-loudest-stat {snr,newsnr,new_snr,newsnr_sgveto,newsnr_sgveto_psdvar,newsnr_sgveto_psdvar_threshold,newsnr_sgveto_psdvar_scaled,newsnr_sgveto_psdvar_scaled_threshold}
                        Statistic used to determine loudest to keep
  --finalize-events-template-rate NUM TEMPLATES
                        After NUM TEMPLATES perform the various clustering and
                        rejection tests that would be performed at the end of
                        this job. Default is to only do those things at the
                        end of the job. This can help control memory usage if
                        a lot of triggers that would be rejected are being
                        retained. A suggested value for this is 500, but a
                        good number may depend on other settings and your
                        specific use-case.
  --gpu-callback-method GPU_CALLBACK_METHOD
  --use-compressed-waveforms
                        Use compressed waveforms from the bank file.
  --waveform-decompression-method WAVEFORM_DECOMPRESSION_METHOD
                        Method to be used decompress waveforms from the bank
                        file.
  --checkpoint-interval CHECKPOINT_INTERVAL
                        Save results to checkpoint file every X seconds.
                        Default is no checkpointing.
  --require-valid-checkpoint
                        If the checkpoint file is invalid, raise an error.
                        Default is to ignore invalid checkpoint files and to
                        delete the broken file.
  --checkpoint-exit-maxtime CHECKPOINT_EXIT_MAXTIME
                        Checkpoint and exit if X seconds of execution time is
                        exceeded. Default is no checkpointing.
  --checkpoint-exit-code CHECKPOINT_EXIT_CODE
                        Exit code returned if exiting after a checkpoint

                        Get PSD from given analytical model.
  --psd-file PSD_FILE   Get PSD using given PSD ASCII file
  --asd-file ASD_FILE   Get PSD using given ASD ASCII file
                        of the overwhitening filter (s)
                        will be raised.
                        probably safe to ignore this option.
                        variation.
                        variability calculation.
                        variability.
                        Separation between PSD estimation segments.
                        PSD estimation (s)
                        consecutive segments (s)
                        the code will fail.
                        (Optional) Write PSD to specified file

                        High pass frequency
                        seconds)
                        as a window (integer seconds)
                        Hz)
                        Cache file containing the frame locations.
                        list of frame files
                        Store of time series data in hdf format
                        the needed frame file(s) of this type.
                        the regular expression given.
                        noise
                        Low frequency cutoff of the fake strain
                        Sample rate of the fake data generation
                        signals to be added to the strain
                        strain data
                        from an XML file
                        (frequency in Hz)
                        gps_time zeros_half_width pad_half_width
                        If given, iteratively apply autogating
                        autogating.
                        Half-width of the gating window.
                        over a duration of SECONDS.
                        of a segment, to avoid filters ringing.
                        (optional) Divide frame data by constant.
                        of zeros for transfer function
                        of poles for transfer function
                        Transfer function gain
                        witness channel data.
                        associated channel names

Options for segmenting the strain:
  These options are used to determine how to segment the strain into smaller
  chunks, and for determining the portion of each to analyze for triggers.

  --trig-start-time TRIG_START_TIME
                        (optional) The gps time to start recording triggers
  --trig-end-time TRIG_END_TIME
                        (optional) The gps time to stop recording triggers
  --segment-length SEGMENT_LENGTH
                        The length of each strain segment in seconds.
  --segment-start-pad SEGMENT_START_PAD
                        The time in seconds to ignore of the beginning of each
                        segment in seconds.
  --segment-end-pad SEGMENT_END_PAD
                        The time in seconds to ignore at the end of each
  --allow-zero-padding  Allow for zero padding of data to analyze requested
                        times, if needed.
  --filter-inj-only     Analyze only segments that contain an injection.
  --injection-window INJECTION_WINDOW
                        If using --filter-inj-only then only search for
                        injections within +/- injection window of the
                        injections's end time. This is useful to speed up a
                        coherent search or a search where we initially filter
                        at lower sample rate, and then filter at full rate
                        where needed. NOTE: Reverts to full analysis if two
                        injections are in the same segment.

Options for selecting the processing scheme in this program.:
  --processing-scheme PROCESSING_SCHEME
                        The choice of processing scheme. Choices are ['mkl',
                        'numpy', 'cuda', 'cpu']. (optional for CPU scheme) The
                        number of execution threads can be indicated by
                        cpu:NUM_THREADS, where NUM_THREADS is an integer. The
                        default is a single thread. If the scheme is provided
                        as cpu:env, the number of threads can be provided by
                        the PYCBC_NUM_THREADS environment variable. If the
                        environment variable is not set, the number of threads
                        matches the number of logical cores.
  --processing-device-id PROCESSING_DEVICE_ID
                        (optional) ID of GPU to use for accelerated processing

Options for selecting the FFT backend and controlling its performance in this program.:
  --fft-backends [FFT_BACKENDS [FFT_BACKENDS ...]]
                        Preference list of the FFT backends. Choices are:
                        ['fftw', 'mkl', 'numpy']
  --fftw-measure-level FFTW_MEASURE_LEVEL
                        Determines the measure level used in planning FFTW
                        FFTs; allowed values are: [0, 1, 2, 3]
  --fftw-threads-backend FFTW_THREADS_BACKEND
                        Give 'openmp', 'pthreads' or 'unthreaded' to specify
                        which threaded FFTW to use
  --fftw-input-float-wisdom-file FFTW_INPUT_FLOAT_WISDOM_FILE
                        Filename from which to read single-precision wisdom
  --fftw-input-double-wisdom-file FFTW_INPUT_DOUBLE_WISDOM_FILE
                        Filename from which to read double-precision wisdom
  --fftw-output-float-wisdom-file FFTW_OUTPUT_FLOAT_WISDOM_FILE
                        Filename to which to write single-precision wisdom
  --fftw-output-double-wisdom-file FFTW_OUTPUT_DOUBLE_WISDOM_FILE
                        Filename to which to write double-precision wisdom
  --fftw-import-system-wisdom
                        If given, call fftw[f]_import_system_wisdom()

Options for selecting optimization-specific settings:
  --cpu-affinity CPU_AFFINITY
                        A set of CPUs on which to run, specified in a format
                        suitable to pass to taskset.
  --cpu-affinity-from-env CPU_AFFINITY_FROM_ENV
                        The name of an enivornment variable containing a set
                        of CPUs on which to run, specified in a format

Options that, if injections are present in this run, are responsible for performing pre-checks between injections in the data being filtered and the current search template to determine if the template has any chance of actually detecting the injection. The parameters of this test are given by the various options below. The --injection-filter-rejector-chirp-time-window and --injection-filter-rejector-match-threshold options need to be provided if those tests are desired. Other options will take default values unless overriden. More details on these options follow.:
  --injection-filter-rejector-chirp-time-window INJECTION_FILTER_REJECTOR_CHIRP_TIME_WINDOW
                        If this value is not None and an injection file is
                        given then we will calculate the difference in chirp
                        time (tau_0) between the template and each injection
                        in the analysis segment. If the difference is greate
                        than this threshold for all injections then filtering
                        is not performed. By default this will be None.
  --injection-filter-rejector-match-threshold INJECTION_FILTER_REJECTOR_MATCH_THRESHOLD
                        provided then we will calculate a 'coarse match'
                        between the template and each injection in the
                        analysis segment. If the match is less than this
                        threshold for all injections then filtering is not
                        performed. Parameters for the 'coarse match' follow.
                        By default this value will be None.
  --injection-filter-rejector-coarsematch-deltaf INJECTION_FILTER_REJECTOR_COARSEMATCH_DELTAF
                        If injections are present and a match threshold is
                        provided, this option specifies the frequency spacing
                        that will be used for injections, templates and PSD
                        when computing the 'coarse match'. Templates will be
                        generated directly with this spacing. The PSD and
                        injections will be resampled.
  --injection-filter-rejector-coarsematch-fmax INJECTION_FILTER_REJECTOR_COARSEMATCH_FMAX
                        provided, this option specifies the maximum frequency
                        generated directly with this max frequency. The PSD
                        and injections' frequency series will be truncated.
  --injection-filter-rejector-seg-buffer INJECTION_FILTER_REJECTOR_SEG_BUFFER
                        If injections are present and either a match threshold
                        or a chirp-time window is given, we will determine if
                        injections are 'in' the specified analysis chunk by
                        using the end times. If this value is non-zero the
                        analysis chunk is extended on both sides by this
                        amount before determining if injections are within the
                        given window.
  --injection-filter-rejector-f-lower INJECTION_FILTER_REJECTOR_F_LOWER
                        or a chirp-time window is given, this value is used to
                        set the lower frequency for determine chirp times or
                        for calculating matches. If this value is None the
                        lower frequency used for the full matched-filter is
                        used. Otherwise this value is used.

Sine-Gaussian Chisq:
  --sgchisq-snr-threshold SGCHISQ_SNR_THRESHOLD
                        Minimum SNR threshold to use SG chisq
  --sgchisq-locations SGCHISQ_LOCATIONS [SGCHISQ_LOCATIONS ...]
                        Frequency offsets and quality factors of the sine-
                        Gaussians to use, format 'region-
                        boolean:q1-offset1,q2-offset2'. Offset is relative to
                        the end frequency of the approximant. Region is a
                        boolean expression selecting templates to apply the
                        sine-Gaussians to, ex. 'mtotal>40'

usage: pycbc_make_coinc_search_workflow [-h] [--version]
                                        [--workflow-name WORKFLOW_NAME]
                                        [-d OUTPUT_DIR]
                                        [--config-files CONFIGFILE [CONFIGFILE ...]]
                                        [--config-overrides [SECTION:OPTION:VALUE [SECTION:OPTION:VALUE ...]]]
                                        [--config-delete [SECTION:OPTION [SECTION:OPTION ...]]]

Program for running multi-detector workflow analysis through coincidence and
then generate post-processing and plots.

                        blank.

usage: pycbc_submit_dax [-h] --dax DAX [optional arguments]

required arguments:
  -d, --dax DAX           name of the dax file to plan

workflow submission required one of:
  -a, --accounting-group GROUP tagged string used for site 
                               resource accounting.
  -F, --force-no-accounting-group submit without an accounting
                               group. Will cause condor submission
                               to fail on LIGO Data Grid clusters

  -h, --help              show this help message and exit
  -c, --cache-file FILE   replica cache file for data reuse
  -g, --local-staging-server uri://host provide a uri on host
                                 for the local site scratch
                                 (default is file://)
  -r, --remote-staging-server uri://host provide a uri on host
                                 for the remote site scratch.
  -p, --pegasus-properties FILE use the specified file as
                               the pegasus properties file
  -P, --append-pegasus-property STRING add the extra property
                                          specified by the argument
  -s, --execution-sites A,B,C specify a comma separated list
                               of execution sites that will be
                               used in addition to the local site
  -S, --staging-sites A=X,B=Y,C=Z  comma separated list of key=value
                                   pairs, where the key is the
                                   execution site and value is the
                                   staging site for that execution site
  -k, --append-site-profile SITE:NAMESPACE|KEY:VALUE
                               append the profile determined by
                               NAMESPACE, KEY, and VALUE to the
                               site catalog entry for SITE
  -t, --transformation-catalog FILE pass the specified
                                   transformation catalog to Pegasus
  -K, --no-create-proxy   Do not run ligo-proxy-init and assume
                             that the user has a valid grid proxy
  -n, --no-submit         Plan the DAX but do not submit it
  -l, --local-dir         Directory to put condor files under
  -G, --no-grid           Disable checks for grid proxy and
                             GLOBUS_LOCATION in pegasus-plan

If the environment variable TMPDIR is set then this is prepended to the 
path to the temporary workflow execute directory passed to pegasus-plan.
If the --local-dir option is not given.

If the environment variable PEGASUS_FILE_DIRECTORY is set then the
script will look there for pegasus site catalog and configuration
otherwise, the script will look for this directory by querying the
pycbc.workflow module.

If the environment variable PEGASUS_SITE_CATALOG_PATH is set then the
script will look for site catalog templates in this directory.
Site catalog templates may contain other environment variables that
must be set in order for them to be rendered correctly.

usage: pycbc_make_offline_grb_workflow [-h] [--version]
                                       [--config-files CONFIGFILE [CONFIGFILE ...]]
                                       [--config-overrides [SECTION:OPTION:VALUE [SECTION:OPTION:VALUE ...]]]
                                       [--config-delete [SECTION:OPTION [SECTION:OPTION ...]]]

                        blank.

usage: pycbc_splitbank [-h] [--version]
                       (--templates-per-bank SAMPLES | -n N | -O [OUTPUT_FILENAME [OUTPUT_FILENAME ...]])
                       [-o OUTPUT_PREFIX] [-z] [-V] -t INPUT_FILE
                       [--sort-frequency-cutoff SORT_FREQUENCY_CUTOFF]
                       [--sort-mchirp] [--random-sort]
                       [--random-seed RANDOM_SEED]

Splits a table in an xml file into multiple pieces

  --templates-per-bank SAMPLES
                        number of templates in the output banks
  -n N, --number-of-banks N
                        Split template bank into N files
  -O [OUTPUT_FILENAME [OUTPUT_FILENAME ...]], --output-filenames [OUTPUT_FILENAME [OUTPUT_FILENAME ...]]
                        Directly specify the names of the output files. The
                        number of files specified here will dictate how to
                        split the bank. It will be split equally between all
                        specified files.
  -o OUTPUT_PREFIX, --output-prefix OUTPUT_PREFIX
                        Prefix to add to the template bank name (name becomes
                        output#.xml[.gz])
  -z, --write-compress  Write compressed xml.gz files.
  -V, --verbose         Print extra debugging information
  -t INPUT_FILE, --bank-file INPUT_FILE
                        Template bank to split
  --sort-frequency-cutoff SORT_FREQUENCY_CUTOFF
                        Frequency cutoff to use for sorting the sub banks
  --sort-mchirp         Sort templates by chirp mass before splitting
  --random-sort         Sort templates randomly before splitting
                        Random seed to use when sorting randomly

from . frame import (locations_to_cache, read_frame, datafind_connection,
                     query_and_read_frame, frame_paths, write_frame,
                     DataBuffer, StatusBuffer)

from . store import (read_store)

# Status flags for the calibration state vector
# See e.g. https://dcc.ligo.org/LIGO-G1700234
# https://wiki.ligo.org/DetChar/DataQuality/O3Flags
HOFT_OK = 1
SCIENCE_INTENT = 2
SCIENCE_QUALITY = 4
HOFT_PROD = 8
FILTERS_OK = 16
NO_STOCH_HW_INJ = 32
NO_CBC_HW_INJ = 64
NO_BURST_HW_INJ = 128
NO_DETCHAR_HW_INJ = 256
KAPPA_A_OK = 512
KAPPA_PU_OK = 1024
KAPPA_TST_OK = 2048
KAPPA_C_OK = 4096
FCC_OK = 8192
NO_GAP = 16384
NO_HWINJ = NO_STOCH_HW_INJ | NO_CBC_HW_INJ | \
           NO_BURST_HW_INJ | NO_DETCHAR_HW_INJ

# relevant bits in the LIGO O2/O3 low-latency DQ vector
# If the bit is 0 then we should veto
# https://wiki.ligo.org/DetChar/DmtDqVector
OMC_DCPD_ADC_OVERFLOW = 2
ETMY_ESD_DAC_OVERFLOW = 4
ETMX_ESD_DAC_OVERFLOW = 16

# CAT1 bit in the Virgo state vector
# https://wiki.virgo-gw.eu/DetChar/DetCharVirgoStateVector
VIRGO_GOOD_DQ = 1 << 10

def flag_names_to_bitmask(flags):
    """Takes a list of flag names corresponding to bits in a status channel
    and returns the corresponding bit mask.
    """
    mask = 0
    for flag in flags:
        mask |= globals()[flag]
    return mask

from bisect import bisect_right
try:
    from fpconst import PosInf, NegInf
except ImportError:
    # fpconst is not part of the standard library and might not be available
    PosInf = float("+inf")
    NegInf = float("-inf")
import numpy
import math

class Bins(object):

    """
    Parent class for 1-dimensional binnings.

    Not intended to be used directly, but to be subclassed for use in real
    bins classes.
    """
    def __init__(self, minv, maxv, n):
        """
        Initialize a Bins instance.  The three arguments are the
        minimum and maximum of the values spanned by the bins, and
        the number of bins to place between them.  Subclasses may
        require additional arguments, or different arguments
        altogether.
        """
        # convenience code to do some common initialization and
        # input checking
        if not isinstance(n, int):
            raise TypeError(n)
        if n < 1:
            raise ValueError(n)
        if maxv <= minv:
            raise ValueError((minv, maxv))
        self.minv = minv
        self.maxv = maxv
        self.n = n

    def __len__(self):
        return self.n

    def __getitem__(self, x):
        """
        Convert a co-ordinate to a bin index.  The co-ordinate can
        be a single value, or a Python slice instance describing a
        range of values.  If a single value is given, it is mapped
        to the bin index corresponding to that value.  If a slice
        is given, it is converted to a slice whose lower bound is
        the index of the bin in which the slice's lower bound
        falls, and whose upper bound is 1 greater than the index of
        the bin in which the slice's upper bound falls.  Steps are
        not supported in slices.
        """
        if isinstance(x, slice):
            if x.step is not None:
                raise NotImplementedError("step not supported: %s" % repr(x))
            return slice(self[x.start] if x.start is not None
                         else 0, self[x.stop] + 1 if x.stop is not None
                         else len(self))
        raise NotImplementedError

    def __iter__(self):
        """
        If __iter__ does not exist, Python uses __getitem__ with
        range(0) as input to define iteration. This is nonsensical
        for bin objects, so explicitly unsupport iteration.
        """

    def lower(self):
        """
        Return an array containing the locations of the lower
        boundaries of the bins.
        """

    def centres(self):
        """
        Return an array containing the locations of the bin
        centres.
        """

    def upper(self):
        """
        Return an array containing the locations of the upper
        """

class IrregularBins(Bins):

    """
    Bins with arbitrary, irregular spacing.  We only require strict
    monotonicity of the bin boundaries.  N boundaries define N-1 bins.

    Example:

    >>> x = IrregularBins([0.0, 11.0, 15.0, numpy.inf])
    >>> len(x)
    3
    >>> x[1]
    0
    >>> x[1.5]
    0
    >>> x[13]
    1
    >>> x[25]
    2
    >>> x[4:17]
    slice(0, 3, None)
    >>> IrregularBins([0.0, 15.0, 11.0])
    Traceback (most recent call last):
        ...
    ValueError: non-monotonic boundaries provided
    >>> y = IrregularBins([0.0, 11.0, 15.0, numpy.inf])
    >>> x == y
    True
    """
    def __init__(self, boundaries):
        """
        Initialize a set of custom bins with the bin boundaries.
        This includes all left edges plus the right edge.  The
        boundaries must be monotonic and there must be at least two
        elements.
        """
        # check pre-conditions
        if len(boundaries) < 2:
            raise ValueError("less than two boundaries provided")
        boundaries = tuple(boundaries)
        if any(a > b for a, b in zip(boundaries[:-1], boundaries[1:])):
            raise ValueError("non-monotonic boundaries provided")

        self.boundaries = boundaries
        self.n = len(boundaries) - 1
        self.minv = boundaries[0]
        self.maxv = boundaries[-1]

            return super(IrregularBins, self).__getitem__(x)
        if self.minv <= x < self.maxv:
            return bisect_right(self.boundaries, x) - 1
        # special measure-zero edge case
        if x == self.maxv:
            return len(self.boundaries) - 2
        raise IndexError(x)

        return numpy.array(self.boundaries[:-1])

        return numpy.array(self.boundaries[1:])

        return (self.lower() + self.upper()) / 2.0

class LinearBins(Bins):

    """
    Linearly-spaced bins.  There are n bins of equal size, the first
    bin starts on the lower bound and the last bin ends on the upper
    bound inclusively.

    Example:

    >>> x = LinearBins(1.0, 25.0, 3)
    >>> x.lower()
    array([  1.,   9.,  17.])
    >>> x.upper()
    array([  9.,  17.,  25.])
    >>> x.centres()
    array([  5.,  13.,  21.])
    >>> x[1]
    0
    >>> x[1.5]
    0
    >>> x[10]
    1
    >>> x[25]
    2
    >>> x[0:27]
        ...
    IndexError: 0
    >>> x[1:25]
    >>> x[:25]
    >>> x[10:16.9]
    slice(1, 2, None)
    >>> x[10:17]
    slice(1, 3, None)
    >>> x[10:]
    """
        super(LinearBins, self).__init__(minv, maxv, n)
        self.delta = float(maxv - minv) / n

            return super(LinearBins, self).__getitem__(x)
            return int(math.floor((x - self.minv) / self.delta))
            # special "measure zero" corner case
            return len(self) - 1

        return numpy.linspace(self.minv, self.maxv - self.delta, len(self))

        return numpy.linspace(self.minv + self.delta / 2.,
                              self.maxv - self.delta / 2., len(self))

        return numpy.linspace(self.minv + self.delta, self.maxv, len(self))

class LinearPlusOverflowBins(Bins):

    """
    Linearly-spaced bins with overflow at the edges.

    There are n-2 bins of equal size.  The bin 1 starts on the lower bound and
    bin n-2 ends on the upper bound.  Bins 0 and n-1 are overflow going from
    -infinity to the lower bound and from the upper bound to +infinity
    respectively.  Must have n >= 3.

    Example:

    >>> x = LinearPlusOverflowBins(1.0, 25.0, 5)
    array([-inf,   5.,  13.,  21.,  inf])
    array([-inf,   1.,   9.,  17.,  25.])
    array([  1.,   9.,  17.,  25.,  inf])
    >>> x[float("-inf")]
    0
    >>> x[0]
    0
    >>> x[1]
    1
    >>> x[10]
    2
    >>> x[24.99999999]
    3
    >>> x[25]
    4
    >>> x[100]
    4
    >>> x[float("+inf")]
    4
    >>> x[float("-inf"):9]
    >>> x[9:float("+inf")]
    slice(2, 5, None)
    """
        if n < 3:
            raise ValueError("n must be >= 3")
        super(LinearPlusOverflowBins, self).__init__(minv, maxv, n)
        self.delta = float(maxv - minv) / (n - 2)

            return super(LinearPlusOverflowBins, self).__getitem__(x)
            return int(math.floor((x - self.minv) / self.delta)) + 1
        if x >= self.maxv:
            # +infinity overflow bin
        if x < self.minv:
            # -infinity overflow bin
            return 0

        return numpy.concatenate(
            (numpy.array([NegInf]),
             self.minv + self.delta * numpy.arange(len(self) - 2),
             numpy.array([self.maxv]))
        )

             self.minv + self.delta * (numpy.arange(len(self) - 2) + 0.5),
             numpy.array([PosInf]))
        )

            (numpy.array([self.minv]),
             self.minv + self.delta * (numpy.arange(len(self) - 2) + 1),
        )

class LogarithmicBins(Bins):

    """
    Logarithmically-spaced bins.

    There are n bins, each of whose upper and lower bounds differ by the same
    factor.  The first bin starts on the lower bound, and the last bin ends on
    the upper bound inclusively.

    Example:

    >>> x = LogarithmicBins(1.0, 25.0, 3)
    >>> x[1]
    0
    >>> x[5]
    1
    >>> x[25]
    2
    """
        super(LogarithmicBins, self).__init__(minv, maxv, n)
        self.delta = (math.log(maxv) - math.log(minv)) / n

            return super(LogarithmicBins, self).__getitem__(x)
            return int(math.floor((math.log(x) - math.log(self.minv)) /
                                  self.delta))

        return numpy.exp(
            numpy.linspace(math.log(self.minv), math.log(self.maxv) -
                           self.delta, len(self))
        )

                           self.delta, len(self)) + self.delta / 2.
        )

            numpy.linspace(math.log(self.minv) + self.delta,
                           math.log(self.maxv), len(self))
        )

class LogarithmicPlusOverflowBins(Bins):

    """
    Logarithmically-spaced bins plus one bin at each end that goes to
    zero and positive infinity respectively.  There are n-2 bins each
    of whose upper and lower bounds differ by the same factor.  Bin 1
    starts on the lower bound, and bin n-2 ends on the upper bound
    inclusively.  Bins 0 and n-1 are overflow bins extending from 0 to
    the lower bound and from the upper bound to +infinity respectively.
    Must have n >= 3.

    Example:

    >>> x = LogarithmicPlusOverflowBins(1.0, 25.0, 5)
    >>> x[0]
    0
    >>> x[1]
    1
    >>> x[5]
    2
    >>> x[24.999]
    3
    >>> x[25]
    4
    >>> x[100]
    4
    array([ 0.   ,  1.        ,  2.92401774,  8.54987973, 25.      ])
    array([ 1.   ,  2.92401774,  8.54987973, 25.        ,       inf])
    array([ 0.   ,  1.70997595,  5.        , 14.62008869,       inf])
    """
        if n < 3:
        super(LogarithmicPlusOverflowBins, self).__init__(minv, maxv, n)
        self.delta = (math.log(maxv) - math.log(minv)) / (n - 2)

            return super(LogarithmicPlusOverflowBins, self).__getitem__(x)
            return 1 + int(math.floor((math.log(x) - math.log(self.minv)) /
                                      self.delta))
            # infinity overflow bin
            # zero overflow bin
            return 0

            (numpy.array([0.]),
             numpy.exp(numpy.linspace(math.log(self.minv), math.log(self.maxv),
                                      len(self) - 1))
             )
        )

             numpy.exp(numpy.linspace(math.log(self.minv), math.log(self.maxv) -
                       self.delta, len(self) - 2) + self.delta / 2.),
             numpy.array([PosInf])
             )
        )

            (numpy.exp(numpy.linspace(math.log(self.minv), math.log(self.maxv),
                       len(self) - 1)),
             )
        )

class NDBins(tuple):

    """
    Multi-dimensional co-ordinate binning.  An instance of this object
    is used to convert a tuple of co-ordinates into a tuple of bin
    indices.  This can be used to allow the contents of an array object
    to be accessed with real-valued coordinates.

    NDBins is a subclass of the tuple builtin, and is initialized with
    an iterable of instances of subclasses of Bins.  Each Bins subclass
    instance describes the binning to apply in the corresponding
    co-ordinate direction, and the number of them sets the dimensions
    of the binning.

    Example:

    >>> x = NDBins((LinearBins(1, 25, 3), LogarithmicBins(1, 25, 3)))
    >>> x[1, 1]
    (0, 0)
    >>> x[1.5, 1]
    (0, 0)
    >>> x[10, 1]
    (1, 0)
    >>> x[1, 5]
    (0, 1)
    >>> x[1, 1:5]
    (0, slice(0, 2, None))
    (array([ 5., 13., 21.]), array([ 1.70997595,  5.        , 14.62008869]))

    Note that the co-ordinates to be converted must be a tuple, even if
    it is only a 1-dimensional co-ordinate.
    """
    def __new__(cls, *args):
        new = tuple.__new__(cls, *args)
        new.minv = tuple(b.minv for b in new)
        new.maxv = tuple(b.maxv for b in new)
        new.shape = tuple(len(b) for b in new)
        return new

    def __getitem__(self, coords):
        """
        When coords is a tuple, it is interpreted as an
        N-dimensional co-ordinate which is converted to an N-tuple
        of bin indices by the Bins instances in this object.
        Otherwise coords is interpeted as an index into the tuple,
        and the corresponding Bins instance is returned.

        Example:

        >>> x = NDBins((LinearBins(1, 25, 3), LogarithmicBins(1, 25, 3)))
        >>> x[1, 1]
        (0, 0)
        >>> type(x[1])
        <class 'pylal.rate.LogarithmicBins'>

        When used to convert co-ordinates to bin indices, each
        co-ordinate can be anything the corresponding Bins instance
        will accept.  Note that the co-ordinates to be converted
        must be a tuple, even if it is only a 1-dimensional
        co-ordinate.
        """
        if isinstance(coords, tuple):
            if len(coords) != len(self):
                raise ValueError("dimension mismatch")
            return tuple(map(lambda b, c: b[c], self, coords))
        else:
            return tuple.__getitem__(self, coords)

        """
        Return a tuple of arrays, where each array contains the
        locations of the lower boundaries of the bins in the
        corresponding dimension.
        """
        return tuple(b.lower() for b in self)

        """
        locations of the bin centres for the corresponding
        dimension.
        """
        return tuple(b.centres() for b in self)

        """
        locations of the upper boundaries of the bins in the
        """
        return tuple(b.upper() for b in self)

class BinnedArray(object):

    """
    A convenience wrapper, using the NDBins class to provide access to
    the elements of an array object.  Technical reasons preclude
    providing a subclass of the array object, so the array data is made
    available as the "array" attribute of this class.

    Examples:

    Note that even for 1 dimensional arrays the index must be a tuple.

    >>> x = BinnedArray(NDBins((LinearBins(0, 10, 5),)))
    >>> x.array
    array([ 0.,  0.,  0.,  0.,  0.])
    >>> x[0,] += 1
    >>> x[0.5,] += 1
    array([ 2.,  0.,  0.,  0.,  0.])
    >>> x.argmax()
    (1.0,)

    Note the relationship between the binning limits, the bin centres,
    and the co-ordinates of the BinnedArray

    >>> x = BinnedArray(NDBins((LinearBins(-0.5, 1.5, 2), \
    LinearBins(-0.5, 1.5, 2))))
    >>> x.bins.centres()
    (array([ 0.,  1.]), array([ 0.,  1.]))
    >>> x[0, 0] = 0
    >>> x[0, 1] = 1
    >>> x[1, 0] = 2
    >>> x[1, 1] = 4
    array([[ 0.,  1.],
           [ 2.,  4.]])
    >>> x[0, 0]
    0.0
    >>> x[0, 1]
    1.0
    >>> x[1, 0]
    2.0
    >>> x[1, 1]
    4.0
    >>> x.argmin()
    (0.0, 0.0)
    (1.0, 1.0)
    """
    def __init__(self, bins, array=None, dtype="double"):
        self.bins = bins
        if array is None:
            self.array = numpy.zeros(bins.shape, dtype=dtype)
        else:
            if array.shape != bins.shape:
                raise ValueError("input array and input bins must have the "
                                 "same shape")
            self.array = array

        return self.array[self.bins[coords]]

    def __setitem__(self, coords, val):
        self.array[self.bins[coords]] = val

        return len(self.array)

    def copy(self):
        """
        Return a copy of the BinnedArray.  The .bins attribute is
        shared with the original.
        """
        return type(self)(self.bins, self.array.copy())

        """
        Return a tuple of arrays containing the bin centres for
        each dimension.
        """
        return self.bins.centres()

    def argmin(self):
        """
        Return the co-ordinates of the bin centre containing the
        minimum value.  Same as numpy.argmin(), converting the
        indexes to bin co-ordinates.
        """
        return tuple(centres[index] for centres, index in
                     zip(self.centres(), numpy.unravel_index(self.array.argmin(),
                                                             self.array.shape)))

    def argmax(self):
        """
        maximum value.  Same as numpy.argmax(), converting the
        """
                     zip(self.centres(), numpy.unravel_index(self.array.argmax(),

    def logregularize(self, epsilon=2**-1074):
        """
        Find bins <= 0, and set them to epsilon, This has the
        effect of allowing the logarithm of the array to be
        evaluated without error.
        """
        self.array[self.array <= 0] = epsilon
        return self

class BinnedRatios(object):

    """
    Like BinnedArray, but provides a numerator array and a denominator
    array.  The incnumerator() method increments a bin in the numerator
    by the given weight, and the incdenominator() method increments a
    bin in the denominator by the given weight.  There are no methods
    provided for setting or decrementing either, but the they are
    accessible as the numerator and denominator attributes, which are
    both BinnedArray objects.
    """
    def __init__(self, bins, dtype="double"):
        self.numerator = BinnedArray(bins, dtype=dtype)
        self.denominator = BinnedArray(bins, dtype=dtype)

        return self.numerator[coords] / self.denominator[coords]

    def bins(self):
        return self.numerator.bins

    def incnumerator(self, coords, weight=1):
        """
        Add weight to the numerator bin at coords.
        """
        self.numerator[coords] += weight

    def incdenominator(self, coords, weight=1):
        """
        Add weight to the denominator bin at coords.
        """
        self.denominator[coords] += weight

    def ratio(self):
        """
        Compute and return the array of ratios.
        """
        return self.numerator.array / self.denominator.array

    def regularize(self):
        """
        Find bins in the denominator that are 0, and set them to 1.
        Presumably the corresponding bin in the numerator is also
        0, so this has the effect of allowing the ratio array to be
        evaluated without error, returning zeros in those bins that
        have had no weight added to them.
        """
        self.denominator.array[self.denominator.array == 0] = 1

        """
        Find bins in the denominator that are 0, and set them to 1,
        while setting the corresponding bin in the numerator to
        float epsilon.  This has the effect of allowing the
        logarithm of the ratio array to be evaluated without error.
        """
        self.numerator.array[self.denominator.array == 0] = epsilon

        """
        """
        return self.numerator.bins.centres()

# Copyright (C) 2016  Collin Capano
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

#
# =============================================================================
#
#                                   Preamble
#
#
"""
This modules provides utilities for manipulating parameter boundaries. Namely,
classes are offered that will map values to a specified domain using either
cyclic boundaries or reflected boundaries.
"""

class _Bound(float):
    """Adds methods to float for boundary comparisons."""

    name = None

    def larger(self, other):
        """A function to determine whether or not `other` is larger
        than the bound. This raises a NotImplementedError; classes that
        inherit from this must define it.
        """
        raise NotImplementedError("larger function not set")

    def smaller(self, other):
        """A function to determine whether or not `other` is smaller
        """
        raise NotImplementedError("smaller function not set")

class OpenBound(_Bound):
    """Sets larger and smaller functions to be `>` and `<`, respectively."""

    name = 'open'

        """Returns True if `other` is `>`, False otherwise"""
        return self > other

        """Returns True if `other` is `<`, False otherwise."""
        return self < other

class ClosedBound(_Bound):
    """Sets larger and smaller functions to be `>=` and `<=`, respectively."""

    name = 'closed'

        return self >= other

        return self <= other

class ReflectedBound(ClosedBound):
    """Inherits from `ClosedBound`, adding reflection functions."""

    name = 'reflected'

    def reflect(self, value):
        return 2*self - value

    def reflect_left(self, value):
        """Only reflects the value if is > self."""
        if value > self:
            value = self.reflect(value)
        return value

    def reflect_right(self, value):
        """Only reflects the value if is < self."""
        if value < self:

boundary_types = {
    OpenBound.name: OpenBound,
    ClosedBound.name: ClosedBound,
    ReflectedBound.name: ReflectedBound
}

#
#   Helper functions for applying conditions to boundaries
#

def apply_cyclic(value, bounds):
    """Given a value, applies cyclic boundary conditions between the minimum
    and maximum bounds.

    Parameters
    ----------
    value : float
        The value to apply the cyclic conditions to.
    bounds : Bounds instance
        Boundaries to use for applying cyclic conditions.

    Returns
    -------
    float
        The value after the cyclic bounds are applied.
    """
    return (value - bounds._min) %(bounds._max - bounds._min) + bounds._min

def reflect_well(value, bounds):
    """Given some boundaries, reflects the value until it falls within both
    boundaries. This is done iteratively, reflecting left off of the
    `boundaries.max`, then right off of the `boundaries.min`, etc.

        The value to apply the reflected boundaries to.
        Boundaries to reflect between. Both `bounds.min` and `bounds.max` must
        be instances of `ReflectedBound`, otherwise an AttributeError is
        raised.

    Returns
    -------
    float
        The value after being reflected between the two bounds.
    """
    while value not in bounds:
        value = bounds._max.reflect_left(value)
        value = bounds._min.reflect_right(value)
    return value

def _pass(value):
    """Just return the given value."""

#
#   Bounds class
#

class Bounds(object):
    """Creates and stores bounds using the given values.

    The type of boundaries used can be set using the `btype_(min|max)`
    parameters. These arguments set what kind of boundary is used at the
    minimum and maximum bounds. Specifically, if `btype_min` (`btype_max`) is
    set to:

     * "open": the minimum (maximum) boundary will be an instance of
       `OpenBound`. This means that a value must be `>` (`<`) the bound
       for it to be considered within the bounds.
     * "closed": the minimum (maximum) boundary will be an instance of
       `ClosedBound`. This means that a value must be `>=` (`<=`) the bound
     * "reflected": the minimum (maximum) boundary will be an isntance of
       `ReflectedBound`. This means that a value will be reflected to the
       right (left) if `apply_conditions` is used on the value. For more
       details see `apply_conditions`.

    If the `cyclic` keyword is set to True, then `apply_conditions` will cause
    values to be wrapped around to the minimum (maximum) bound if the value
    is > (<=) the maximum (minimum) bound. For more details see
    `apply_conditions`.

    Values can be checked whether or not they occur within the bounds using
    `in`; e.g., `6 in bounds`. This is done without applying any boundary
    conditions. To apply conditions, then check whether the value is in
    bounds, use the `contains_conditioned` method.

    The default is for the minimum bound to be "closed" and the maximum bound
    to be "open", i.e., a right-open interval.

    min_bound : {-numpy.inf, float}
        The value of the lower bound. Default is `-inf`.
    max_bound : {numpy.inf, float}
        The value of the upper bound. Default is `inf`.
    btype_min : {'closed', string}
        The type of the lower bound; options are "closed", "open", or
        "reflected". Default is "closed".
    btype_min : {'open', string}
        "reflected". Default is "open".
    cyclic : {False, bool}
        Whether or not to make the bounds cyclic; default is False. If True,
        both the minimum and maximum bounds must be finite.

    Attributes
    min : _Bound instance
        The minimum bound.
    max : _Bound instance
        The maximum bound.
    cyclic : bool
        Whether the bounds are cyclic or not.

    Examples
    --------
    Create a right-open interval between -1 and 1 and test whether various
    values are within them:
    >>> bounds = Bounds(-1., 1.)
    >>> -1 in bounds
    True
    >>> 0 in bounds
    True
    >>> 1 in bounds
    False

    Create an open interval between -1 and 1 and test the same values:
    >>> bounds = Bounds(-1, 1, btype_min="open")
    False
    True
    False

    Create cyclic bounds between -1 and 1 and plot the effect of conditioning
    on points between -10 and 10:
    >>> bounds = Bounds(-1, 1, cyclic=True)
    >>> x = numpy.linspace(-10, 10, num=1000)
    >>> conditioned_x = bounds.apply_conditions(x)
    >>> fig = pyplot.figure()
    >>> ax = fig.add_subplot(111)
    >>> ax.plot(x, x, c='b', lw=2, label='input')
    >>> ax.plot(conditioned_x, x, c='r', lw=1)
    >>> ax.vlines([-1., 1.], x.min(), x.max(), color='k', linestyle='--')
    >>> ax.set_title('cyclic bounds between x=-1,1')
    >>> fig.show()

    Create a reflected bound at -1 and plot the effect of conditioning:
    >>> bounds = Bounds(-1, 1, btype_min='reflected')
    >>> ax.set_title('reflected right at x=-1')

    Create a reflected bound at 1 and plot the effect of conditioning:
    >>> bounds = Bounds(-1, 1, btype_max='reflected')
    >>> ax.set_title('reflected left at x=1')

    Create reflected bounds at -1 and 1 and plot the effect of conditioning:
    >>> bounds = Bounds(-1, 1, btype_min='reflected', btype_max='reflected')
    >>> ax.set_title('reflected betewen x=-1,1')
    """

    def __init__(self, min_bound=-numpy.inf, max_bound=numpy.inf,
            btype_min='closed', btype_max='open', cyclic=False):
        # check boundary values
        if min_bound >= max_bound:
            raise ValueError("min_bound must be < max_bound")
        if cyclic and not (
                numpy.isfinite(min_bound) and numpy.isfinite(max_bound)):
            raise ValueError("if using cyclic, min and max bounds must both "
                "be finite")
        # store bounds
        try:
            self._min = boundary_types[btype_min](min_bound)
        except KeyError:
            raise ValueError("unrecognized btype_min {}".format(btype_min))
        try:
            self._max = boundary_types[btype_max](max_bound)
            raise ValueError("unrecognized btype_max {}".format(btype_max))
        # store cyclic conditions
        self._cyclic = bool(cyclic)
        # store reflection conditions; we'll vectorize them here so that they
        # can be used with arrays
        if self._min.name == 'reflected' and self._max.name == 'reflected':
            self._reflect = numpy.vectorize(self._reflect_well)
            self.reflected = 'well'
        elif self._min.name == 'reflected':
            self._reflect = numpy.vectorize(self._min.reflect_right)
            self.reflected = 'min'
        elif self._max.name == 'reflected':
            self._reflect = numpy.vectorize(self._max.reflect_left)
            self.reflected = 'max'
        else:
            self._reflect = _pass
            self.reflected = False

    def __repr__(self):
        return str(self.__class__)[:-1] + " " + " ".join(
                   map(str, ["min", self._min, "max", self._max,
                             "cyclic", self._cyclic])) + ">"

    @property
    def min(self):
        return self._min

    @property
    def max(self):
        return self._max

    @property
    def cyclic(self):
        return self._cyclic

    def __getitem__(self, ii):
        if ii == 0:
            return self._min
        elif ii == 1:
            return self._max
        else:
            raise IndexError("index {} out of range".format(ii))

    def __abs__(self):
        return abs(self._max - self._min)

    def __contains__(self, value):
        return self._min.smaller(value) & self._max.larger(value)

    def _reflect_well(self, value):
        """Thin wrapper around `reflect_well` that passes self as the `bounds`.
        """
        return reflect_well(value, self)

    def _apply_cyclic(self, value):
        """Thin wrapper around `apply_cyclic` that passes self as the `bounds`.
        """
        return apply_cyclic(value, self)

    def apply_conditions(self, value):
        """Applies any boundary conditions to the given value.

        The value is manipulated according based on the following conditions:

         * If `self.cyclic` is True then `value` is wrapped around to the
           minimum (maximum) bound if `value` is `>= self.max` (`< self.min`)
           bound. For example, if the minimum and maximum bounds are `0, 2*pi`
           and `value = 5*pi`, then the returned value will be `pi`.
         * If `self.min` is a reflected boundary then `value` will be
           reflected to the right if it is `< self.min`. For example, if
           `self.min = 10` and `value = 3`, then the returned value will be
           17.
         * If `self.max` is a reflected boundary then `value` will be
           reflected to the left if it is `> self.max`. For example, if
           `self.max = 20` and `value = 27`, then the returned value will be
           13.
         * If `self.min` and `self.max` are both reflected boundaries, then
           `value` will be reflected between the two boundaries until it
           falls within the bounds. The first reflection occurs off of the
           maximum boundary. For example, if `self.min = 10`, `self.max =
           20`, and `value = 42`, the returned value will be 18 ( the first
           reflection yields -2, the second 22, and the last 18).
         * If neither bounds are reflected and cyclic is False, then the
           value is just returned as-is.

        Parameters
        ----------
        value : float
            The value to apply the conditions to.

        Returns
        -------
        float
            The value after the conditions are applied; see above for details.
        """
        retval = value
        if self._cyclic:
            retval = apply_cyclic(value, self)
        retval = self._reflect(retval)
        if isinstance(retval, numpy.ndarray) and retval.size == 1:
            try:
                retval = retval[0]
            except IndexError:
                retval = float(retval)
        return retval

    def contains_conditioned(self, value):
        """Runs `apply_conditions` on the given value before testing whether it
        is in bounds. Note that if `cyclic` is True, or both bounds
        are reflected, than this will always return True.

            The value to test.

        Returns
        -------
        bool
            Whether or not the value is within the bounds after the boundary
            conditions are applied.
        """
        return self.apply_conditions(value) in self

# Copyright (C) 2017  Collin Capano, Christopher M. Biwer, Duncan Brown,
# and Steven Reyes
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides a library of functions that calculate waveform parameters
from other parameters. All exposed functions in this module's namespace return
one parameter given a set of inputs.
"""
from __future__ import division

import copy
import lal
import lalsimulation as lalsim
from pycbc.detector import Detector
import pycbc.cosmology
from .coordinates import spherical_to_cartesian as _spherical_to_cartesian

#
#
#                           Helper functions
#
#
def ensurearray(*args):
    """Apply numpy's broadcast rules to the given arguments.

    This will ensure that all of the arguments are numpy arrays and that they
    all have the same shape. See ``numpy.broadcast_arrays`` for more details.

    It also returns a boolean indicating whether any of the inputs were
    originally arrays.

    *args :
        The arguments to check.

    Returns
    -------
    list :
        A list with length ``N+1`` where ``N`` is the number of given
        arguments. The first N values are the input arguments as ``ndarrays``s.
        The last value is a boolean indicating whether any of the
        inputs was an array.
    """
    input_is_array = any(isinstance(arg, numpy.ndarray) for arg in args)
    args = numpy.broadcast_arrays(*args)
    args.append(input_is_array)
    return args

def formatreturn(arg, input_is_array=False):
    """If the given argument is a numpy array with shape (1,), just returns
    that value."""
    if not input_is_array and arg.size == 1:
        arg = arg.item()
    return arg

#
#
#                           Fundamental conversions
#
#

def sec_to_year(sec):
    """ Converts number of seconds to number of years """
    return sec / lal.YRJUL_SI

#
#
#                           CBC mass functions
#
#
def primary_mass(mass1, mass2):
    """Returns the larger of mass1 and mass2 (p = primary)."""
    mass1, mass2, input_is_array = ensurearray(mass1, mass2)
    mp = copy.copy(mass1)
    mask = mass1 < mass2
    mp[mask] = mass2[mask]
    return formatreturn(mp, input_is_array)

def secondary_mass(mass1, mass2):
    """Returns the smaller of mass1 and mass2 (s = secondary)."""
    if mass1.shape != mass2.shape:
        raise ValueError("mass1 and mass2 must have same shape")
    ms = copy.copy(mass2)
    ms[mask] = mass1[mask]
    return formatreturn(ms, input_is_array)

def mtotal_from_mass1_mass2(mass1, mass2):
    """Returns the total mass from mass1 and mass2."""
    return mass1 + mass2

def q_from_mass1_mass2(mass1, mass2):
    """Returns the mass ratio m1/m2, where m1 >= m2."""
    return primary_mass(mass1, mass2) / secondary_mass(mass1, mass2)

def invq_from_mass1_mass2(mass1, mass2):
    """Returns the inverse mass ratio m2/m1, where m1 >= m2."""
    return secondary_mass(mass1, mass2) / primary_mass(mass1, mass2)

def eta_from_mass1_mass2(mass1, mass2):
    """Returns the symmetric mass ratio from mass1 and mass2."""
    return mass1*mass2 / (mass1+mass2)**2.

def mchirp_from_mass1_mass2(mass1, mass2):
    """Returns the chirp mass from mass1 and mass2."""
    return eta_from_mass1_mass2(mass1, mass2)**(3./5) * (mass1+mass2)

def mass1_from_mtotal_q(mtotal, q):
    """Returns a component mass from the given total mass and mass ratio.

    If the mass ratio q is >= 1, the returned mass will be the primary
    (heavier) mass. If q < 1, the returned mass will be the secondary
    (lighter) mass.
    """
    return q*mtotal / (1.+q)

def mass2_from_mtotal_q(mtotal, q):

    If the mass ratio q is >= 1, the returned mass will be the secondary
    (lighter) mass. If q < 1, the returned mass will be the primary (heavier)
    mass.
    """
    return mtotal / (1.+q)

def mass1_from_mtotal_eta(mtotal, eta):
    """Returns the primary mass from the total mass and symmetric mass
    ratio.
    """
    return 0.5 * mtotal * (1.0 + (1.0 - 4.0 * eta)**0.5)

def mass2_from_mtotal_eta(mtotal, eta):
    """Returns the secondary mass from the total mass and symmetric mass
    ratio.
    """
    return 0.5 * mtotal * (1.0 - (1.0 - 4.0 * eta)**0.5)

def mtotal_from_mchirp_eta(mchirp, eta):
    """Returns the total mass from the chirp mass and symmetric mass ratio.
    """
    return mchirp / (eta**(3./5.))

def mass1_from_mchirp_eta(mchirp, eta):
    """Returns the primary mass from the chirp mass and symmetric mass ratio.
    """
    mtotal = mtotal_from_mchirp_eta(mchirp, eta)
    return mass1_from_mtotal_eta(mtotal, eta)

def mass2_from_mchirp_eta(mchirp, eta):
    """
    return mass2_from_mtotal_eta(mtotal, eta)

def _mass2_from_mchirp_mass1(mchirp, mass1):
    r"""Returns the secondary mass from the chirp mass and primary mass.

    As this is a cubic equation this requires finding the roots and returning
    the one that is real. Basically it can be shown that:

    .. math::
        m_2^3 - a(m_2 + m_1) = 0,

    where

    .. math::
        a = \frac{\mathcal{M}^5}{m_1^3}.

    This has 3 solutions but only one will be real.
    """
    a = mchirp**5 / mass1**3
    roots = numpy.roots([1,0,-a,-a*mass1])
    # Find the real one
    real_root = roots[(abs(roots - roots.real)).argmin()]
    return real_root.real

mass2_from_mchirp_mass1 = numpy.vectorize(_mass2_from_mchirp_mass1)

def _mass_from_knownmass_eta(known_mass, eta, known_is_secondary=False,
                            force_real=True):
    r"""Returns the other component mass given one of the component masses
    and the symmetric mass ratio.

    This requires finding the roots of the quadratic equation:

    .. math::
        \eta m_2^2 + (2\eta - 1)m_1 m_2 + \eta m_1^2 = 0.

    This has two solutions which correspond to :math:`m_1` being the heavier
    mass or it being the lighter mass. By default, `known_mass` is assumed to
    be the heavier (primary) mass, and the smaller solution is returned. Use
    the `other_is_secondary` to invert.

    known_mass : float
        The known component mass.
    eta : float
        The symmetric mass ratio.
    known_is_secondary : {False, bool}
        Whether the known component mass is the primary or the secondary. If
        True, `known_mass` is assumed to be the secondary (lighter) mass and
        the larger solution is returned. Otherwise, the smaller solution is
        returned. Default is False.
    force_real : {True, bool}
        Force the returned mass to be real.

    Returns
    -------
    float
        The other component mass.
    """
    roots = numpy.roots([eta, (2*eta - 1)*known_mass, eta*known_mass**2.])
    if force_real:
        roots = numpy.real(roots)
    if known_is_secondary:
        return roots[roots.argmax()]
    else:
        return roots[roots.argmin()]

mass_from_knownmass_eta = numpy.vectorize(_mass_from_knownmass_eta)

def mass2_from_mass1_eta(mass1, eta, force_real=True):
    """Returns the secondary mass from the primary mass and symmetric mass
    ratio.
    """
    return mass_from_knownmass_eta(mass1, eta, known_is_secondary=False,
                                   force_real=force_real)

def mass1_from_mass2_eta(mass2, eta, force_real=True):
    """Returns the primary mass from the secondary mass and symmetric mass
    ratio.
    """
    return mass_from_knownmass_eta(mass2, eta, known_is_secondary=True,

def eta_from_q(q):
    r"""Returns the symmetric mass ratio from the given mass ratio.

    This is given by:

    .. math::
        \eta = \frac{q}{(1+q)^2}.

    Note that the mass ratio may be either < 1 or > 1.
    """
    return q / (1.+q)**2

def mass1_from_mchirp_q(mchirp, q):
    """Returns the primary mass from the given chirp mass and mass ratio."""
    mass1 = (q**(2./5.))*((1.0 + q)**(1./5.))*mchirp
    return mass1

def mass2_from_mchirp_q(mchirp, q):
    """Returns the secondary mass from the given chirp mass and mass ratio."""
    mass2 = (q**(-3./5.))*((1.0 + q)**(1./5.))*mchirp
    return mass2

def _a0(f_lower):
    """Used in calculating chirp times: see Cokelaer, arxiv.org:0706.4437
       appendix 1, also lalinspiral/python/sbank/tau0tau3.py.
    """
    return 5. / (256. * (numpy.pi * f_lower)**(8./3.))

def _a3(f_lower):
    """Another parameter used for chirp times"""
    return numpy.pi / (8. * (numpy.pi * f_lower)**(5./3.))

def tau0_from_mtotal_eta(mtotal, eta, f_lower):
    r"""Returns :math:`\tau_0` from the total mass, symmetric mass ratio, and
    the given frequency.
    """
    # convert to seconds
    mtotal = mtotal * lal.MTSUN_SI
    # formulae from arxiv.org:0706.4437
    return _a0(f_lower) / (mtotal**(5./3.) * eta)

def tau3_from_mtotal_eta(mtotal, eta, f_lower):
    """
    return _a3(f_lower) / (mtotal**(2./3.) * eta)

def tau0_from_mass1_mass2(mass1, mass2, f_lower):
    r"""Returns :math:`\tau_0` from the component masses and given frequency.
    """
    mtotal = mass1 + mass2
    eta = eta_from_mass1_mass2(mass1, mass2)
    return tau0_from_mtotal_eta(mtotal, eta, f_lower)

def tau3_from_mass1_mass2(mass1, mass2, f_lower):
    r"""Returns :math:`\tau_3` from the component masses and given frequency.
    """
    return tau3_from_mtotal_eta(mtotal, eta, f_lower)

def mtotal_from_tau0_tau3(tau0, tau3, f_lower,
                          in_seconds=False):
    r"""Returns total mass from :math:`\tau_0, \tau_3`."""
    mtotal = (tau3 / _a3(f_lower)) / (tau0 / _a0(f_lower))
    if not in_seconds:
        # convert back to solar mass units
        mtotal /= lal.MTSUN_SI
    return mtotal

def eta_from_tau0_tau3(tau0, tau3, f_lower):
    r"""Returns symmetric mass ratio from :math:`\tau_0, \tau_3`."""
    mtotal = mtotal_from_tau0_tau3(tau0, tau3, f_lower,
                                   in_seconds=True)
    eta = mtotal**(-2./3.) * (_a3(f_lower) / tau3)
    return eta

def mass1_from_tau0_tau3(tau0, tau3, f_lower):
    r"""Returns the primary mass from the given :math:`\tau_0, \tau_3`."""
    mtotal = mtotal_from_tau0_tau3(tau0, tau3, f_lower)
    eta = eta_from_tau0_tau3(tau0, tau3, f_lower)

def mass2_from_tau0_tau3(tau0, tau3, f_lower):
    r"""Returns the secondary mass from the given :math:`\tau_0, \tau_3`."""

def lambda_tilde(mass1, mass2, lambda1, lambda2):
    """ The effective lambda parameter

    The mass-weighted dominant effective lambda parameter defined in
    https://journals.aps.org/prd/pdf/10.1103/PhysRevD.91.043002
    """
    m1, m2, lambda1, lambda2, input_is_array = ensurearray(
        mass1, mass2, lambda1, lambda2)
    lsum = lambda1 + lambda2
    ldiff, _ = ensurearray(lambda1 - lambda2)
    mask = m1 < m2
    ldiff[mask] = -ldiff[mask]
    eta = eta_from_mass1_mass2(m1, m2)
    p1 = (lsum) * (1 + 7. * eta - 31 * eta ** 2.0)
    p2 = (1 - 4 * eta)**0.5 * (1 + 9 * eta - 11 * eta ** 2.0) * (ldiff)
    return formatreturn(8.0 / 13.0 * (p1 + p2), input_is_array)

def lambda_from_mass_tov_file(mass, tov_file, distance=0.):
    """Return the lambda parameter(s) corresponding to the input mass(es)
    interpolating from the mass-Lambda data for a particular EOS read in from
    an ASCII file.
    """
    data = numpy.loadtxt(tov_file)
    mass_from_file = data[:, 0]
    lambda_from_file = data[:, 1]
    mass_src = mass/(1.0 + pycbc.cosmology.redshift(distance))
    lambdav = numpy.interp(mass_src, mass_from_file, lambda_from_file)
    return lambdav

#
#
#                           CBC spin functions
#
#
def chi_eff(mass1, mass2, spin1z, spin2z):
    """Returns the effective spin from mass1, mass2, spin1z, and spin2z."""
    return (spin1z * mass1 + spin2z * mass2) / (mass1 + mass2)

def chi_a(mass1, mass2, spin1z, spin2z):
    """ Returns the aligned mass-weighted spin difference from mass1, mass2,
    spin1z, and spin2z.
    """
    return (spin2z * mass2 - spin1z * mass1) / (mass2 + mass1)

def chi_p(mass1, mass2, spin1x, spin1y, spin2x, spin2y):
    """Returns the effective precession spin from mass1, mass2, spin1x,
    spin1y, spin2x, and spin2y.
    """
    xi1 = secondary_xi(mass1, mass2, spin1x, spin1y, spin2x, spin2y)
    xi2 = primary_xi(mass1, mass2, spin1x, spin1y, spin2x, spin2y)
    return chi_p_from_xi1_xi2(xi1, xi2)

def phi_a(mass1, mass2, spin1x, spin1y, spin2x, spin2y):
    """ Returns the angle between the in-plane perpendicular spins."""
    phi1 = phi_from_spinx_spiny(primary_spin(mass1, mass2, spin1x, spin2x),
                                primary_spin(mass1, mass2, spin1y, spin2y))
    phi2 = phi_from_spinx_spiny(secondary_spin(mass1, mass2, spin1x, spin2x),
                                secondary_spin(mass1, mass2, spin1y, spin2y))
    return (phi1 - phi2) % (2 * numpy.pi)

def phi_s(spin1x, spin1y, spin2x, spin2y):
    """ Returns the sum of the in-plane perpendicular spins."""
    phi1 = phi_from_spinx_spiny(spin1x, spin1y)
    phi2 = phi_from_spinx_spiny(spin2x, spin2y)
    return (phi1 + phi2) % (2 * numpy.pi)

def chi_eff_from_spherical(mass1, mass2, spin1_a, spin1_polar,
                           spin2_a, spin2_polar):
    """Returns the effective spin using spins in spherical coordinates."""
    spin1z = spin1_a * numpy.cos(spin1_polar)
    spin2z = spin2_a * numpy.cos(spin2_polar)
    return chi_eff(mass1, mass2, spin1z, spin2z)

def chi_p_from_spherical(mass1, mass2, spin1_a, spin1_azimuthal, spin1_polar,
                         spin2_a, spin2_azimuthal, spin2_polar):
    """Returns the effective precession spin using spins in spherical
    coordinates.
    """
    spin1x, spin1y, _ = _spherical_to_cartesian(
        spin1_a, spin1_azimuthal, spin1_polar)
    spin2x, spin2y, _ = _spherical_to_cartesian(
        spin2_a, spin2_azimuthal, spin2_polar)
    return chi_p(mass1, mass2, spin1x, spin1y, spin2x, spin2y)

def primary_spin(mass1, mass2, spin1, spin2):
    """Returns the dimensionless spin of the primary mass."""
    mass1, mass2, spin1, spin2, input_is_array = ensurearray(
        mass1, mass2, spin1, spin2)
    sp = copy.copy(spin1)
    sp[mask] = spin2[mask]
    return formatreturn(sp, input_is_array)

def secondary_spin(mass1, mass2, spin1, spin2):
    """Returns the dimensionless spin of the secondary mass."""
    ss = copy.copy(spin2)
    ss[mask] = spin1[mask]
    return formatreturn(ss, input_is_array)

def primary_xi(mass1, mass2, spin1x, spin1y, spin2x, spin2y):
    """Returns the effective precession spin argument for the larger mass.
    """
    spinx = primary_spin(mass1, mass2, spin1x, spin2x)
    spiny = primary_spin(mass1, mass2, spin1y, spin2y)
    return chi_perp_from_spinx_spiny(spinx, spiny)

def secondary_xi(mass1, mass2, spin1x, spin1y, spin2x, spin2y):
    """Returns the effective precession spin argument for the smaller mass.
    """
    spinx = secondary_spin(mass1, mass2, spin1x, spin2x)
    spiny = secondary_spin(mass1, mass2, spin1y, spin2y)
    return xi2_from_mass1_mass2_spin2x_spin2y(mass1, mass2, spinx, spiny)

def xi1_from_spin1x_spin1y(spin1x, spin1y):
    This function assumes it's given spins of the primary mass.
    """
    return chi_perp_from_spinx_spiny(spin1x, spin1y)

def xi2_from_mass1_mass2_spin2x_spin2y(mass1, mass2, spin2x, spin2y):
    This function assumes it's given spins of the secondary mass.
    """
    q = q_from_mass1_mass2(mass1, mass2)
    a1 = 2 + 3 * q / 2
    a2 = 2 + 3 / (2 * q)
    return a1 / (q**2 * a2) * chi_perp_from_spinx_spiny(spin2x, spin2y)

def chi_perp_from_spinx_spiny(spinx, spiny):
    """Returns the in-plane spin from the x/y components of the spin.
    """
    return numpy.sqrt(spinx**2 + spiny**2)

def chi_perp_from_mass1_mass2_xi2(mass1, mass2, xi2):
    """Returns the in-plane spin from mass1, mass2, and xi2 for the
    secondary mass.
    """
    return q**2 * a2 / a1 * xi2

def chi_p_from_xi1_xi2(xi1, xi2):
    """Returns effective precession spin from xi1 and xi2.
    """
    xi1, xi2, input_is_array = ensurearray(xi1, xi2)
    chi_p = copy.copy(xi1)
    mask = xi1 < xi2
    chi_p[mask] = xi2[mask]
    return formatreturn(chi_p, input_is_array)

def phi1_from_phi_a_phi_s(phi_a, phi_s):
    """Returns the angle between the x-component axis and the in-plane
    spin for the primary mass from phi_s and phi_a.
    """
    return (phi_s + phi_a) / 2.0

def phi2_from_phi_a_phi_s(phi_a, phi_s):
    spin for the secondary mass from phi_s and phi_a.
    """
    return (phi_s - phi_a) / 2.0

def phi_from_spinx_spiny(spinx, spiny):
    """Returns the angle between the x-component axis and the in-plane spin.
    """
    phi = numpy.arctan2(spiny, spinx)
    return phi % (2 * numpy.pi)

def spin1z_from_mass1_mass2_chi_eff_chi_a(mass1, mass2, chi_eff, chi_a):
    """Returns spin1z.
    """
    return (mass1 + mass2) / (2.0 * mass1) * (chi_eff - chi_a)

def spin2z_from_mass1_mass2_chi_eff_chi_a(mass1, mass2, chi_eff, chi_a):
    """Returns spin2z.
    """
    return (mass1 + mass2) / (2.0 * mass2) * (chi_eff + chi_a)

def spin1x_from_xi1_phi_a_phi_s(xi1, phi_a, phi_s):
    """Returns x-component spin for primary mass.
    """
    phi1 = phi1_from_phi_a_phi_s(phi_a, phi_s)
    return xi1 * numpy.cos(phi1)

def spin1y_from_xi1_phi_a_phi_s(xi1, phi_a, phi_s):
    """Returns y-component spin for primary mass.
    """
    phi1 = phi1_from_phi_a_phi_s(phi_s, phi_a)
    return xi1 * numpy.sin(phi1)

def spin2x_from_mass1_mass2_xi2_phi_a_phi_s(mass1, mass2, xi2, phi_a, phi_s):
    """Returns x-component spin for secondary mass.
    """
    chi_perp = chi_perp_from_mass1_mass2_xi2(mass1, mass2, xi2)
    phi2 = phi2_from_phi_a_phi_s(phi_a, phi_s)
    return chi_perp * numpy.cos(phi2)

def spin2y_from_mass1_mass2_xi2_phi_a_phi_s(mass1, mass2, xi2, phi_a, phi_s):
    """Returns y-component spin for secondary mass.
    """
    return chi_perp * numpy.sin(phi2)

def dquadmon_from_lambda(lambdav):
    r"""Return the quadrupole moment of a neutron star given its lambda

    We use the relations defined here. https://arxiv.org/pdf/1302.4499.pdf.
    Note that the convention we use is that:

    .. math::

        \mathrm{dquadmon} = \bar{Q} - 1.

    Where :math:`\bar{Q}` (dimensionless) is the reduced quadrupole moment.
    """
    ll = numpy.log(lambdav)
    ai = .194
    bi = .0936
    ci = 0.0474
    di = -4.21 * 10**-3.0
    ei = 1.23 * 10**-4.0
    ln_quad_moment = ai + bi*ll + ci*ll**2.0 + di*ll**3.0 + ei*ll**4.0
    return numpy.exp(ln_quad_moment) - 1

def spin_from_pulsar_freq(mass, radius, freq):
    """Returns the dimensionless spin of a pulsar.

    Assumes the pulsar is a solid sphere when computing the moment of inertia.

    mass : float
        The mass of the pulsar, in solar masses.
    radius : float
        The assumed radius of the pulsar, in kilometers.
    freq : float
        The spin frequency of the pulsar, in Hz.
    """
    omega = 2 * numpy.pi * freq
    mt = mass * lal.MTSUN_SI
    mominert = (2/3.) * mt * (radius * 1000 / lal.C_SI)**2
    return mominert * omega / mt**2

#
#
#                         Extrinsic parameter functions
#
#
def chirp_distance(dist, mchirp, ref_mass=1.4):
    """Returns the chirp distance given the luminosity distance and chirp mass.
    """
    return dist * (2.**(-1./5) * ref_mass / mchirp)**(5./6)

def distance_from_chirp_distance_mchirp(chirp_distance, mchirp, ref_mass=1.4):
    """Returns the luminosity distance given a chirp distance and chirp mass.
    """
    return chirp_distance * (2.**(-1./5) * ref_mass / mchirp)**(-5./6)

_detector_cache = {}
def det_tc(detector_name, ra, dec, tc, ref_frame='geocentric', relative=False):
    """Returns the coalescence time of a signal in the given detector.

    detector_name : string
        The name of the detector, e.g., 'H1'.
    ra : float
        The right ascension of the signal, in radians.
    dec : float
        The declination of the signal, in radians.
    tc : float
        The GPS time of the coalescence of the signal in the `ref_frame`.
    ref_frame : {'geocentric', string}
        The reference frame that the given coalescence time is defined in.
        May specify 'geocentric', or a detector name; default is 'geocentric'.

    Returns
    -------
    float :
        The GPS time of the coalescence in detector `detector_name`.
    """
    ref_time = tc
    if relative:
        tc = 0

    if ref_frame == detector_name:
        return tc
    if detector_name not in _detector_cache:
        _detector_cache[detector_name] = Detector(detector_name)
    detector = _detector_cache[detector_name]
    if ref_frame == 'geocentric':
        return tc + detector.time_delay_from_earth_center(ra, dec, ref_time)
    else:
        other = Detector(ref_frame)
        return tc + detector.time_delay_from_detector(other, ra, dec, ref_time)

def optimal_orientation_from_detector(detector_name, tc):
    """ Low-level function to be called from _optimal_dec_from_detector
    and _optimal_ra_from_detector"""

    d = Detector(detector_name)
    ra, dec = d.optimal_orientation(tc)
    return ra, dec

def optimal_dec_from_detector(detector_name, tc):
    """For a given detector and GPS time, return the optimal orientation
    (directly overhead of the detector) in declination.

    tc : float

    Returns
    -------
    float :
    """
    return optimal_orientation_from_detector(detector_name, tc)[1]

def optimal_ra_from_detector(detector_name, tc):
    (directly overhead of the detector) in right ascension.

    tc : float

    Returns
    -------
    float :
    """
    return optimal_orientation_from_detector(detector_name, tc)[0]

#
#
#                         Likelihood statistic parameter functions
#
#
def snr_from_loglr(loglr):
    """Returns SNR computed from the given log likelihood ratio(s). This is
    defined as `sqrt(2*loglr)`.If the log likelihood ratio is < 0, returns 0.

    loglr : array or float
        The log likelihood ratio(s) to evaluate.

    Returns
    -------
    array or float
        The SNRs computed from the log likelihood ratios.
    """
    singleval = isinstance(loglr, float)
    if singleval:
        loglr = numpy.array([loglr])
    # temporarily quiet sqrt(-1) warnings
    numpysettings = numpy.seterr(invalid='ignore')
    snrs = numpy.sqrt(2*loglr)
    numpy.seterr(**numpysettings)
    snrs[numpy.isnan(snrs)] = 0.
        snrs = snrs[0]
    return snrs

#
#
#                         BH Ringdown functions
#
#
def _genqnmfreq(mass, spin, l, m, nmodes, qnmfreq=None):
    """Convenience function to generate QNM frequencies from lalsimulation.

        The mass of the black hole (in solar masses).
    spin : float
        The dimensionless spin of the black hole.
    l : int
        l-index of the harmonic.
    m : int
        m-index of the harmonic.
    nmodes : int
        The number of overtones to generate.
    qnmfreq : lal.COMPLEX16Vector, optional
        LAL vector to write the results into. Must be the same length as
        ``nmodes``. If None, will create one.

    Returns
    -------
    lal.COMPLEX16Vector
        LAL vector containing the complex QNM frequencies.
    """
    if qnmfreq is None:
        qnmfreq = lal.CreateCOMPLEX16Vector(int(nmodes))
    lalsim.SimIMREOBGenerateQNMFreqV2fromFinal(
        qnmfreq, float(mass), float(spin), int(l), int(m), int(nmodes))
    return qnmfreq

def get_lm_f0tau(mass, spin, l, m, nmodes):
    """Return the f0 and the tau of each overtone for a given l, m mode.

    mass : float or array
        Mass of the black hole (in solar masses).
    spin : float or array
        Dimensionless spin of the final black hole.
    l : int or array
    m : int or array

    Returns
    -------
    f0 : float or array
        The frequency of the QNM(s), in Hz. If only a single mode is requested
        (and mass, spin, l, and m are not arrays), this will be a float. If
        multiple modes requested, will be an array with shape
        ``[input shape x] nmodes``, where ``input shape`` is the broadcasted
        shape of the inputs.
    tau : float or array
        The damping time of the QNM(s), in seconds. Return type is same as f0.
    """
    # convert to arrays
    mass, spin, l, m, input_is_array = ensurearray(
        mass, spin, l, m)
    # we'll ravel the arrays so we can evaluate each parameter combination
    # one at a a time
    origshape = mass.shape
    if nmodes < 1:
        raise ValueError("nmodes must be >= 1")
    if nmodes > 1:
        newshape = tuple(list(origshape)+[nmodes])
    else:
        newshape = origshape
    f0s = numpy.zeros((mass.size, nmodes))
    taus = numpy.zeros((mass.size, nmodes))
    mass = mass.ravel()
    spin = spin.ravel()
    l = l.ravel()
    m = m.ravel()
    qnmfreq = None
    modes = range(nmodes)
    for ii in range(mass.size):
        qnmfreq = _genqnmfreq(mass[ii], spin[ii], l[ii], m[ii], nmodes,
                              qnmfreq=qnmfreq)
        f0s[ii, :] = [qnmfreq.data[n].real/(2 * numpy.pi) for n in modes]
        taus[ii, :] = [1./qnmfreq.data[n].imag for n in modes]
    f0s = f0s.reshape(newshape)
    taus = taus.reshape(newshape)
    return (formatreturn(f0s, input_is_array),
            formatreturn(taus, input_is_array))

def get_lm_f0tau_allmodes(mass, spin, modes):
    """Returns a dictionary of all of the frequencies and damping times for the
    requested modes.

    modes : list of str
        The modes to get. Each string in the list should be formatted 'lmN',
        where l (m) is the l (m) index of the harmonic and N is the number of
        overtones to generate (note, N is not the index of the overtone). For
        example, '221' will generate the 0th overtone of the l = m = 2 mode.

    Returns
    -------
    f0 : dict
        Dictionary mapping the modes to the frequencies. The dictionary keys
        are 'lmn' string, where l (m) is the l (m) index of the harmonic and
        n is the index of the overtone. For example, '220' is the l = m = 2
        mode and the 0th overtone.
    tau : dict
        Dictionary mapping the modes to the damping times. The keys are the
        same as ``f0``.
    """
    f0, tau = {}, {}
    key = '{}{}{}'
    for lmn in modes:
        l, m, nmodes = int(lmn[0]), int(lmn[1]), int(lmn[2])
        tmp_f0, tmp_tau = get_lm_f0tau(mass, spin, l, m, nmodes)
        if nmodes == 1:
            # in this case, tmp_f0 and tmp_tau will just be floats
            f0[key.format(l, m, '0')] = tmp_f0
            tau[key.format(l, m, '0')] = tmp_tau
        else:
            for n in range(nmodes):
                # we need to wrap tmp_f0 with formatreturn to ensure that if
                # only a mass, spin pair was requested, the value stored to
                # the dict is a float
                f0[key.format(l, m, n)] = formatreturn(tmp_f0[..., n])
                tau[key.format(l, m, n)] = formatreturn(tmp_tau[..., n])
    return f0, tau

def freq_from_final_mass_spin(final_mass, final_spin, l=2, m=2, nmodes=1):
    """Returns QNM frequency for the given mass and spin and mode.

    final_mass : float or array
    final_spin : float or array
    l : int or array, optional
        l-index of the harmonic. Default is 2.
    m : int or array, optional
        m-index of the harmonic. Default is 2.
    nmodes : int, optional
        The number of overtones to generate. Default is 1.

    Returns
    -------
    float or array
    """
    return get_lm_f0tau(final_mass, final_spin, l, m, nmodes)[0]

def tau_from_final_mass_spin(final_mass, final_spin, l=2, m=2, nmodes=1):
    """Returns QNM damping time for the given mass and spin and mode.

    Returns
    -------
        The damping time of the QNM(s), in seconds. If only a single mode is
        requested (and mass, spin, l, and m are not arrays), this will be a
        float. If multiple modes requested, will be an array with shape
    """
    return get_lm_f0tau(final_mass, final_spin, l, m, nmodes)[1]

# The following are from Table VIII, IX, X of Berti et al.,
# PRD 73 064030, arXiv:gr-qc/0512160 (2006).
# Keys are l,m (only n=0 supported). Constants are for converting from
# frequency and damping time to mass and spin.
_berti_spin_constants = {
    (2, 2): (0.7, 1.4187, -0.4990),
    (3, 3): (0.9, 2.343, -0.4810),
    (4, 4): (1.1929, 3.1191, -0.4825),
    (2, 1): (-0.3, 2.3561, -0.2277)
    }

_berti_mass_constants = {
    (2, 2): (1.5251, -1.1568, 0.1292),
    (3, 3): (1.8956, -1.3043, 0.1818),
    (4, 4): (2.3, -1.5056, 0.2244),
    (2, 1): (0.6, -0.2339, 0.4175)
    }

def final_spin_from_f0_tau(f0, tau, l=2, m=2):
    """Returns the final spin based on the given frequency and damping time.

    .. note::
        Currently, only (l,m) = (2,2), (3,3), (4,4), (2,1) are supported.
        Any other indices will raise a ``KeyError``.

        Frequency of the QNM (in Hz).
        Damping time of the QNM (in seconds).
    l : int, optional
    m : int, optional

    Returns
    -------
        The spin of the final black hole. If the combination of frequency
        and damping times give an unphysical result, ``numpy.nan`` will be
        returned.
    """
    f0, tau, input_is_array = ensurearray(f0, tau)
    # from Berti et al. 2006
    a, b, c = _berti_spin_constants[l,m]
    origshape = f0.shape
    # flatten inputs for storing results
    f0 = f0.ravel()
    tau = tau.ravel()
    spins = numpy.zeros(f0.size)
    for ii in range(spins.size):
        Q = f0[ii] * tau[ii] * numpy.pi
        try:
            s = 1. - ((Q-a)/b)**(1./c)
        except ValueError:
            s = numpy.nan
        spins[ii] = s
    spins = spins.reshape(origshape)
    return formatreturn(spins, input_is_array)

def final_mass_from_f0_tau(f0, tau, l=2, m=2):
    """Returns the final mass (in solar masses) based on the given frequency
    and damping time.

    .. note::

    Returns
    -------
        The mass of the final black hole. If the combination of frequency
        returned.
    """
    spin = final_spin_from_f0_tau(f0, tau, l=l, m=m)
    a, b, c = _berti_mass_constants[l,m]
    return (a + b*(1-spin)**c)/(2*numpy.pi*f0*lal.MTSUN_SI)

def freqlmn_from_other_lmn(f0, tau, current_l, current_m, new_l, new_m):
    """Returns the QNM frequency (in Hz) of a chosen new (l,m) mode from the
    given current (l,m) mode.

        Frequency of the current QNM (in Hz).
        Damping time of the current QNM (in seconds).
    current_l : int, optional
        l-index of the current QNM.
    current_m : int, optional
        m-index of the current QNM.
    new_l : int, optional
        l-index of the new QNM to convert to.
    new_m : int, optional
        m-index of the new QNM to convert to.

    Returns
    -------
        The frequency of the new (l, m) QNM mode. If the combination of
        frequency and damping time provided for the current (l, m) QNM mode
        correspond to an unphysical Kerr black hole mass and/or spin,
        ``numpy.nan`` will be returned.
    """
    mass = final_mass_from_f0_tau(f0, tau, l=current_l, m=current_m)
    spin = final_spin_from_f0_tau(f0, tau, l=current_l, m=current_m)
    mass, spin, input_is_array = ensurearray(mass, spin)

    mass[mass < 0] = numpy.nan
    spin[numpy.abs(spin) > 0.9996] = numpy.nan

    new_f0 = freq_from_final_mass_spin(mass, spin, l=new_l, m=new_m, nmodes=1)
    return formatreturn(new_f0, input_is_array)

def taulmn_from_other_lmn(f0, tau, current_l, current_m, new_l, new_m):
    """Returns the QNM damping time (in seconds) of a chosen new (l,m) mode
    from the given current (l,m) mode.

    Returns
    -------
        The daming time of the new (l, m) QNM mode. If the combination of
    """

    new_tau = tau_from_final_mass_spin(mass, spin, l=new_l, m=new_m, nmodes=1)
    return formatreturn(new_tau, input_is_array)

def get_final_from_initial(mass1, mass2, spin1x=0., spin1y=0., spin1z=0.,
                           spin2x=0., spin2y=0., spin2z=0.,
                           approximant='SEOBNRv4'):
    """Estimates the final mass and spin from the given initial parameters.

    This uses the fits used by the EOBNR models for converting from initial
    parameters to final. Which version used can be controlled by the
    ``approximant`` argument.

    mass1 : float
        The mass of one of the components, in solar masses.
    mass2 : float
        The mass of the other component, in solar masses.
    spin1x : float, optional
        The dimensionless x-component of the spin of mass1. Default is 0.
    spin1y : float, optional
        The dimensionless y-component of the spin of mass1. Default is 0.
    spin1z : float, optional
        The dimensionless z-component of the spin of mass1. Default is 0.
    spin2x : float, optional
        The dimensionless x-component of the spin of mass2. Default is 0.
    spin2y : float, optional
        The dimensionless y-component of the spin of mass2. Default is 0.
    spin2z : float, optional
        The dimensionless z-component of the spin of mass2. Default is 0.
    approximant : str, optional
        The waveform approximant to use for the fit function. Default is
        "SEOBNRv4".

    Returns
    -------
    final_mass : float
        The final mass, in solar masses.
    final_spin : float
        The dimensionless final spin.
    """
    args = (mass1, mass2, spin1x, spin1y, spin1z, spin2x, spin2y, spin2z)
    args = ensurearray(*args)
    input_is_array = args[-1]
    origshape = args[0].shape
    args = [a.ravel() for a in args[:-1]]
    mass1, mass2, spin1x, spin1y, spin1z, spin2x, spin2y, spin2z = args
    final_mass = numpy.zeros(mass1.shape)
    final_spin = numpy.zeros(mass1.shape)
    for ii in range(final_mass.size):
        m1 = numpy.float(mass1[ii])
        m2 = numpy.float(mass2[ii])
        spin1 = list(map(float, [spin1x[ii], spin1y[ii], spin1z[ii]]))
        spin2 = list(map(float, [spin2x[ii], spin2y[ii], spin2z[ii]]))
        _, fm, fs = lalsim.SimIMREOBFinalMassSpin(m1, m2, spin1, spin2,
                                                  getattr(lalsim, approximant))
        final_mass[ii] = fm * (m1 + m2)
        final_spin[ii] = fs
    final_mass = final_mass.reshape(origshape)
    final_spin = final_spin.reshape(origshape)
    return (formatreturn(final_mass, input_is_array),
            formatreturn(final_spin, input_is_array))

def final_mass_from_initial(mass1, mass2, spin1x=0., spin1y=0., spin1z=0.,
                            spin2x=0., spin2y=0., spin2z=0.,
                            approximant='SEOBNRv4'):
    """Estimates the final mass from the given initial parameters.

    Returns
    -------
    float
    """
    return get_final_from_initial(mass1, mass2, spin1x, spin1y, spin1z,
                                  spin2x, spin2y, spin2z, approximant)[0]

def final_spin_from_initial(mass1, mass2, spin1x=0., spin1y=0., spin1z=0.,
    """Estimates the final spin from the given initial parameters.

    Returns
    -------
    float
    """
                                  spin2x, spin2y, spin2z, approximant)[1]

#
#
#                         post-Newtonian functions
#
#

def velocity_to_frequency(v, M):
    """ Calculate the gravitational-wave frequency from the
    total mass and invariant velocity.

    v : float
        Invariant velocity
    M : float
        Binary total mass

    Returns
    -------
    f : float
        Gravitational-wave frequency
    """
    return v**(3.0) / (M * lal.MTSUN_SI * lal.PI)

def frequency_to_velocity(f, M):
    """ Calculate the invariant velocity from the total
    mass and gravitational-wave frequency.

    f: float
    M: float

    Returns
    -------
    v : float or numpy.array
    """
    return (lal.PI * M * lal.MTSUN_SI * f)**(1.0/3.0)

def f_schwarzchild_isco(M):
    """
    Innermost stable circular orbit (ISCO) for a test particle
    orbiting a Schwarzschild black hole

    M : float or numpy.array
        Total mass in solar mass units

    Returns
    -------
    f : float or numpy.array
        Frequency in Hz
    """
    return velocity_to_frequency((1.0/6.0)**(0.5), M)

#
# ============================================================================
#
#                          p-g mode non-linear tide functions
#
#

def nltides_coefs(amplitude, n, m1, m2):
    """Calculate the coefficents needed to compute the
    shift in t(f) and phi(f) due to non-linear tides.

    amplitude: float
        Amplitude of effect
    n: float
        Growth dependence of effect
    m1: float
        Mass of component 1
    m2: float
        Mass of component 2

    Returns
    -------
    f_ref : float
        Reference frequency used to define A and n
    t_of_f_factor: float
        The constant factor needed to compute t(f)
    phi_of_f_factor: float
        The constant factor needed to compute phi(f)
    """

    # Use 100.0 Hz as a reference frequency
    f_ref = 100.0

    # Calculate chirp mass
    mc = mchirp_from_mass1_mass2(m1, m2)
    mc *= lal.lal.MSUN_SI

    # Calculate constants in phasing
    a = (96./5.) * \
        (lal.lal.G_SI * lal.lal.PI * mc * f_ref / lal.lal.C_SI**3.)**(5./3.)
    b = 6. * amplitude
    t_of_f_factor = -1./(lal.lal.PI*f_ref) * b/(a*a * (n-4.))
    phi_of_f_factor = -2.*b / (a*a * (n-3.))

    return f_ref, t_of_f_factor, phi_of_f_factor

def nltides_gw_phase_difference(f, f0, amplitude, n, m1, m2):
    """Calculate the gravitational-wave phase shift bwtween
    f and f_coalescence = infinity due to non-linear tides.
    To compute the phase shift between e.g. f_low and f_isco,
    call this function twice and compute the difference.

    f: float or numpy.array
        Frequency from which to compute phase
    f0: float or numpy.array
        Frequency that NL effects switch on
    amplitude: float or numpy.array
    n: float or numpy.array
    m1: float or numpy.array
    m2: float or numpy.array

    Returns
    -------
    delta_phi: float or numpy.array
        Phase in radians
    """
    f, f0, amplitude, n, m1, m2, input_is_array = ensurearray(
        f, f0, amplitude, n, m1, m2)

    delta_phi = numpy.zeros(m1.shape)

    f_ref, _, phi_of_f_factor = nltides_coefs(amplitude, n, m1, m2)

    mask = f <= f0
    delta_phi[mask] = - phi_of_f_factor[mask] * (f0[mask]/f_ref)**(n[mask]-3.)

    mask = f > f0
    delta_phi[mask] = - phi_of_f_factor[mask] * (f[mask]/f_ref)**(n[mask]-3.)

    return formatreturn(delta_phi, input_is_array)

def nltides_gw_phase_diff_isco(f_low, f0, amplitude, n, m1, m2):
    f_low and f_isco due to non-linear tides.

    f_low: float
        Frequency from which to compute phase. If the other
        arguments are passed as numpy arrays then the value
        of f_low is duplicated for all elements in the array

    Returns
    -------
    """
    f0, amplitude, n, m1, m2, input_is_array = ensurearray(
        f0, amplitude, n, m1, m2)

    f_low = numpy.zeros(m1.shape) + f_low

    phi_l = nltides_gw_phase_difference(
                f_low, f0, amplitude, n, m1, m2)

    f_isco = f_schwarzchild_isco(m1+m2)

    phi_i = nltides_gw_phase_difference(
                f_isco, f0, amplitude, n, m1, m2)

    return formatreturn(phi_i - phi_l, input_is_array)

__all__ = ['dquadmon_from_lambda', 'lambda_tilde',
           'lambda_from_mass_tov_file', 'primary_mass',
           'secondary_mass', 'mtotal_from_mass1_mass2',
           'q_from_mass1_mass2', 'invq_from_mass1_mass2',
           'eta_from_mass1_mass2', 'mchirp_from_mass1_mass2',
           'mass1_from_mtotal_q', 'mass2_from_mtotal_q',
           'mass1_from_mtotal_eta', 'mass2_from_mtotal_eta',
           'mtotal_from_mchirp_eta', 'mass1_from_mchirp_eta',
           'mass2_from_mchirp_eta', 'mass2_from_mchirp_mass1',
           'mass_from_knownmass_eta', 'mass2_from_mass1_eta',
           'mass1_from_mass2_eta', 'eta_from_q', 'mass1_from_mchirp_q',
           'mass2_from_mchirp_q', 'tau0_from_mtotal_eta',
           'tau3_from_mtotal_eta', 'tau0_from_mass1_mass2',
           'tau3_from_mass1_mass2', 'mtotal_from_tau0_tau3',
           'eta_from_tau0_tau3', 'mass1_from_tau0_tau3',
           'mass2_from_tau0_tau3', 'primary_spin', 'secondary_spin',
           'chi_eff', 'chi_a', 'chi_p', 'phi_a', 'phi_s',
           'primary_xi', 'secondary_xi',
           'xi1_from_spin1x_spin1y', 'xi2_from_mass1_mass2_spin2x_spin2y',
           'chi_perp_from_spinx_spiny', 'chi_perp_from_mass1_mass2_xi2',
           'chi_p_from_xi1_xi2', 'phi_from_spinx_spiny',
           'phi1_from_phi_a_phi_s', 'phi2_from_phi_a_phi_s',
           'spin1z_from_mass1_mass2_chi_eff_chi_a',
           'spin2z_from_mass1_mass2_chi_eff_chi_a',
           'spin1x_from_xi1_phi_a_phi_s', 'spin1y_from_xi1_phi_a_phi_s',
           'spin2x_from_mass1_mass2_xi2_phi_a_phi_s',
           'spin2y_from_mass1_mass2_xi2_phi_a_phi_s',
           'chirp_distance', 'det_tc', 'snr_from_loglr',
           'freq_from_final_mass_spin', 'tau_from_final_mass_spin',
           'final_spin_from_f0_tau', 'final_mass_from_f0_tau',
           'final_mass_from_initial', 'final_spin_from_initial',
           'optimal_dec_from_detector', 'optimal_ra_from_detector',
           'chi_eff_from_spherical', 'chi_p_from_spherical',
           'nltides_gw_phase_diff_isco', 'spin_from_pulsar_freq',
           'freqlmn_from_other_lmn', 'taulmn_from_other_lmn'
          ]

# Copyright (C) 2016 Christopher M. Biwer
#
#
#
#
""" Coordinate transformations.
"""

def cartesian_to_spherical_rho(x, y, z):
    """ Calculates the magnitude in spherical coordinates from Cartesian

    x : {numpy.array, float}
        X-coordinate.
    y : {numpy.array, float}
        Y-coordinate.
    z : {numpy.array, float}
        Z-coordinate.

    Returns
    -------
    rho : {numpy.array, float}
        The radial amplitude.
    """
    return numpy.sqrt(x**2 + y**2 + z**2)

def cartesian_to_spherical_azimuthal(x, y):
    """ Calculates the azimuthal angle in spherical coordinates from Cartesian
    coordinates. The azimuthal angle is in [0,2*pi].

    Returns
    -------
    phi : {numpy.array, float}
        The azimuthal angle.
    """
    y = float(y) if isinstance(y, int) else y
    phi = numpy.arctan2(y, x)

def cartesian_to_spherical_polar(x, y, z):
    """ Calculates the polar angle in spherical coordinates from Cartesian
    coordinates. The polar angle is in [0,pi].

    Returns
    -------
    theta : {numpy.array, float}
        The polar angle.
    """
    return numpy.arccos(z / cartesian_to_spherical_rho(x, y, z))

def cartesian_to_spherical(x, y, z):
    """ Maps cartesian coordinates (x,y,z) to spherical coordinates
    (rho,phi,theta) where phi is in [0,2*pi] and theta is in [0,pi].

    Returns
    -------
    """
    rho = cartesian_to_spherical_rho(x, y, z)
    phi = cartesian_to_spherical_azimuthal(x, y)
    theta = cartesian_to_spherical_polar(x, y, z)
    return rho, phi, theta

def spherical_to_cartesian(rho, phi, theta):
    """ Maps spherical coordinates (rho,phi,theta) to cartesian coordinates
    (x,y,z) where phi is in [0,2*pi] and theta is in [0,pi].

    Returns
    -------
    """
    x = rho * numpy.cos(phi) * numpy.sin(theta)
    y = rho * numpy.sin(phi) * numpy.sin(theta)
    z = rho * numpy.cos(theta)
    return x, y, z

__all__ = ['cartesian_to_spherical_rho', 'cartesian_to_spherical_azimuthal',
           'cartesian_to_spherical_polar', 'cartesian_to_spherical',
           'spherical_to_cartesian',
          ]

# Copyright (C) 2017  Collin Capano
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides functions for computing cosmological quantities, such as
redshift. This is mostly a wrapper around ``astropy.cosmology``.

Note: in all functions, ``distance`` is short hand for ``luminosity_distance``.
Any other distance measure is explicitly named; e.g., ``comoving_distance``.
"""

import logging
from scipy import interpolate
import astropy.cosmology
from astropy import units
from astropy.cosmology.core import CosmologyError
import pycbc.conversions

DEFAULT_COSMOLOGY = 'Planck15'

def get_cosmology(cosmology=None, **kwargs):
    r"""Gets an astropy cosmology class.

    cosmology : str or astropy.cosmology.FlatLambdaCDM, optional
        The name of the cosmology to use. For the list of options, see
        :py:attr:`astropy.cosmology.parameters.available`. If None, and no
        other keyword arguments are provided, will default to
        :py:attr:`DEFAULT_COSMOLOGY`. If an instance of
        :py:class:`astropy.cosmology.FlatLambdaCDM`, will just return that.
    \**kwargs :
        If any other keyword arguments are provided they will be passed to
        :py:attr:`astropy.cosmology.FlatLambdaCDM` to create a custom
        cosmology.

    Returns
    -------
    astropy.cosmology.FlatLambdaCDM
        The cosmology to use.

    Examples
    --------
    Use the default:

    >>> from pycbc.cosmology import get_cosmology
    >>> get_cosmology()
    FlatLambdaCDM(name="Planck15", H0=67.7 km / (Mpc s), Om0=0.307,
                  Tcmb0=2.725 K, Neff=3.05, m_nu=[0.   0.   0.06] eV,
                  Ob0=0.0486)

    Use properties measured by WMAP instead:

    >>> get_cosmology("WMAP9")
    FlatLambdaCDM(name="WMAP9", H0=69.3 km / (Mpc s), Om0=0.286, Tcmb0=2.725 K,
                  Neff=3.04, m_nu=[0. 0. 0.] eV, Ob0=0.0463)

    Create your own cosmology (see :py:class:`astropy.cosmology.FlatLambdaCDM`
    for details on the default values used):

    >>> get_cosmology(H0=70., Om0=0.3)
    FlatLambdaCDM(H0=70 km / (Mpc s), Om0=0.3, Tcmb0=0 K, Neff=3.04, m_nu=None,
                  Ob0=None)

    """
    if kwargs and cosmology is not None:
        raise ValueError("if providing custom cosmological parameters, do "
                         "not provide a `cosmology` argument")
    if isinstance(cosmology, astropy.cosmology.FlatLambdaCDM):
        # just return
        return cosmology
    if kwargs:
        cosmology = astropy.cosmology.FlatLambdaCDM(**kwargs)
    else:
        if cosmology is None:
            cosmology = DEFAULT_COSMOLOGY
        if cosmology not in astropy.cosmology.parameters.available:
            raise ValueError("unrecognized cosmology {}".format(cosmology))
        cosmology = getattr(astropy.cosmology, cosmology)
    return cosmology

def z_at_value(func, fval, unit, zmax=1000., **kwargs):
    r"""Wrapper around astropy.cosmology.z_at_value to handle numpy arrays.

    Getting a z for a cosmological quantity involves numerically inverting
    ``func``. The ``zmax`` argument sets how large of a z to guess (see
    :py:func:`astropy.cosmology.z_at_value` for details). If a z is larger than
    ``zmax``, this will try a larger zmax up to ``zmax * 10**5``. If that still
    is not large enough, will just return ``numpy.inf``.

    func : function or method
        A function that takes redshift as input.
    fval : float
        The value of ``func(z)``.
    unit : astropy.unit
        The unit of ``fval``.
    zmax : float, optional
        The initial maximum search limit for ``z``. Default is 1000.
        All other keyword arguments are passed to
        :py:func:``astropy.cosmology.z_at_value``.

    Returns
    -------
    float
        The redshift at the requested values.
    """
    fval, input_is_array = pycbc.conversions.ensurearray(fval)
    # make sure fval is atleast 1D
    if fval.size == 1 and fval.ndim == 0:
        fval = fval.reshape(1)
    zs = numpy.zeros(fval.shape, dtype=float)  # the output array
    for (ii, val) in enumerate(fval):
        try:
            zs[ii] = astropy.cosmology.z_at_value(func, val*unit, zmax=zmax,
                                                  **kwargs)
        except CosmologyError:
            # we'll get this if the z was larger than zmax; in that case we'll
            # try bumping up zmax later to get a value
            zs[ii] = numpy.inf
    # check if there were any zs > zmax
    replacemask = numpy.isinf(zs)
    # try bumping up zmax to get a result
    if replacemask.any():
        # we'll keep bumping up the maxz until we can get a result
        counter = 0  # to prevent running forever
        while replacemask.any():
            kwargs['zmin'] = zmax
            zmax = 10 * zmax
            idx = numpy.where(replacemask)
            for ii in idx:
                val = fval[ii]
                try:
                    zs[ii] = astropy.cosmology.z_at_value(
                        func, val*unit, zmax=zmax, **kwargs)
                    replacemask[ii] = False
                except CosmologyError:
                    # didn't work, try on next loop
                    pass
            counter += 1
            if counter == 5:
                # give up and warn the user
                logging.warning("One or more values correspond to a "
                                "redshift > {0:.1e}. The redshift for these "
                                "have been set to inf. If you would like "
                                "better precision, call God.".format(zmax))
                break
    return pycbc.conversions.formatreturn(zs, input_is_array)

def _redshift(distance, **kwargs):
    r"""Uses astropy to get redshift from the given luminosity distance.

    distance : float
        The luminosity distance, in Mpc.
        All other keyword args are passed to :py:func:`get_cosmology` to
        select a cosmology. If none provided, will use
        :py:attr:`DEFAULT_COSMOLOGY`.

    Returns
    -------
    float :
        The redshift corresponding to the given luminosity distance.
    """
    cosmology = get_cosmology(**kwargs)
    return z_at_value(cosmology.luminosity_distance, distance, units.Mpc)

class DistToZ(object):
    r"""Interpolates luminosity distance as a function of redshift to allow for
    fast conversion.

    The :mod:`astropy.cosmology` module provides methods for converting any
    cosmological parameter (like luminosity distance) to redshift. This can be
    very slow when operating on a large array, as it involves numerically
    inverting :math:`z(D)` (where :math:`D` is the luminosity distance). This
    class speeds that up by pre-interpolating :math:`D(z)`. It works by setting
    up a dense grid of redshifts, then using linear interpolation to find the
    inverse function.  The interpolation uses a grid linear in z for z < 1, and
    log in z for ``default_maxz`` > z > 1. This interpolater is setup the first
    time `get_redshift` is called.  If a distance is requested that results in
    a z > ``default_maxz``, the class falls back to calling astropy directly.

    Instances of this class can be called like a function on luminosity
    distances, which will return the corresponding redshifts.

    default_maxz : float, optional
        The maximum z to interpolate up to before falling back to calling
        astropy directly. Default is 1000.
    numpoints : int, optional
        The number of points to use in the linear interpolation between 0 to 1
        and 1 to ``default_maxz``. Default is 10000.
    """
    def __init__(self, default_maxz=1000., numpoints=10000, **kwargs):
        self.numpoints = int(numpoints)
        self.default_maxz = default_maxz
        self.cosmology = get_cosmology(**kwargs)
        # the interpolating functions; we'll set them to None for now, then set
        # them up when get_redshift is first called
        self.nearby_d2z = None
        self.faraway_d2z = None
        self.default_maxdist = None

    def setup_interpolant(self):
        """Initializes the z(d) interpolation."""
        # for computing nearby (z < 1) redshifts
        zs = numpy.linspace(0., 1., num=self.numpoints)
        ds = self.cosmology.luminosity_distance(zs).value
        self.nearby_d2z = interpolate.interp1d(ds, zs, kind='linear',
                                                bounds_error=False)
        # for computing far away (z > 1) redshifts
        zs = numpy.logspace(0, numpy.log10(self.default_maxz),
                            num=self.numpoints)
        self.faraway_d2z = interpolate.interp1d(ds, zs, kind='linear',
                                                 bounds_error=False)
        # store the default maximum distance
        self.default_maxdist = ds.max()

    def get_redshift(self, dist):
        """Returns the redshift for the given distance.
        """
        dist, input_is_array = pycbc.conversions.ensurearray(dist)
        try:
            zs = self.nearby_d2z(dist)
        except TypeError:
            # interpolant hasn't been setup yet
            self.setup_interpolant()
        # if any points had red shifts beyond the nearby, will have nans;
        # replace using the faraway interpolation
        replacemask = numpy.isnan(zs)
        if replacemask.any():
            zs[replacemask] = self.faraway_d2z(dist[replacemask])
            replacemask = numpy.isnan(zs)
        # if we still have nans, means that some distances are beyond our
        # furthest default; fall back to using astropy
            # well... check that the distance is positive and finite first
            if not (dist > 0.).all() and numpy.isfinite(dist).all():
                raise ValueError("distance must be finite and > 0")
            zs[replacemask] = _redshift(dist[replacemask],
                                        cosmology=self.cosmology)
        return pycbc.conversions.formatreturn(zs, input_is_array)

    def __call__(self, dist):
        return self.get_redshift(dist)

# set up D(z) interpolating classes for the standard cosmologies
_d2zs = {_c: DistToZ(cosmology=_c)
         for _c in astropy.cosmology.parameters.available}

def redshift(distance, **kwargs):
    r"""Returns the redshift associated with the given luminosity distance.

    If the requested cosmology is one of the pre-defined ones in
    :py:attr:`astropy.cosmology.parameters.available`, :py:class:`DistToZ` is
    used to provide a fast interpolation. This takes a few seconds to setup
    on the first call.

    Returns
    -------
    float :
        The redshift corresponding to the given distance.
    """
    try:
        z = _d2zs[cosmology.name](distance)
    except KeyError:
        # not a standard cosmology, call the redshift function
        z = _redshift(distance, cosmology=cosmology)
    return z

class ComovingVolInterpolator(object):
    r"""Interpolates comoving volume to distance or redshift.

    parameter : {'luminosity_distance', 'redshift'}
        What parameter to interpolate.
    """
    def __init__(self, parameter, default_maxz=10., numpoints=1000, **kwargs):
        self.parameter = parameter
        self.nearby_interp = None
        self.faraway_interp = None
        self.default_maxvol = None

    def _create_interpolant(self, minz, maxz):
        minlogv = numpy.log(self.cosmology.comoving_volume(minz).value)
        maxlogv = numpy.log(self.cosmology.comoving_volume(maxz).value)
        logvs = numpy.linspace(minlogv, maxlogv, num=self.numpoints)
        zs = z_at_value(self.cosmology.comoving_volume, numpy.exp(logvs),
                        units.Mpc**3)
        if self.parameter != 'redshift':
            ys = cosmological_quantity_from_redshift(zs, self.parameter)
        else:
            ys = zs
        return interpolate.interp1d(logvs, ys, kind='linear',
                                    bounds_error=False)

        # get VC bounds
        minz = 0.001
        maxz = 1.
        self.nearby_interp = self._create_interpolant(minz, maxz)
        minz = 1.
        maxz = self.default_maxz
        self.faraway_interp = self._create_interpolant(minz, maxz)
        # store the default maximum volume
        self.default_maxvol = \
            numpy.log(self.cosmology.comoving_volume(maxz).value)

    def get_value_from_logv(self, logv):
        """
        logv, input_is_array = pycbc.conversions.ensurearray(logv)
        try:
            vals = self.nearby_interp(logv)
        replacemask = numpy.isnan(vals)
            vals[replacemask] = self.faraway_interp(logv[replacemask])
            replacemask = numpy.isnan(vals)
            # well... check that the logv is finite first
            if not numpy.isfinite(logv).all():
                raise ValueError("comoving volume must be finite and > 0")
            zs = z_at_value(self.cosmology.comoving_volume,
                            numpy.exp(logv[replacemask]), units.Mpc**3)
            if self.parameter == 'redshift':
                vals[replacemask] = zs
            else:
                vals[replacemask] = \
                    getattr(self.cosmology, self.parameter)(zs).value
        return pycbc.conversions.formatreturn(vals, input_is_array)

    def get_value(self, volume):
        return self.get_value_from_logv(numpy.log(volume))

    def __call__(self, volume):
        return self.get_value(volume)

_v2ds = {_c: ComovingVolInterpolator('luminosity_distance', cosmology=_c)

_v2zs = {_c: ComovingVolInterpolator('redshift', cosmology=_c)

def redshift_from_comoving_volume(vc, interp=True, **kwargs):
    r"""Returns the redshift from the given comoving volume.

    vc : float
        The comoving volume, in units of cubed Mpc.
    interp : bool, optional
        If true, this will setup an interpolator between redshift and comoving
        volume the first time this function is called. This is useful when
        making many successive calls to this function (and is necessary when
        using this function in a transform when doing parameter estimation).
        However, setting up the interpolator the first time takes O(10)s of
        seconds. If you will only be making a single call to this function, or
        will only run it on an array with < ~100000 elements, it is faster to
        not use the interpolator (i.e., set ``interp=False``). Default is
        ``True``.

    Returns
    -------
    float :
        The redshift at the given comoving volume.
    """
    lookup = _v2zs if interp else {}
    try:
        z = lookup[cosmology.name](vc)
        # not using interp or not a standard cosmology,
        # call the redshift function directly
        z = z_at_value(cosmology.comoving_volume, vc, units.Mpc**3)
    return z

def distance_from_comoving_volume(vc, interp=True, **kwargs):
    r"""Returns the luminosity distance from the given comoving volume.

    vc : float
        If true, this will setup an interpolator between distance and comoving
        making many successive calls to this function (such as when using this
        function in a transform for parameter estimation).  However, setting up
        the interpolator the first time takes O(10)s of seconds. If you will
        only be making a single call to this function, or will only run it on
        an array with < ~100000 elements, it is faster to not use the
        interpolator (i.e., set ``interp=False``). Default is ``True``.

    Returns
    -------
    float :
        The luminosity distance at the given comoving volume.
    """
    lookup = _v2ds if interp else {}
    try:
        dist = lookup[cosmology.name](vc)
        dist = cosmology.luminosity_distance(z).value
    return dist

def cosmological_quantity_from_redshift(z, quantity, strip_unit=True,
                                        **kwargs):
    r"""Returns the value of a cosmological quantity (e.g., age) at a redshift.

    z : float
        The redshift.
    quantity : str
        The name of the quantity to get. The name may be any attribute of
        :py:class:`astropy.cosmology.FlatLambdaCDM`.
    strip_unit : bool, optional
        Just return the value of the quantity, sans units. Default is True.

    Returns
    -------
    float or astropy.units.quantity :
        The value of the quantity at the requested value. If ``strip_unit`` is
        ``True``, will return the value. Otherwise, will return the value with
        units.
    """
    val = getattr(cosmology, quantity)(z)
    if strip_unit:
        val = val.value
    return val

__all__ = ['redshift', 'redshift_from_comoving_volume',
           'distance_from_comoving_volume',
           'cosmological_quantity_from_redshift',
           ]

# -*- coding: UTF-8 -*-

# Copyright (C) 2012  Alex Nitz
#
#
#
#

#
#
#                                   Preamble
#
#
"""This module provides utilities for calculating detector responses and timing
between observatories.
"""
import os
import numpy as np
import lal
from pycbc.types import TimeSeries
from pycbc.types.config import InterpolatingConfigParser
from astropy.time import Time
from astropy import constants, coordinates, units
from astropy.coordinates.matrix_utilities import rotation_matrix
from astropy.units.si import sday, meter
from numpy import cos, sin, pi

# Response functions are modelled after those in lalsuite and as also
# presented in https://arxiv.org/pdf/gr-qc/0008066.pdf

def gmst_accurate(gps_time):
    gmst = Time(gps_time, format='gps', scale='utc',
                location=(0, 0)).sidereal_time('mean').rad
    return gmst

def get_available_detectors():
    """Return list of detectors known in the currently sourced lalsuite.
    This function will query lalsuite about which detectors are known to
    lalsuite. Detectors are identified by a two character string e.g. 'K1',
    but also by a longer, and clearer name, e.g. KAGRA. This function returns
    both. As LAL doesn't really expose this functionality we have to make some
    assumptions about how this information is stored in LAL. Therefore while
    we hope this function will work correctly, it's possible it will need
    updating in the future. Better if lal would expose this information
    properly.
    """
    ld = lal.__dict__
    known_lal_names = [j for j in ld.keys() if "DETECTOR_PREFIX" in j]
    known_prefixes = [ld[k] for k in known_lal_names]
    known_names = [ld[k.replace('PREFIX', 'NAME')] for k in known_lal_names]
    return list(zip(known_prefixes, known_names))

_custom_ground_detectors = {}
def add_detector_on_earth(name, longitude, latitude,
                          yangle=0, xangle=None, height=0):
    """ Add a new detector on the earth

    name: str
        two-letter name to identify the detector
    longitude: float
        Longitude in radians using geodetic coordinates of the detector
    latitude: float
        Latitude in radians using geodetic coordinates of the detector
    yangle: float
        Azimuthal angle of the y-arm (angle drawn from pointing north)
    xangle: float
        Azimuthal angle of the x-arm (angle drawn from point north). If not set
        we assume a right angle detector following the right-hand rule.
    height: float
        The height in meters of the detector above the standard
        reference ellipsoidal earth
    """
    if xangle is None:
        # assume right angle detector if no separate xarm direction given
        xangle = yangle + np.pi / 2.0

    # Calculate response in earth centered coordinates
    # by rotation of response in coordinates aligned
    # with the detector arms
    a, b = cos(2*xangle), sin(2*xangle)
    xresp = np.array([[-a, b, 0], [b, a, 0], [0, 0, 0]])
    a, b = cos(2*yangle), sin(2*yangle)
    yresp = np.array([[-a, b, 0], [b, a, 0], [0, 0, 0]])
    resp = (yresp - xresp) / 4.0

    rm1 = rotation_matrix(longitude * units.rad, 'z')
    rm2 = rotation_matrix((np.pi / 2.0 - latitude) * units.rad, 'y')
    rm = np.matmul(rm2, rm1)

    resp = np.matmul(resp, rm)
    resp = np.matmul(rm.T, resp)

    loc = coordinates.EarthLocation.from_geodetic(longitude * units.rad,
                                                  latitude * units.rad,
                                                  height=height*units.meter)
    loc = np.array([loc.x.value,
                    loc.y.value,
                    loc.z.value])
    _custom_ground_detectors[name] = {'location': loc,
                                      'response': resp,
                                      'yangle': yangle,
                                      'xangle': xangle,
                                      'height': height,
                                      'xaltitude': 0.0,
                                      'yaltitude': 0.0,
                                      }

def load_detector_config(config_files):
    """ Add custom detectors from a configuration file

    config_files: str or list of strs
        The config file(s) which specify new detectors
    """
    methods = {'earth_normal': (add_detector_on_earth,
                                ['longitude', 'latitude'])}
    conf = InterpolatingConfigParser(config_files)
    dets = conf.get_subsections('detector')
    for det in dets:
        kwds = dict(conf.items('detector-{}'.format(det)))
        try:
            method, arg_names = methods[kwds.pop('method')]
            raise ValueError("Missing or unkown method, "
                             "options are {}".format(methods.keys()))
        for k in kwds:
            kwds[k] = float(kwds[k])
        try:
            args = [kwds.pop(arg) for arg in arg_names]
        except KeyError as e:
            raise ValueError("missing required detector argument"
                             " {} are required".format(arg_names))
        method(det.upper(), *args, **kwds)

# autoload detector config files
if 'PYCBC_DETECTOR_CONFIG' in os.environ:
    load_detector_config(os.environ['PYCBC_DETECTOR_CONFIG'].split(':'))

class Detector(object):
    """A gravitational wave detector
    """
    def __init__(self, detector_name, reference_time=1126259462.0):
        """ Create class representing a gravitational-wave detector
        detector_name: str
            The two-character detector string, i.e. H1, L1, V1, K1, I1
        reference_time: float
            Default is time of GW150914. In this case, the earth's rotation
        will be estimated from a reference time. If 'None', we will
        calculate the time for each gps time requested explicitly
        using a slower but higher precision method.
        """
        self.name = str(detector_name)

        if detector_name in [pfx for pfx, name in get_available_detectors()]:
            import lalsimulation as lalsim
            self._lal = lalsim.DetectorPrefixToLALDetector(self.name)
            self.response = self._lal.response
            self.location = self._lal.location
        elif detector_name in _custom_ground_detectors:
            self.info = _custom_ground_detectors[detector_name]
            self.response = self.info['response']
            self.location = self.info['location']
        else:
            raise ValueError("Unkown detector {}".format(detector_name))

        loc = coordinates.EarthLocation(self.location[0],
                                        self.location[1],
                                        self.location[2],
                                        unit=meter)
        self.latitude = loc.lat.rad
        self.longitude = loc.lon.rad

        self.reference_time = reference_time
        self.sday = None
        self.gmst_reference = None

    def set_gmst_reference(self):
        if self.reference_time is not None:
            self.sday = float(sday.si.scale)
            self.gmst_reference = gmst_accurate(self.reference_time)
        else:
            raise RuntimeError("Can't get accurate sidereal time without GPS "
                               "reference time!")

    def lal(self):
        """ Return lal data type detector instance """
        if hasattr(self, '_lal'):
            return self._lal
        else:
            import lal
            d = lal.FrDetector()
            d.vertexLongitudeRadians = self.longitude
            d.vertexLatitudeRadians = self.latitude
            d.vertexElevation = self.info['height']
            d.xArmAzimuthRadians = self.info['xangle']
            d.yArmAzimuthRadians = self.info['yangle']
            d.xArmAltitudeRadians = self.info['yaltitude']
            d.xArmAltitudeRadians = self.info['xaltitude']

            # This is somewhat abused by lalsimulation at the moment
            # to determine a filter kernel size. We set this only so that
            # value gets a similar number of samples as other detectors
            # it is used for nothing else
            d.yArmMidpoint = 4000.0

            x = lal.Detector()
            r = lal.CreateDetector(x, d, lal.LALDETECTORTYPE_IFODIFF)
            self._lal = r
            return r

    def gmst_estimate(self, gps_time):
        if self.reference_time is None:
            return gmst_accurate(gps_time)

        if self.gmst_reference is None:
            self.set_gmst_reference()
        dphase = (gps_time - self.reference_time) / self.sday * (2.0 * np.pi)
        gmst = (self.gmst_reference + dphase) % (2.0 * np.pi)
        return gmst

    def light_travel_time_to_detector(self, det):
        """ Return the light travel time from this detector
        det: Detector
            The other detector to determine the light travel time to.
        Returns
        -------
        time: float
            The light travel time in seconds
        """
        d = self.location - det.location
        return float(d.dot(d)**0.5 / constants.c.value)

    def antenna_pattern(self, right_ascension, declination, polarization, t_gps, polarization_type='tensor'):
        """Return the detector response.

        right_ascension: float or numpy.ndarray
            The right ascension of the source
        declination: float or numpy.ndarray
            The declination of the source
        polarization: float or numpy.ndarray
            The polarization angle of the source
        polarization_type: string flag: Tensor, Vector or Scalar
            The gravitational wave polarizations. Default: 'Tensor'

        Returns
        -------
        fplus(default) or fx or fb : float or numpy.ndarray
            The plus or vector-x or breathing polarization factor for this sky location / orientation
        fcross(default) or fy or fl : float or numpy.ndarray
            The cross or vector-y or longitudnal polarization factor for this sky location / orientation
        """
        if isinstance(t_gps, lal.LIGOTimeGPS):
            t_gps = float(t_gps)
        gha = self.gmst_estimate(t_gps) - right_ascension

        cosgha = cos(gha)
        singha = sin(gha)
        cosdec = cos(declination)
        sindec = sin(declination)
        cospsi = cos(polarization)
        sinpsi = sin(polarization)

        x0 = -cospsi * singha - sinpsi * cosgha * sindec
        x1 = -cospsi * cosgha + sinpsi * singha * sindec
        x2 =  sinpsi * cosdec

        x = np.array([x0, x1, x2], dtype=object)
        dx = self.response.dot(x)

        y0 =  sinpsi * singha - cospsi * cosgha * sindec
        y1 =  sinpsi * cosgha + cospsi * singha * sindec
        y2 =  cospsi * cosdec

        y = np.array([y0, y1, y2], dtype=object)
        dy = self.response.dot(y)

        if polarization_type != 'tensor':
            z0 = -cosdec * cosgha
            z1 = cosdec * singha
            z2 = -sindec
            z = np.array([z0, z1, z2], dtype=object)
            dz = self.response.dot(z)

        if polarization_type == 'tensor':
            if hasattr(dx, 'shape'):
                fplus = (x * dx - y * dy).sum(axis=0).astype(np.float64)
                fcross = (x * dy + y * dx).sum(axis=0).astype(np.float64)
            else:
                fplus = (x * dx - y * dy).sum()
                fcross = (x * dy + y * dx).sum()
            return fplus, fcross

        elif polarization_type == 'vector':
                fx = (z * dx + x * dz).sum(axis=0).astype(np.float64)
                fy = (z * dy + y * dz).sum(axis=0).astype(np.float64)
            else:
                fx = (z * dx + x * dz).sum()
                fy = (z * dy + y * dz).sum()
            return fx, fy

        elif polarization_type == 'scalar':
                fb = (x * dx + y * dy).sum(axis=0).astype(np.float64)
                fl = (z * dz).sum(axis=0)
            else:
                fb = (x * dx + y * dy).sum()
                fl = (z * dz).sum()
            return fb, fl

    def time_delay_from_earth_center(self, right_ascension, declination, t_gps):
        """Return the time delay from the earth center
        """
        return self.time_delay_from_location(np.array([0, 0, 0]),
                                             right_ascension,
                                             declination,
                                             t_gps)

    def time_delay_from_location(self, other_location, right_ascension,
                                 declination, t_gps):
        """Return the time delay from the given location to detector for
        a signal with the given sky location
        In other words return `t1 - t2` where `t1` is the
        arrival time in this detector and `t2` is the arrival time in the
        other location.

        other_location : numpy.ndarray of coordinates
            A detector instance.
        right_ascension : float
            The right ascension (in rad) of the signal.
        declination : float
            The declination (in rad) of the signal.
        t_gps : float
            The GPS time (in s) of the signal.

        Returns
        -------
        float
            The arrival time difference between the detectors.
        """
        ra_angle = self.gmst_estimate(t_gps) - right_ascension
        cosd = cos(declination)

        e0 = cosd * cos(ra_angle)
        e1 = cosd * -sin(ra_angle)
        e2 = sin(declination)

        ehat = np.array([e0, e1, e2], dtype=object)
        dx = other_location - self.location
        return dx.dot(ehat).astype(np.float64) / constants.c.value

    def time_delay_from_detector(self, other_detector, right_ascension,
        """Return the time delay from the given to detector for a signal with
        the given sky location; i.e. return `t1 - t2` where `t1` is the
        other detector. Note that this would return the same value as
        `time_delay_from_earth_center` if `other_detector` was geocentric.
        other_detector : detector.Detector
        Returns
        -------
        float
        """
        return self.time_delay_from_location(other_detector.location,
                                             t_gps)

    def project_wave(self, hp, hc, ra, dec, polarization,
                     method='lal',
                     reference_time=None):
        """Return the strain of a waveform as measured by the detector.
        Apply the time shift for the given detector relative to the assumed
        geocentric frame and apply the antenna patterns to the plus and cross
        polarizations.

        hp: pycbc.types.TimeSeries
            Plus polarization of the GW
        hc: pycbc.types.TimeSeries
            Cross polarization of the GW
        ra: float
            Right ascension of source location
        dec: float
            Declination of source location
        polarization: float
            Polarization angle of the source
        method: {'lal', 'constant', 'vary_polarization'}
            The method to use for projecting the polarizations into the
            detector frame. Default is 'lal'.
        reference_time: float, Optional
            The time to use as, a reference for some methods of projection.
            Used by 'constant' and 'vary_polarization' methods. Uses average
            time if not provided.
        """
        # The robust and most fefature rich method which includes
        # time changing antenna patterns and doppler shifts due to the
        # earth rotation and orbit
        if method == 'lal':
            import lalsimulation
            h_lal = lalsimulation.SimDetectorStrainREAL8TimeSeries(
                    hp.astype(np.float64).lal(), hc.astype(np.float64).lal(),
                    ra, dec, polarization, self.lal())
            ts = TimeSeries(
                    h_lal.data.data, delta_t=h_lal.deltaT, epoch=h_lal.epoch,
                    dtype=np.float64, copy=False)

        # 'constant' assume fixed orientation relative to source over the
        # duration of the signal, accurate for short duration signals
        # 'fixed_polarization' applies only time changing orientation
        # but no doppler corrections
        elif method in ['constant', 'vary_polarization']:
            if reference_time is not None:
                rtime = reference_time
            else:
                # In many cases, one should set the reference time if using
                # this method as we don't know where the signal is within
                # the given time series. If not provided, we'll choose
                # the midpoint time.
                rtime = (float(hp.end_time) + float(hp.start_time)) / 2.0

            if method == 'constant':
                time = rtime
            elif method == 'vary_polarization':
                if (not isinstance(hp, TimeSeries) or
                   not isinstance(hc, TimeSeries)):
                    raise TypeError('Waveform polarizations must be given'
                                    ' as time series for this method')

                # this is more granular than needed, may be optimized later
                # assume earth rotation in ~30 ms needed for earth ceneter
                # to detector is completely negligible.
                time = hp.sample_times.numpy()

            fp, fc = self.antenna_pattern(ra, dec, polarization, time)
            dt = self.time_delay_from_earth_center(ra, dec, rtime)
            ts = fp * hp + fc * hc
            ts.start_time = float(ts.start_time) + dt

        # add in only the correction for the time variance in the polarization
        # due to the earth's rotation, no doppler correction applied
        else:
            raise ValueError("Unkown projection method {}".format(method))
        return ts

    def optimal_orientation(self, t_gps):
        """Return the optimal orientation in right ascension and declination
           for a given GPS time.

        t_gps: float
            Time in gps seconds

        Returns
        -------
        ra: float
            Right ascension that is optimally oriented for the detector
        dec: float
            Declination that is optimally oriented for the detector
        """
        ra = self.longitude + (self.gmst_estimate(t_gps) % (2.0*np.pi))
        dec = self.latitude
        return ra, dec

    def get_icrs_pos(self):
        """ Transforms GCRS frame to ICRS frame

        Returns
        loc: numpy.ndarray shape (3,1) units: AU
             ICRS coordinates in cartesian system
        """
        loc = self.location
        loc = coordinates.SkyCoord(x=loc[0], y=loc[1], z=loc[2], unit=units.m,
                frame='gcrs', representation_type='cartesian').transform_to('icrs')
        loc.representation_type = 'cartesian'
        conv = np.float32(((loc.x.unit/units.AU).decompose()).to_string())
        loc = np.array([np.float32(loc.x), np.float32(loc.y),
                        np.float32(loc.z)])*conv
        return loc

    def effective_distance(self, distance, ra, dec, pol, time, inclination):
        """ Distance scaled to account for amplitude factors

        The effective distance of the source. This scales the distance so that
        the amplitude is equal to a source which is optimally oriented with
        respect to the detector. For fixed detector-frame intrinsic parameters
        this is a measure of the expected signal strength.

        distance: float
            Source luminosity distance in megaparsecs
        ra: float
            The right ascension in radians
        dec: float
            The declination in radians
        pol: float
            Polarization angle of the gravitational wave in radians
            GPS time in seconds
        inclination:
            The inclination of the binary's orbital plane

        Returns
        -------
        eff_dist: float
            The effective distance of the source
        """
        fp, fc = self.antenna_pattern(ra, dec, pol, time)
        ic = np.cos(inclination)
        ip = 0.5 * (1. + ic * ic)
        scale = ((fp * ip) ** 2.0 + (fc * ic) ** 2.0) ** 0.5
        return distance / scale

def overhead_antenna_pattern(right_ascension, declination, polarization):
    """Return the antenna pattern factors F+ and Fx as a function of sky
    location and polarization angle for a hypothetical interferometer located
    at the north pole. Angles are in radians. Declinations of ±π/2 correspond
    to the normal to the detector plane (i.e. overhead and underneath) while
    the point with zero right ascension and declination is the direction
    of one of the interferometer arms.
    right_ascension: float
    declination: float
    polarization: float
    Returns
    -------
    f_plus: float
    f_cros: float
    """
    # convert from declination coordinate to polar (angle dropped from north axis)
    theta = np.pi / 2.0 - declination

    f_plus  = - (1.0/2.0) * (1.0 + cos(theta)*cos(theta)) * \
                cos (2.0 * right_ascension) * cos (2.0 * polarization) - \
                cos(theta) * sin(2.0*right_ascension) * sin (2.0 * polarization)

    f_cross =   (1.0/2.0) * (1.0 + cos(theta)*cos(theta)) * \
                cos (2.0 * right_ascension) * sin (2.0* polarization) - \
                cos(theta) * sin(2.0*right_ascension) * cos (2.0 * polarization)

    return f_plus, f_cross

"""     LISA class      """

class LISA(object):
    """For LISA detector
    """
    def __init__(self):
        None

    def get_pos(self, ref_time):
        """Return the position of LISA detector for a given reference time
        ref_time : numpy.ScalarType

        Returns
        -------
        location : numpy.ndarray of shape (3,3)
               Returns the position of all 3 sattelites with each row
               correspoding to a single axis.
        """
        ref_time = Time(val=ref_time, format='gps', scale='utc').jyear
        n = np.array(range(1, 4))
        kappa, _lambda_ = 0, 0
        alpha = 2. * np.pi * ref_time/1 + kappa
        beta_n = (n - 1) * 2.0 * pi / 3.0 + _lambda_
        a, L = 1., 0.03342293561
        e = L/(2. * a * np.sqrt(3))

        x = a*cos(alpha) + a*e*(sin(alpha)*cos(alpha)*sin(beta_n) - (1 + sin(alpha)**2)*cos(beta_n))
        y = a*sin(alpha) + a*e*(sin(alpha)*cos(alpha)*cos(beta_n) - (1 + cos(alpha)**2)*sin(beta_n))
        z = -np.sqrt(3)*a*e*cos(alpha - beta_n)
        self.location = np.array([x, y, z])

        return self.location

    def get_gcrs_pos(self, location):
        """ Transforms ICRS frame to GCRS frame

        loc : numpy.ndarray shape (3,1) units: AU
              Cartesian Coordinates of the location
              in ICRS frame

        Returns
        loc : numpy.ndarray shape (3,1) units: meters
              GCRS coordinates in cartesian system
        """
        loc = location
        loc = coordinates.SkyCoord(x=loc[0], y=loc[1], z=loc[2], unit=units.AU,
                frame='icrs', representation_type='cartesian').transform_to('gcrs')
        conv = np.float32(((loc.x.unit/units.m).decompose()).to_string())
        return loc

        """Return the time delay from the LISA detector to detector for
        a signal with the given sky location. In other words return
        `t1 - t2` where `t1` is the arrival time in this detector and
        `t2` is the arrival time in the other location. Units(AU)

        other_location : numpy.ndarray of coordinates in ICRS frame

        Returns
        -------
        numpy.ndarray
        """
        dx = self.location - other_location
        e0 = cosd * cos(right_ascension)
        e1 = cosd * -sin(right_ascension)
        ehat = np.array([e0, e1, e2])
        return dx.dot(ehat) / constants.c.value

    def time_delay_from_detector(self, det, right_ascension,
        """Return the time delay from the LISA detector for a signal with
        the given sky location in ICRS frame; i.e. return `t1 - t2` where
        `t1` is the arrival time in this detector and `t2` is the arrival
        time in the other detector.

        Returns
        -------
        """
        loc = Detector(det, t_gps).get_icrs_pos()
        return self.time_delay_from_location(loc, right_ascension,
                                             declination, t_gps)

        """Return the time delay from the earth center in ICRS frame
        """
        t_gps = Time(val=t_gps, format='gps', scale='utc')
        earth = coordinates.get_body('earth', t_gps,
                                     location=None).transform_to('icrs')
        earth.representation_type = 'cartesian'
        return self.time_delay_from_location(
            np.array([np.float32(earth.x), np.float32(earth.y),
                      np.float32(earth.z)]), right_ascension,
            declination, t_gps)

# Copyright (C) 2018 Alex Nitz

#
#

#
#
#                                   Preamble
#
#
""" Utilities to query archival instrument status information of
gravitational-wave detectors from public sources and/or dqsegdb.
"""

import json
from ligo.segments import segmentlist, segment
from pycbc.frame.losc import get_run
from pycbc.io import get_file

def parse_veto_definer(veto_def_filename, ifos):
    """ Parse a veto definer file from the filename and return a dictionary
    indexed by ifo and veto definer category level.

    veto_def_filename: str
        The path to the veto definer file
    ifos: str
        The list of ifos for which we require information from the veto
        definer file

    Returns
    --------
    parsed_definition: dict
        Returns a dictionary first indexed by ifo, then category level, and
        finally a list of veto definitions.
    """
    from glue.ligolw import table, lsctables, utils as ligolw_utils
    from glue.ligolw.ligolw import LIGOLWContentHandler as h
    lsctables.use_in(h)

    data = {}
    for ifo_name in ifos:
        data[ifo_name] = {}
        data[ifo_name]['CAT_H'] = []
        for cat_num in range(1, 5):
            data[ifo_name]['CAT_{}'.format(cat_num)] = []

    indoc = ligolw_utils.load_filename(veto_def_filename, False,
                                       contenthandler=h)
    veto_table = table.get_table(indoc, 'veto_definer')

    ifo = veto_table.getColumnByName('ifo')
    name = veto_table.getColumnByName('name')
    version = numpy.array(veto_table.getColumnByName('version'))
    category = numpy.array(veto_table.getColumnByName('category'))
    start = numpy.array(veto_table.getColumnByName('start_time'))
    end = numpy.array(veto_table.getColumnByName('end_time'))
    start_pad = numpy.array(veto_table.getColumnByName('start_pad'))
    end_pad = numpy.array(veto_table.getColumnByName('end_pad'))

    for i in range(len(veto_table)):
        if ifo[i] not in data:
            continue

        # The veto-definer categories are weird! Hardware injections are stored
        # in "3" and numbers above that are bumped up by one (although not
        # often used any more). So we remap 3 to H and anything above 3 to
        # N-1. 2 and 1 correspond to 2 and 1 (YAY!)
        if category[i] > 3:
            curr_cat = "CAT_{}".format(category[i]-1)
        elif category[i] == 3:
            curr_cat = "CAT_H"
        else:
            curr_cat = "CAT_{}".format(category[i])

        veto_info = {'name': name[i],
                     'version': version[i],
                     'full_name': name[i]+':'+str(version[i]),
                     'start': start[i],
                     'end': end[i],
                     'start_pad': start_pad[i],
                     'end_pad': end_pad[i],
                     }
        data[ifo[i]][curr_cat].append(veto_info)
    return data

GWOSC_URL = 'https://www.gw-openscience.org/timeline/segments/json/{}/{}_{}/{}/{}/'

def query_dqsegdb2(detector, flag_name, start_time, end_time, server):
    """Utility function for better error reporting when calling dqsegdb2.
    """
    from dqsegdb2.query import query_segments

    complete_flag = detector + ':' + flag_name
    try:
        query_res = query_segments(complete_flag,
                                   int(start_time),
                                   int(end_time),
                                   host=server)
        return query_res['active']
    except Exception as e:
        logging.error('Could not query segment database, check name '
                      '(%s), times (%d-%d) and server (%s)',
                      complete_flag, int(start_time), int(end_time),
                      server)
        raise e

def query_flag(ifo, segment_name, start_time, end_time,
               source='any', server="https://segments.ligo.org",
               veto_definer=None, cache=False):
    """Return the times where the flag is active

    ifo: string
        The interferometer to query (H1, L1).
    segment_name: string
        The status flag to query from LOSC.
    start_time: int
        The starting gps time to begin querying from LOSC
    end_time: int
        The end gps time of the query
    source: str, Optional
        Choice between "GWOSC" or "dqsegdb". If dqsegdb, the server option may
        also be given. The default is to try GWOSC first then try dqsegdb.
    server: str, Optional
        The server path. Only used with dqsegdb atm.
    veto_definer: str, Optional
        The path to a veto definer to define groups of flags which
        themselves define a set of segments.
    cache: bool
        If true cache the query. Default is not to cache

    Returns
    ---------
    segments: glue.segments.segmentlist
        List of segments
    """
    flag_segments = segmentlist([])

    if source in ['GWOSC', 'any']:
        # Special cases as the LOSC convention is backwards from normal
        # LIGO / Virgo operation!!!!
        if (('_HW_INJ' in segment_name and 'NO' not in segment_name) or
                'VETO' in segment_name):
            data = query_flag(ifo, 'DATA', start_time, end_time)

            if '_HW_INJ' in segment_name:
                name = 'NO_' + segment_name
            else:
                name = segment_name.replace('_VETO', '')

            negate = query_flag(ifo, name, start_time, end_time, cache=cache)
            return (data - negate).coalesce()

        duration = end_time - start_time
        try:
            url = GWOSC_URL.format(get_run(start_time + duration/2),
                                   ifo, segment_name,
                                   int(start_time), int(duration))

            fname = get_file(url, cache=cache, timeout=10)
            data = json.load(open(fname, 'r'))
            if 'segments' in data:
                flag_segments = data['segments']

        except Exception as e:
            if source != 'any':
                raise ValueError("Unable to find {} segments in GWOSC, check "
                                 "flag name or times".format(segment_name))

            return query_flag(ifo, segment_name, start_time, end_time,
                              source='dqsegdb', server=server,
                              veto_definer=veto_definer)

    elif source == 'dqsegdb':
        # The veto definer will allow the use of MACRO names
        # These directly correspond to the name in the veto definer file
        if veto_definer is not None:
            veto_def = parse_veto_definer(veto_definer, [ifo])

        # We treat the veto definer name as if it were its own flag and
        # process the flags in the veto definer
        if veto_definer is not None and segment_name in veto_def[ifo]:
            for flag in veto_def[ifo][segment_name]:
                partial = segmentlist([])
                segs = query_dqsegdb2(ifo, flag['full_name'],
                                      start_time, end_time, server)
                # Apply padding to each segment
                for rseg in segs:
                    seg_start = rseg[0] + flag['start_pad']
                    seg_end = rseg[1] + flag['end_pad']
                    partial.append(segment(seg_start, seg_end))

                # Limit to the veto definer stated valid region of this flag
                flag_start = flag['start']
                flag_end = flag['end']
                # Corner case: if the flag end time is 0 it means 'no limit'
                # so use the query end time
                if flag_end == 0:
                    flag_end = int(end_time)
                send = segmentlist([segment(flag_start, flag_end)])
                flag_segments += (partial.coalesce() & send)

        else:  # Standard case just query directly
            segs = query_dqsegdb2(ifo, segment_name, start_time, end_time,
                                  server)
            for rseg in segs:
                flag_segments.append(segment(rseg[0], rseg[1]))

        # dqsegdb output is not guaranteed to lie entirely within start
        # and end times, hence restrict to this range
        flag_segments = flag_segments.coalesce() & \
            segmentlist([segment(int(start_time), int(end_time))])

    else:
        raise ValueError("Source must be `dqsegdb`, `GWOSC` or `any`."
                         " Got {}".format(source))

    return segmentlist(flag_segments).coalesce()

def query_cumulative_flags(ifo, segment_names, start_time, end_time,
                           source='any', server="https://segments.ligo.org",
                           veto_definer=None,
                           bounds=None,
                           padding=None,
                           override_ifos=None,
                           cache=False):
    """Return the times where any flag is active

    ifo: string or dict
        The interferometer to query (H1, L1). If a dict, an element for each
        flag name must be provided.
    segment_name: list of strings
    bounds: dict, Optional
        Dict containing start-end tuples keyed by the flag name which
        indicate places which should have a distinct time period to be active.
    padding: dict, Optional
        Dict keyed by the flag name. Each element is a tuple
    (start_pad, end_pad) which indicates how to change the segment boundaries.
    override_ifos: dict, Optional
        A dict keyed by flag_name to override the ifo option on a per flag
        basis.

    Returns
    ---------
    """
    total_segs = segmentlist([])
    for flag_name in segment_names:
        ifo_name = ifo
        if override_ifos is not None and flag_name in override_ifos:
            ifo_name = override_ifos[flag_name]

        segs = query_flag(ifo_name, flag_name, start_time, end_time,
                          source=source, server=server,
                          veto_definer=veto_definer,
                          cache=cache)

        if padding and flag_name in padding:
            s, e = padding[flag_name]
            segs2 = segmentlist([])
            for seg in segs:
                segs2.append(segment(seg[0] + s, seg[1] + e))
            segs = segs2

        if bounds is not None and flag_name in bounds:
            s, e = bounds[flag_name]
            valid = segmentlist([segment([s, e])])
            segs = (segs & valid).coalesce()

        total_segs = (total_segs + segs).coalesce()
    return total_segs

def parse_flag_str(flag_str):
    """ Parse a dq flag query string

    flag_str: str
        String to be parsed

    Returns
    -------
    flags: list of strings
        List of reduced name strings which can be passed to lower level
        query commands
    signs: dict
        Dict of bools indicating if the flag should add positively to the
        segmentlist
    ifos: dict
        Ifo specified for the given flag
    bounds: dict
        The boundary of a given flag
    padding: dict
        Any padding that should be applied to the segments for a given flag
    """
    flags = flag_str.replace(' ', '').strip().split(',')

    signs = {}
    ifos = {}
    bounds = {}
    padding = {}
    bflags = []

        # Check if the flag should add or subtract time
        if not (flag[0] == '+' or flag[0] == '-'):
            err_msg = "DQ flags must begin with a '+' or a '-' character. "
            err_msg += "You provided {}. ".format(flag)
            err_msg += "See http://pycbc.org/pycbc/latest/html/workflow/segments.html"
            err_msg += " for more information."
            raise ValueError(err_msg)
        sign = flag[0] == '+'
        flag = flag[1:]

        ifo = pad = bound = None

        # Check for non-default IFO
        if len(flag.split(':')[0]) == 2:
            ifo = flag.split(':')[0]
            flag = flag[3:]

        # Check for padding options
        if '<' in flag:
            popt = flag.split('<')[1].split('>')[0]
            spad, epad = popt.split(':')
            pad = (float(spad), float(epad))
            flag = flag.replace(popt, '').replace('<>', '')

        # Check if there are bounds on the flag
        if '[' in flag:
            bopt = flag.split('[')[1].split(']')[0]
            start, end = bopt.split(':')
            bound = (int(start), int(end))
            flag = flag.replace(bopt, '').replace('[]', '')

        if ifo:
            ifos[flag] = ifo
        if pad:
            padding[flag] = pad
        if bound:
            bounds[flag] = bound
        bflags.append(flag)
        signs[flag] = sign

    return bflags, signs, ifos, bounds, padding

def query_str(ifo, flag_str, start_time, end_time, source='any',
              server="https://segments.ligo.org", veto_definer=None):
    """ Query for flags based on a special str syntax

    ifo: str
        The ifo to query for (may be overridden in syntax)
        Specification of how to do the query. Ex. +H1:DATA:1<-8,8>[0,100000000]
        would return H1 time for the DATA available flag with version 1. It
        would then apply an 8 second padding and only return times within
        the chosen range 0,1000000000.
        The start gps time. May be overridden for individual flags with the
        flag str bounds syntax
        The end gps time. May be overridden for individual flags with the

    Returns
    -------
    segs: segmentlist
        A list of segments corresponding to the flag query string
    """
    flags, sign, ifos, bounds, padding = parse_flag_str(flag_str)
    up = [f for f in flags if sign[f]]
    down = [f for f in flags if not sign[f]]

    if len(up) + len(down) != len(flags):
        raise ValueError('Not all flags could be parsed, check +/- prefix')
    segs = query_cumulative_flags(ifo, up, start_time, end_time,
                                  source=source,
                                  server=server,
                                  veto_definer=veto_definer,
                                  bounds=bounds,
                                  padding=padding,
                                  override_ifos=ifos)

    mseg = query_cumulative_flags(ifo, down, start_time, end_time,

    segs = (segs - mseg).coalesce()
    return segs

# Copyright (C) 2014 Josh Willis
#
# Free Software Foundation; either version 2 of the License, or (at your
#
#

"""
This module provides a simple interface for loading a shared library via ctypes,
allowing it to be specified in an OS-independent way and searched for preferentially
according to the paths that pkg-config specifies.
"""

from __future__ import print_function
import os, fnmatch, ctypes, sys, subprocess
from ctypes.util import find_library
from collections import deque
try:
    from subprocess import getoutput
    from commands import getoutput

def pkg_config(pkg_libraries):
    """Use pkg-config to query for the location of libraries, library directories,
       and header directories

       Arguments:
           pkg_libries(list): A list of packages as strings

       Returns:
           libraries(list), library_dirs(list), include_dirs(list)
    """
    libraries=[]
    library_dirs=[]
    include_dirs=[]

    # Check that we have the packages
    for pkg in pkg_libraries:
        if os.system('pkg-config --exists %s 2>/dev/null' % pkg) == 0:
            pass
        else:
            print("Could not find library {0}".format(pkg))
            sys.exit(1)

    # Get the pck-config flags
    if len(pkg_libraries)>0 :
        # PKG_CONFIG_ALLOW_SYSTEM_CFLAGS explicitly lists system paths.
        # On system-wide LAL installs, this is needed for swig to find lalswig.i
        for token in getoutput("PKG_CONFIG_ALLOW_SYSTEM_CFLAGS=1 pkg-config --libs --cflags %s" % ' '.join(pkg_libraries)).split():
            if token.startswith("-l"):
                libraries.append(token[2:])
            elif token.startswith("-L"):
                library_dirs.append(token[2:])
            elif token.startswith("-I"):
                include_dirs.append(token[2:])

    return libraries, library_dirs, include_dirs

def pkg_config_header_strings(pkg_libraries):
    """ Returns a list of header strings that could be passed to a compiler
    """
    _, _, header_dirs = pkg_config(pkg_libraries)

    header_strings = []

    for header_dir in header_dirs:
        header_strings.append("-I" + header_dir)

    return header_strings

def pkg_config_check_exists(package):
    return (os.system('pkg-config --exists {0} 2>/dev/null'.format(package)) == 0)

def pkg_config_libdirs(packages):
    """
    Returns a list of all library paths that pkg-config says should be included when
    linking against the list of packages given as 'packages'. An empty return list means
    that the package may be found in the standard system locations, irrespective of
    pkg-config.
    """

    # don't try calling pkg-config if NO_PKGCONFIG is set in environment
    if os.environ.get("NO_PKGCONFIG", None):
        return []

    # if calling pkg-config failes, don't continue and don't try again.
    try:
        FNULL = open(os.devnull, 'w')
        subprocess.check_call(["pkg-config", "--version"], stdout=FNULL, close_fds=True)
    except:
        print("PyCBC.libutils: pkg-config call failed, setting NO_PKGCONFIG=1",
              file=sys.stderr)
        os.environ['NO_PKGCONFIG'] = "1"
        return []

    # First, check that we can call pkg-config on each package in the list
    for pkg in packages:
        if not pkg_config_check_exists(pkg):
            raise ValueError("Package {0} cannot be found on the pkg-config search path".format(pkg))

    libdirs = []
    for token in getoutput("PKG_CONFIG_ALLOW_SYSTEM_LIBS=1 pkg-config --libs-only-L {0}".format(' '.join(packages))).split():
        if token.startswith("-L"):
            libdirs.append(token[2:])
    return libdirs

def get_libpath_from_dirlist(libname, dirs):
    """
    This function tries to find the architecture-independent library given by libname in the first
    available directory in the list dirs. 'Architecture-independent' means omitting any prefix such
    as 'lib' or suffix such as 'so' or 'dylib' or version number.  Within the first directory in which
    a matching pattern can be found, the lexicographically first such file is returned, as a string
    giving the full path name.  The only supported OSes at the moment are posix and mac, and this
    function does not attempt to determine which is being run.  So if for some reason your directory
    has both '.so' and '.dylib' libraries, who knows what will happen.  If the library cannot be found,
    None is returned.
    """
    dirqueue = deque(dirs)
    while (len(dirqueue) > 0):
        nextdir = dirqueue.popleft()
        possible = []
        # Our directory might be no good, so try/except
        try:
            for libfile in os.listdir(nextdir):
                if fnmatch.fnmatch(libfile,'lib'+libname+'.so*') or \
                        fnmatch.fnmatch(libfile,'lib'+libname+'.dylib*') or \
                        fnmatch.fnmatch(libfile,libname+'.dll') or \
                        fnmatch.fnmatch(libfile,'cyg'+libname+'-*.dll'):
                    possible.append(libfile)
        except OSError:
            pass
        # There might be more than one library found, we want the highest-numbered
        if (len(possible) > 0):
            possible.sort()
            return os.path.join(nextdir,possible[-1])
    # If we get here, we didn't find it...
    return None

def get_ctypes_library(libname, packages, mode=None):
    """
    This function takes a library name, specified in architecture-independent fashion (i.e.
    omitting any prefix such as 'lib' or suffix such as 'so' or 'dylib' or version number) and
    a list of packages that may provide that library, and according first to LD_LIBRARY_PATH,
    then the results of pkg-config, and falling back to the system search path, will try to
    return a CDLL ctypes object.  If 'mode' is given it will be used when loading the library.
    """
    # First try to get from LD_LIBRARY_PATH
    if "LD_LIBRARY_PATH" in os.environ:
        libdirs += os.environ["LD_LIBRARY_PATH"].split(":")
    # Next try to append via pkg_config
    try:
        libdirs += pkg_config_libdirs(packages)
    except ValueError:
        pass
    # Next, if we are in a virtual environment, search inside its '/lib'
    if "VIRTUAL_ENV" in os.environ:
        libdirs.append(os.path.join(os.environ["VIRTUAL_ENV"], "lib"))

    # Note that the function below can accept an empty list for libdirs, in which case
    # it will return None
    fullpath = get_libpath_from_dirlist(libname,libdirs)

    if fullpath is None:
        # This won't actually return a full-path, but it should be something
        # that can be found by CDLL
        fullpath = find_library(libname)

        # We got nothin'
        return None
    else:
        if mode is None:
            return ctypes.CDLL(fullpath)
        else:
            return ctypes.CDLL(fullpath,mode=mode)

# Module with utilities for estimating candidate events source probabilities
# Initial code by A. Curiel Barroso, August 2019
# Modified by V. Villa-Ortega, January 2020

"""Functions to compute the area corresponding to different CBC on the m1 & m2
plane when given a central mchirp value and uncertainty.
It also includes a function that calculates the source frame when given the
detector frame mass and redshift.
"""

from pycbc.conversions import mass2_from_mchirp_mass1 as m2mcm1
from scipy.integrate import quad
from pycbc.cosmology import _redshift

def insert_args(parser):
    mchirp_group = parser.add_argument_group("Arguments for estimating the "
                                             "source probabilities of a "
                                             "candidate event using the snr, "
                                             "mchirp, and effective distance.")
    mchirp_group.add_argument('--src-class-mass-range', type=float, nargs=2,
                              metavar=('MIN_M2', 'MAX_M1'),
                              default=[1.0, 45.0],
                              help="Minimum and maximum values for the mass "
                                   "of the binary components, used as limits "
                                   "of the mass plane when computing the area "
                                   "corresponding to different CBC sources.")
    mchirp_group.add_argument('--src-class-mass-gap', type=float, nargs=2,
                              metavar=('MAX_NS', 'MIN_BH'), default=[3.0, 5.0],
                              help="Limits of the mass gap, that correspond "
                                   "to the maximum mass of a neutron star "
                                   "and the minimum mass of a black hole. "
                                   "Used as limits of integration of the "
                                   "different CBC regions.")
    mchirp_group.add_argument('--src-class-mchirp-to-delta', type=float,
                              metavar='m0', default=0.01,
                              help='Coefficient to estimate the value of the '
                                   'mchirp uncertainty by mchirp_delta = '
                                   'm0 * mchirp.')
    mchirp_group.add_argument('--src-class-eff-to-lum-distance', type=float,
                              metavar='a0', default=0.759,
                                   'luminosity distance from the minimum '
                                   'eff distance by D_lum = a0 * min(D_eff).')
    mchirp_group.add_argument('--src-class-lum-distance-to-delta', type=float,
                              nargs=2, metavar=('b0', 'b1'),
                              default=[-0.449, -0.342],
                              help='Coefficients to estimate the value of the '
                                   'uncertainty on the luminosity distance '
                                   'from the estimated luminosity distance and'
                                   ' the coinc snr by delta_lum = D_lum * '
                                   'exp(b0) * coinc_snr ** b1.')
    mchirp_group.add_argument('--src-class-mass-gap-separate',
                              action='store_true',
                              help='Gives separate probabilities for each kind'
                                   ' of mass gap CBC sources: GNS, GG, BHG.')

def from_cli(args):
    return {'mass_limits': {'max_m1': args.src_class_mass_range[1],
                            'min_m2': args.src_class_mass_range[0]},
            'mass_bdary': {'ns_max': args.src_class_mass_gap[0],
                           'gap_max': args.src_class_mass_gap[1]},
            'estimation_coeff': {'a0': args.src_class_eff_to_lum_distance,
                                 'b0': args.src_class_lum_distance_to_delta[0],
                                 'b1': args.src_class_lum_distance_to_delta[1],
                                 'm0': args.src_class_mchirp_to_delta},
            'mass_gap': args.src_class_mass_gap_separate}

def src_mass_from_z_det_mass(z, del_z, mdet, del_mdet):
    """Takes values of redshift, redshift uncertainty, detector mass and its
    uncertainty and computes the source mass and its uncertainty.
    """
    msrc = mdet / (1. + z)
    del_msrc = msrc * ((del_mdet / mdet) ** 2.
                       + (del_z / (1. + z)) ** 2.) ** 0.5
    return (msrc, del_msrc)

def intmc(mc, x_min, x_max):
    """Returns the integral of a component mass as a function of the mass of
       the other component, taking mchirp as an argument.
    """
    integral = quad(lambda x, mc: m2mcm1(mc, x), x_min, x_max, args=mc)
    return integral[0]

def calc_areas(trig_mc_det, mass_limits, mass_bdary, z, mass_gap):
    """Computes the area inside the lines of the second component mass as a
    function of the first component mass for the two extreme values
    of mchirp: mchirp +/- mchirp_uncertainty, for each region of the source
    classifying diagram.
    """
    trig_mc = src_mass_from_z_det_mass(z["central"], z["delta"],
                                       trig_mc_det["central"],
                                       trig_mc_det["delta"])
    mcb = trig_mc[0] + trig_mc[1]
    mcs = trig_mc[0] - trig_mc[1]
    m2_min = mass_limits["min_m2"]
    m1_max = mass_limits["max_m1"]
    ns_max = mass_bdary["ns_max"]
    gap_max = mass_bdary["gap_max"]
    # The points where the equal mass line and a chirp mass
    # curve intersect is m1 = m2 = 2**0.2 * mchirp
    mib = (2.**0.2) * mcb
    mis = (2.**0.2) * mcs

    # AREA FOR BBH
    if mib < gap_max:
        abbh = 0.0
    else:
        limb_bbh = min(m1_max, m2mcm1(mcb, gap_max))
        intb_bbh = intmc(mcb, mib, limb_bbh)

        if mis < gap_max:
            lims1_bbh = gap_max
            lims2_bbh = lims1_bbh
        else:
            lims1_bbh = mis
            lims2_bbh = min(m1_max, m2mcm1(mcs, gap_max))

        ints_bbh = intmc(mcs, lims1_bbh, lims2_bbh)

        limdiag_bbh = max(m2mcm1(mcs, lims1_bbh), gap_max)
        intline_sup_bbh = 0.5 * (limdiag_bbh + mib) * (mib - lims1_bbh)
        intline_inf_bbh = (limb_bbh - lims2_bbh) * gap_max
        int_sup_bbh = intb_bbh + intline_sup_bbh
        int_inf_bbh = ints_bbh + intline_inf_bbh

        abbh = int_sup_bbh - int_inf_bbh

    # AREA FOR BHG
    if m2mcm1(mcb, gap_max) < ns_max or m2mcm1(mcs, m1_max) > gap_max:
        abhg = 0.0
    else:
        if m2mcm1(mcb, m1_max) > gap_max:
            limb2_bhg = m1_max
            limb1_bhg = limb2_bhg
        else:
            limb2_bhg = min(m1_max, m2mcm1(mcb, ns_max))
            limb1_bhg = max(gap_max, m2mcm1(mcb, gap_max))

        intb_bhg = intmc(mcb, limb1_bhg, limb2_bhg)

        if m2mcm1(mcs, gap_max) < ns_max:
            lims2_bhg = gap_max
            lims1_bhg = lims2_bhg
        else:
            lims1_bhg = max(gap_max, m2mcm1(mcs, gap_max))
            lims2_bhg = min(m1_max, m2mcm1(mcs, ns_max))

        intline_inf_bhg = (limb2_bhg - lims2_bhg) * ns_max
        intline_sup_bhg = (limb1_bhg - lims1_bhg) * gap_max
        ints_bhg = intmc(mcs, lims1_bhg, lims2_bhg)
        int_sup_bhg = intb_bhg + intline_sup_bhg
        int_inf_bhg = ints_bhg + intline_inf_bhg

        abhg = int_sup_bhg - int_inf_bhg

    # AREA FOR GG
    if m2mcm1(mcs, gap_max) > gap_max or m2mcm1(mcb, ns_max) < ns_max:
        agg = 0.0
    else:
        if m2mcm1(mcb, gap_max) > gap_max:
            limb2_gg = gap_max
            limb1_gg = limb2_gg
        else:
            limb1_gg = mib
            limb2_gg = min(gap_max, m2mcm1(mcb, ns_max))

        intb_gg = intmc(mcb, limb1_gg, limb2_gg)

        if m2mcm1(mcs, ns_max) < ns_max:
            lims2_gg = ns_max
            lims1_gg = lims2_gg
        else:
            lims1_gg = mis
            lims2_gg = min(gap_max, m2mcm1(mcs, ns_max))

        ints_gg = intmc(mcs, lims1_gg, lims2_gg)
        limdiag1_gg = max(m2mcm1(mcs, lims1_gg), ns_max)
        limdiag2_gg = min(m2mcm1(mcb, limb1_gg), gap_max)
        intline_sup_gg = (0.5 * (limb1_gg - lims1_gg)
                          * (limdiag1_gg + limdiag2_gg))
        intline_inf_gg = (limb2_gg - lims2_gg) * ns_max
        int_sup_gg = intb_gg + intline_sup_gg
        int_inf_gg = ints_gg + intline_inf_gg

        agg = int_sup_gg - int_inf_gg

    # AREA FOR BNS
    if m2mcm1(mcs, ns_max) > ns_max:
        abns = 0.0
    else:
        if m2mcm1(mcb, ns_max) > ns_max:
            limb2_bns = ns_max
            limb1_bns = limb2_bns
        else:
            limb2_bns = min(ns_max, m2mcm1(mcb, m2_min))
            limb1_bns = mib

        intb_bns = intmc(mcb, limb1_bns, limb2_bns)

        if mis < m2_min:
            lims2_bns = m2_min
            lims1_bns = lims2_bns
        else:
            lims2_bns = min(ns_max, m2mcm1(mcs, m2_min))
            lims1_bns = mis

        ints_bns = intmc(mcs, lims1_bns, lims2_bns)
        intline_inf_bns = (limb2_bns - lims2_bns) * m2_min
        limdiag1_bns = max(m2mcm1(mcs, lims1_bns), m2_min)
        limdiag2_bns = min(m2mcm1(mcb, limb1_bns), ns_max)
        intline_sup_bns = (0.5 * (limdiag1_bns + limdiag2_bns)
                           * (limb1_bns - lims1_bns))
        int_sup_bns = intb_bns + intline_sup_bns
        int_inf_bns = ints_bns + intline_inf_bns

        abns = int_sup_bns - int_inf_bns

    # AREA FOR GNS
    if m2mcm1(mcs, gap_max) > ns_max or m2mcm1(mcb, ns_max) < m2_min:
        agns = 0.0
    else:
        if m2mcm1(mcb, gap_max) > ns_max:
            limb2_gns = gap_max
            limb1_gns = limb2_gns
        else:
            limb2_gns = min(gap_max, m2mcm1(mcb, m2_min))
            limb1_gns = max(ns_max, m2mcm1(mcb, ns_max))

        intb_gns = intmc(mcb, limb1_gns, limb2_gns)

        if m2mcm1(mcs, ns_max) < m2_min:
            lims2_gns = ns_max
            lims1_gns = lims2_gns
        else:
            lims1_gns = max(ns_max, m2mcm1(mcs, ns_max))
            lims2_gns = min(gap_max, m2mcm1(mcs, m2_min))

        intline_inf_gns = (limb2_gns - lims2_gns) * m2_min
        intline_sup_gns = (limb1_gns - lims1_gns) * ns_max
        ints_gns = intmc(mcs, lims1_gns, lims2_gns)
        int_sup_gns = intb_gns + intline_sup_gns
        int_inf_gns = ints_gns + intline_inf_gns

        agns = int_sup_gns - int_inf_gns

    # AREA FOR NSBH
    if m2mcm1(mcs, m1_max) > ns_max or m2mcm1(mcb, gap_max) < m2_min:
        ansbh = 0.0
    else:
        if m2mcm1(mcb, m1_max) > ns_max:
            limb2_nsbh = m1_max
            limb1_nsbh = limb2_nsbh
        else:
            limb1_nsbh = max(gap_max, m2mcm1(mcb, ns_max))
            limb2_nsbh = min(m1_max, m2mcm1(mcb, m2_min))

        intb_nsbh = intmc(mcb, limb1_nsbh, limb2_nsbh)

        if m2mcm1(mcs, gap_max) < m2_min:
            lims1_nsbh = gap_max
            lims2_nsbh = lims1_nsbh
        else:
            lims1_nsbh = max(gap_max, m2mcm1(mcs, ns_max))
            lims2_nsbh = min(m1_max, m2mcm1(mcs, m2_min))

        intline_inf_nsbh = (limb2_nsbh - lims2_nsbh) * m2_min
        intline_sup_nsbh = (limb1_nsbh - lims1_nsbh) * ns_max
        ints_nsbh = intmc(mcs, lims1_nsbh, lims2_nsbh)
        int_sup_nsbh = intb_nsbh + intline_sup_nsbh
        int_inf_nsbh = ints_nsbh + intline_inf_nsbh

        ansbh = int_sup_nsbh - int_inf_nsbh
    if mass_gap:
        return {
            "BNS": abns,
            "GNS": agns,
            "NSBH": ansbh,
            "GG": agg,
            "BHG": abhg,
            "BBH": abbh
            }
    return {
        "BNS": abns,
        "NSBH": ansbh,
        "BBH": abbh,
        "Mass Gap": agns + agg + abhg
        }

def calc_probabilities(mchirp, snr, eff_distance, src_args):
    """Computes the different probabilities that a candidate event belongs to
       each CBC source category taking as arguments the chirp mass, the
       coincident SNR and the effective distance, and estimating the
       chirp mass uncertainty, the luminosity distance (and its uncertainty)
       and the redshift (and its uncertainty). Probability estimation is done
       assuming it is directly proportional to the area laying in the
       correspondent CBC region.
    """
    mass_limits = src_args['mass_limits']
    mass_bdary = src_args['mass_bdary']
    coeff = src_args['estimation_coeff']
    trig_mc_det = {'central': mchirp, 'delta': mchirp * coeff['m0']}
    dist_estimation = coeff['a0'] * eff_distance
    dist_std_estimation = (dist_estimation * math.exp(coeff['b0']) *
                           snr ** coeff['b1'])
    z_estimation = _redshift(dist_estimation)
    z_est_max = _redshift(dist_estimation + dist_std_estimation)
    z_est_min = _redshift(dist_estimation - dist_std_estimation)
    z_std_estimation = 0.5 * (z_est_max - z_est_min)
    z = {'central': z_estimation, 'delta': z_std_estimation}
    mass_gap = src_args['mass_gap']

    # If the mchirp is greater than the mchirp corresponding to two masses
    # equal to the maximum mass, the probability for BBH is 100%
    mc_max = mass_limits['max_m1'] / (2 ** 0.2)
    if trig_mc_det['central'] > mc_max * (1 + z['central']):
        if mass_gap:
            probabilities = {"BNS": 0.0, "GNS": 0.0, "NSBH": 0.0, "GG": 0.0,
                             "BHG": 0.0, "BBH": 1.0}
        else:
            probabilities = {"BNS": 0.0, "NSBH": 0.0, "BBH": 1.0,
                             "Mass Gap": 0.0}
    else:
        areas = calc_areas(trig_mc_det, mass_limits, mass_bdary, z, mass_gap)
        total_area = sum(areas.values())
        probabilities = {key: areas[key]/total_area for key in areas}
    return probabilities

# Copyright (C) 2015 Joshua Willis
#
#
#

"""
This module defines optimization flags and determines hardware features that some
other modules and packages may use in addition to some optimized utilities.
"""
import os, sys
from collections import OrderedDict

# Work around different Python versions to get runtime
# info on hardware cache sizes
_USE_SUBPROCESS = False
HAVE_GETCONF = False
if os.environ.get("LEVEL2_CACHE_SIZE", None) or os.environ.get("NO_GETCONF", None):
    HAVE_GETCONF = False
elif sys.platform == 'darwin':
    # Mac has getconf, but we can do nothing useful with it
elif sys.version_info >= (2, 7):
    import subprocess
    _USE_SUBPROCESS = True
    HAVE_GETCONF = True
else:
    try:
        import commands # Only available on Unix?
        HAVE_GETCONF = True
    except:
        pass

if os.environ.get("LEVEL2_CACHE_SIZE", None):
    LEVEL2_CACHE_SIZE = int(os.environ["LEVEL2_CACHE_SIZE"])
    logging.info("opt: using LEVEL2_CACHE_SIZE %d from environment" % LEVEL2_CACHE_SIZE)
elif HAVE_GETCONF:
    if _USE_SUBPROCESS:
        def getconf(confvar):
            return int(subprocess.check_output(['getconf', confvar]))
    else:
            retlist = commands.getstatusoutput('getconf ' + confvar)
            return int(retlist[1])

    LEVEL1_DCACHE_SIZE = getconf('LEVEL1_DCACHE_SIZE')
    LEVEL1_DCACHE_ASSOC = getconf('LEVEL1_DCACHE_ASSOC')
    LEVEL1_DCACHE_LINESIZE = getconf('LEVEL1_DCACHE_LINESIZE')
    LEVEL2_CACHE_SIZE = getconf('LEVEL2_CACHE_SIZE')
    LEVEL2_CACHE_ASSOC = getconf('LEVEL2_CACHE_ASSOC')
    LEVEL2_CACHE_LINESIZE = getconf('LEVEL2_CACHE_LINESIZE')
    LEVEL3_CACHE_SIZE = getconf('LEVEL3_CACHE_SIZE')
    LEVEL3_CACHE_ASSOC = getconf('LEVEL3_CACHE_ASSOC')
    LEVEL3_CACHE_LINESIZE = getconf('LEVEL3_CACHE_LINESIZE')

def insert_optimization_option_group(parser):
    """
    Adds the options used to specify optimization-specific options.

    parser : object
        OptionParser instance
    """
    optimization_group = parser.add_argument_group("Options for selecting "
                                   "optimization-specific settings")

    optimization_group.add_argument("--cpu-affinity", help="""
                    A set of CPUs on which to run, specified in a format suitable
                    to pass to taskset.""")
    optimization_group.add_argument("--cpu-affinity-from-env", help="""
                    The name of an enivornment variable containing a set
                    of CPUs on which to run,  specified in a format suitable

def verify_optimization_options(opt, parser):
    """Parses the CLI options, verifies that they are consistent and
    reasonable, and acts on them if they are

    opt : object
        Result of parsing the CLI with OptionParser, or any object with the
        required attributes
        OptionParser instance.
    """

    # Pin to specified CPUs if requested
    requested_cpus = None

    if opt.cpu_affinity_from_env is not None:
        if opt.cpu_affinity is not None:
            logging.error("Both --cpu_affinity_from_env and --cpu_affinity specified")

        requested_cpus = os.environ.get(opt.cpu_affinity_from_env)

        if requested_cpus is None:
            logging.error("CPU affinity requested from environment variable %s "
                          "but this variable is not defined" % opt.cpu_affinity_from_env)

        if requested_cpus == '':
                          "but this variable is empty" % opt.cpu_affinity_from_env)

    if requested_cpus is None:
        requested_cpus = opt.cpu_affinity

    if requested_cpus is not None:
        command = 'taskset -pc %s %d' % (requested_cpus, os.getpid())
        retcode = os.system(command)

        if retcode != 0:
            logging.error('taskset command <%s> failed with return code %d' % \
                          (command, retcode))

        logging.info("Pinned to CPUs %s " % requested_cpus)

class LimitedSizeDict(OrderedDict):
    """ Fixed sized dict for FIFO caching"""

    def __init__(self, *args, **kwds):
        self.size_limit = kwds.pop("size_limit", None)
        OrderedDict.__init__(self, *args, **kwds)
        self._check_size_limit()

    def __setitem__(self, key, value):
        OrderedDict.__setitem__(self, key, value)

    def _check_size_limit(self):
        if self.size_limit is not None:
            while len(self) > self.size_limit:
                self.popitem(last=False)

#
#
#
#

#
#
#                                   Preamble
#
#
"""This module contains convenience pN functions. This includes calculating conversions
between quantities.
"""
import lal, lalsimulation
from scipy.optimize import bisect, brentq, minimize
from pycbc import conversions

def nearest_larger_binary_number(input_len):
    """ Return the nearest binary number larger than input_len.
    """
    return int(2**numpy.ceil(numpy.log2(input_len)))

    return conversions.chirp_distance(dist, mchirp, ref_mass=ref_mass)

def mass1_mass2_to_mtotal_eta(mass1, mass2):
    m_total = conversions.mtotal_from_mass1_mass2(mass1, mass2)
    eta = conversions.eta_from_mass1_mass2(mass1, mass2)
    return m_total,eta

def mtotal_eta_to_mass1_mass2(m_total, eta):
    mass1 = conversions.mass1_from_mtotal_eta(m_total, eta)
    mass2 = conversions.mass2_from_mtotal_eta(m_total, eta)
    return mass1,mass2

def mass1_mass2_to_mchirp_eta(mass1, mass2):
    m_chirp = conversions.mchirp_from_mass1_mass2(mass1, mass2)
    return m_chirp,eta

def mchirp_eta_to_mass1_mass2(m_chirp, eta):
    mtotal = conversions.mtotal_from_mchirp_eta(m_chirp, eta)
    mass1 = conversions.mass1_from_mtotal_eta(mtotal, eta)
    mass2 = conversions.mass2_from_mtotal_eta(mtotal, eta)
    return mass1, mass2

def mchirp_mass1_to_mass2(mchirp, mass1):
    """
    This function takes a value of mchirp and one component mass and returns
    the second component mass. As this is a cubic equation this requires
    finding the roots and returning the one that is real.
    Basically it can be shown that:

    m2^3 - a(m2 + m1) = 0

    where

    a = Mc^5 / m1^3

    this has 3 solutions but only one will be real.
    """
    return conversions.mass2_from_mchirp_mass1(mchirp, mass1)

def eta_mass1_to_mass2(eta, mass1, return_mass_heavier=False, force_real=True):
    """
    This function takes values for eta and one component mass and returns the
    second component mass. Similar to mchirp_mass1_to_mass2 this requires
    finding the roots of a quadratic equation. Basically:

    eta m2^2 + (2 eta - 1)m1 m2 + \eta m1^2 = 0

    This has two solutions which correspond to mass1 being the heavier mass
    or it being the lighter mass. By default the value corresponding to
    mass1 > mass2 is returned. Use the return_mass_heavier kwarg to invert this
    behaviour.
    """
    return conversions.mass_from_knownmass_eta(mass1, eta,
        known_is_secondary=return_mass_heavier, force_real=force_real)

def mchirp_q_to_mass1_mass2(mchirp, q):
    """ This function takes a value of mchirp and the mass ratio
    mass1/mass2 and returns the two component masses.

    The map from q to eta is

        eta = (mass1*mass2)/(mass1+mass2)**2 = (q)/(1+q)**2

    Then we can map from (mchirp,eta) to (mass1,mass2).
    """
    eta = conversions.eta_from_q(q)
    mass1 = conversions.mass1_from_mchirp_eta(mchirp, eta)
    mass2 = conversions.mass2_from_mchirp_eta(mchirp, eta)

def A0(f_lower):
    """used in calculating chirp times: see Cokelaer, arxiv.org:0706.4437
       appendix 1, also lalinspiral/python/sbank/tau0tau3.py
    """
    return conversions._a0(f_lower)

def A3(f_lower):
    """another parameter used for chirp times"""
    return conversions._a3(f_lower)

def mass1_mass2_to_tau0_tau3(mass1, mass2, f_lower):
    tau0 = conversions.tau0_from_mass1_mass2(mass1, mass2, f_lower)
    tau3 = conversions.tau3_from_mass1_mass2(mass1, mass2, f_lower)
    return tau0,tau3

def tau0_tau3_to_mtotal_eta(tau0, tau3, f_lower):
    mtotal = conversions.mtotal_from_tau0_tau3(tau0, tau3, f_lower)
    eta = conversions.eta_from_tau0_tau3(tau0, tau3, f_lower)
    return mtotal, eta

def tau0_tau3_to_mass1_mass2(tau0, tau3, f_lower):
    m_total,eta = tau0_tau3_to_mtotal_eta(tau0, tau3, f_lower)
    return mtotal_eta_to_mass1_mass2(m_total, eta)

def mass1_mass2_spin1z_spin2z_to_beta_sigma_gamma(mass1, mass2,
                                                  spin1z, spin2z):
    _, eta = mass1_mass2_to_mtotal_eta(mass1, mass2)
    # get_beta_sigma_from_aligned_spins() takes
    # the spin of the heaviest body first
    heavy_spin = numpy.where(mass2 <= mass1, spin1z, spin2z)
    light_spin = numpy.where(mass2 > mass1, spin1z, spin2z)
    beta, sigma, gamma = get_beta_sigma_from_aligned_spins(
        eta, heavy_spin, light_spin)
    return beta, sigma, gamma

def get_beta_sigma_from_aligned_spins(eta, spin1z, spin2z):
    """
    Calculate the various PN spin combinations from the masses and spins.
    See <http://arxiv.org/pdf/0810.5336v3.pdf>.

    -----------
    eta : float or numpy.array
        Symmetric mass ratio of the input system(s)
    spin1z : float or numpy.array
        Spin(s) parallel to the orbit of the heaviest body(ies)
    spin2z : float or numpy.array
        Spin(s) parallel to the orbit of the smallest body(ies)

    Returns
    --------
    beta : float or numpy.array
        The 1.5PN spin combination
    sigma : float or numpy.array
        The 2PN spin combination
    gamma : float or numpy.array
        The 2.5PN spin combination
    chis : float or numpy.array
        (spin1z + spin2z) / 2.
    """
    chiS = 0.5 * (spin1z + spin2z)
    chiA = 0.5 * (spin1z - spin2z)
    delta = (1 - 4 * eta) ** 0.5
    spinspin = spin1z * spin2z
    beta = (113. / 12. - 19. / 3. * eta) * chiS
    beta += 113. / 12. * delta * chiA
    sigma = eta / 48. * (474 * spinspin)
    sigma += (1 - 2 * eta) * (81. / 16. * (chiS * chiS + chiA * chiA))
    sigma += delta * (81. / 8. * (chiS * chiA))
    gamma = (732985. / 2268. - 24260. / 81. * eta - \
            340. / 9. * eta * eta) * chiS
    gamma += (732985. / 2268. + 140. / 9. * eta) * delta * chiA

def solar_mass_to_kg(solar_masses):
    return solar_masses * lal.MSUN_SI

def parsecs_to_meters(distance):
    return distance * lal.PC_SI

def megaparsecs_to_meters(distance):
    return parsecs_to_meters(distance) * 1e6

    return conversions.velocity_to_frequency(v, M)

    return conversions.frequency_to_velocity(f, M)

def f_SchwarzISCO(M):
    """

    Returns
    -------
    """
    return conversions.f_schwarzchild_isco(M)

def f_BKLISCO(m1, m2):
    """
    Mass ratio dependent ISCO derived from estimates of the final spin
    of a merged black hole in a paper by Buonanno, Kidder, Lehner
    (arXiv:0709.3839).  See also arxiv:0801.4297v2 eq.(5)

    m1 : float or numpy.array
        First component mass in solar mass units
    m2 : float or numpy.array
        Second component mass in solar mass units

    Returns
    -------
    """
    # q is defined to be in [0,1] for this formula
    q = numpy.minimum(m1/m2, m2/m1)
    return f_SchwarzISCO(m1+m2) * ( 1 + 2.8*q - 2.6*q*q + 0.8*q*q*q )

def f_LightRing(M):
    """
    Gravitational wave frequency corresponding to the light-ring orbit,
    equal to 1/(3**(3/2) pi M) : see InspiralBankGeneration.c

    Returns
    -------
    """
    return 1.0 / (3.0**(1.5) * lal.PI * M * lal.MTSUN_SI)

def f_ERD(M):
    """
    Effective RingDown frequency studied in Pan et al. (arXiv:0704.1964)
    found to give good fit between stationary-phase templates and
    numerical relativity waveforms [NB equal-mass & nonspinning!]
    Equal to 1.07*omega_220/2*pi

    Returns
    -------
    """
    return 1.07 * 0.5326 / (2*lal.PI * 0.955 * M * lal.MTSUN_SI)

def f_FRD(m1, m2):
    """
    Fundamental RingDown frequency calculated from the Berti, Cardoso and
    Will (gr-qc/0512160) value for the omega_220 QNM frequency using
    mass-ratio dependent fits to the final BH mass and spin from Buonanno
    et al. (arXiv:0706.3732) : see also InspiralBankGeneration.c

    Returns
    -------
    """
    m_total, eta = mass1_mass2_to_mtotal_eta(m1, m2)
    tmp = ( (1. - 0.63*(1. - 3.4641016*eta + 2.9*eta**2)**(0.3)) /
    (1. - 0.057191*eta - 0.498*eta**2) )
    return tmp / (2.*lal.PI * m_total*lal.MTSUN_SI)

def f_LRD(m1, m2):
    """
    Lorentzian RingDown frequency = 1.2*FRD which captures part of
    the Lorentzian tail from the decay of the QNMs

    Returns
    -------
    """
    return 1.2 * f_FRD(m1, m2)

def _get_freq(freqfunc, m1, m2, s1z, s2z):
    """
    Wrapper of the LALSimulation function returning the frequency
    for a given frequency function and template parameters.

    freqfunc : lalsimulation FrequencyFunction wrapped object e.g.
        lalsimulation.fEOBNRv2RD
    m1 : float-ish, i.e. castable to float
        First component mass in solar masses
    m2 : float-ish
        Second component mass in solar masses
    s1z : float-ish
        First component dimensionless spin S_1/m_1^2 projected onto L
    s2z : float-ish
        Second component dimensionless spin S_2/m_2^2 projected onto L

    Returns
    -------
    f : float
    """
    # Convert to SI units for lalsimulation
    m1kg = float(m1) * lal.MSUN_SI
    m2kg = float(m2) * lal.MSUN_SI
    return lalsimulation.SimInspiralGetFrequency(
        m1kg, m2kg, 0, 0, float(s1z), 0, 0, float(s2z), int(freqfunc))

# vectorize to enable calls with numpy arrays
_vec_get_freq = numpy.vectorize(_get_freq)

def get_freq(freqfunc, m1, m2, s1z, s2z):
    """
    Returns the LALSimulation function which evaluates the frequency
    for the given frequency function and template parameters.

    freqfunc : string
        Name of the frequency function to use, e.g., 'fEOBNRv2RD'
    s1z : float or numpy.array
    s2z : float or numpy.array

    Returns
    -------
    """
    lalsim_ffunc = getattr(lalsimulation, freqfunc)
    return _vec_get_freq(lalsim_ffunc, m1, m2, s1z, s2z)

def _get_final_freq(approx, m1, m2, s1z, s2z):
    """
    Wrapper of the LALSimulation function returning the final (highest)
    frequency for a given approximant an template parameters

    approx : lalsimulation approximant wrapped object e.g.
        lalsimulation.EOBNRv2

    Returns
    -------
    f : float
    """
    return lalsimulation.SimInspiralGetFinalFreq(
        m1kg, m2kg, 0, 0, float(s1z), 0, 0, float(s2z), int(approx))

_vec_get_final_freq = numpy.vectorize(_get_final_freq)

def get_final_freq(approx, m1, m2, s1z, s2z):
    """
    Returns the LALSimulation function which evaluates the final
    (highest) frequency for a given approximant using given template
    parameters.
    NOTE: TaylorTx and TaylorFx are currently all given an ISCO cutoff !!

    approx : string
        Name of the approximant e.g. 'EOBNRv2'

    Returns
    -------
    """
    lalsim_approx = lalsimulation.GetApproximantFromString(approx)
    return _vec_get_final_freq(lalsim_approx, m1, m2, s1z, s2z)

# Dictionary of functions with uniform API taking a
# parameter dict indexed on mass1, mass2, spin1z, spin2z
named_frequency_cutoffs = {
    # functions depending on the total mass alone
    "SchwarzISCO": lambda p: f_SchwarzISCO(p["mass1"]+p["mass2"]),
    "LightRing"  : lambda p: f_LightRing(p["mass1"]+p["mass2"]),
    "ERD"        : lambda p: f_ERD(p["mass1"]+p["mass2"]),
    # functions depending on the 2 component masses
    "BKLISCO"    : lambda p: f_BKLISCO(p["mass1"], p["mass2"]),
    "FRD"        : lambda p: f_FRD(p["mass1"], p["mass2"]),
    "LRD"        : lambda p: f_LRD(p["mass1"], p["mass2"]),
    # functions depending on 2 component masses and aligned spins
    "MECO"       : lambda p: meco_frequency(p["mass1"], p["mass2"],
                                              p["spin1z"], p["spin2z"]),
    "HybridMECO" : lambda p: hybrid_meco_frequency(
        p["mass1"], p["mass2"], p["spin1z"], p["spin2z"], qm1=None, qm2=None),
    "IMRPhenomBFinal": lambda p: get_freq("fIMRPhenomBFinal",
                                              p["mass1"], p["mass2"],
    "IMRPhenomCFinal": lambda p: get_freq("fIMRPhenomCFinal",
    "IMRPhenomDPeak": lambda p: get_freq("fIMRPhenomDPeak",
    "EOBNRv2RD"   : lambda p: get_freq("fEOBNRv2RD", p["mass1"], p["mass2"],
    "EOBNRv2HMRD" : lambda p: get_freq("fEOBNRv2HMRD", p["mass1"], p["mass2"],
    "SEOBNRv1RD"  : lambda p: get_freq("fSEOBNRv1RD",  p["mass1"], p["mass2"],
    "SEOBNRv1Peak": lambda p: get_freq("fSEOBNRv1Peak", p["mass1"], p["mass2"],
    "SEOBNRv2RD": lambda p: get_freq("fSEOBNRv2RD", p["mass1"], p["mass2"],
                                     p["spin1z"], p["spin2z"]),
    "SEOBNRv2Peak": lambda p: get_freq("fSEOBNRv2Peak", p["mass1"], p["mass2"],
                                       p["spin1z"], p["spin2z"]),
    "SEOBNRv4RD": lambda p: get_freq("fSEOBNRv4RD", p["mass1"], p["mass2"],
    "SEOBNRv4Peak": lambda p: get_freq("fSEOBNRv4Peak", p["mass1"], p["mass2"],
                                       p["spin1z"], p["spin2z"])
    }

def frequency_cutoff_from_name(name, m1, m2, s1z, s2z):
    """
    Returns the result of evaluating the frequency cutoff function
    specified by 'name' on a template with given parameters.

    name : string
        Name of the cutoff function

    Returns
    -------
    """
    params = {"mass1":m1, "mass2":m2, "spin1z":s1z, "spin2z":s2z}
    return named_frequency_cutoffs[name](params)

def _get_imr_duration(m1, m2, s1z, s2z, f_low, approximant="SEOBNRv4"):
    """Wrapper of lalsimulation template duration approximate formula"""
    m1, m2, s1z, s2z, f_low = float(m1), float(m2), float(s1z), float(s2z),\
                              float(f_low)
    if approximant == "SEOBNRv2":
        chi = lalsimulation.SimIMRPhenomBComputeChi(m1, m2, s1z, s2z)
        time_length = lalsimulation.SimIMRSEOBNRv2ChirpTimeSingleSpin(
                                m1 * lal.MSUN_SI, m2 * lal.MSUN_SI, chi, f_low)
    elif approximant == "IMRPhenomD":
        time_length = lalsimulation.SimIMRPhenomDChirpTime(
                           m1 * lal.MSUN_SI, m2 * lal.MSUN_SI, s1z, s2z, f_low)
    elif approximant == "SEOBNRv4":
        # NB for no clear reason this function has f_low as first argument
        time_length = lalsimulation.SimIMRSEOBNRv4ROMTimeOfFrequency(
                           f_low, m1 * lal.MSUN_SI, m2 * lal.MSUN_SI, s1z, s2z)
    elif approximant == 'SPAtmplt' or approximant == 'TaylorF2':
        chi = lalsimulation.SimInspiralTaylorF2ReducedSpinComputeChi(
            m1, m2, s1z, s2z
        )
        time_length = lalsimulation.SimInspiralTaylorF2ReducedSpinChirpTime(
            f_low, m1 * lal.MSUN_SI, m2 * lal.MSUN_SI, chi, -1
        )
    else:
        raise RuntimeError("I can't calculate a duration for %s" % approximant)
    # FIXME Add an extra factor of 1.1 for 'safety' since the duration
    # functions are approximate
    return time_length * 1.1

get_imr_duration = numpy.vectorize(_get_imr_duration)

def get_inspiral_tf(tc, mass1, mass2, spin1, spin2, f_low, n_points=100,
        pn_2order=7, approximant='TaylorF2'):
    """Compute the time-frequency evolution of an inspiral signal.

    Return a tuple of time and frequency vectors tracking the evolution of an
    inspiral signal in the time-frequency plane.
    """
    # handle param-dependent approximant specification
    class Params:
        pass
    params = Params()
    params.mass1 = mass1
    params.mass2 = mass2
    params.spin1z = spin1
    params.spin2z = spin2
    try:
        approximant = eval(approximant, {'__builtins__': None},
                           dict(params=params))
    except (NameError, TypeError):
        pass

    if approximant in ['TaylorF2', 'SPAtmplt']:
        from pycbc.waveform.spa_tmplt import findchirp_chirptime

        # FIXME spins are not taken into account
        f_high = f_SchwarzISCO(mass1 + mass2)
        track_f = numpy.logspace(numpy.log10(f_low), numpy.log10(f_high),
                                 n_points)
        track_t = numpy.array([findchirp_chirptime(float(mass1), float(mass2),
                                        float(f), pn_2order) for f in track_f])
    elif approximant in ['SEOBNRv2', 'SEOBNRv2_ROM_DoubleSpin',
                         'SEOBNRv2_ROM_DoubleSpin_HI']:
        f_high = get_final_freq('SEOBNRv2', mass1, mass2, spin1, spin2)
                                 n_points)
        # use HI function as it has wider freq range validity
        track_t = numpy.array([
                lalsimulation.SimIMRSEOBNRv2ROMDoubleSpinHITimeOfFrequency(f,
                    solar_mass_to_kg(mass1), solar_mass_to_kg(mass2),
                    float(spin1), float(spin2)) for f in track_f])
    elif approximant in ['SEOBNRv4', 'SEOBNRv4_ROM']:
        f_high = get_final_freq('SEOBNRv4', mass1, mass2, spin1, spin2)
        # use frequency below final freq in case of rounding error
        track_f = numpy.logspace(numpy.log10(f_low), numpy.log10(0.999*f_high),
                                 n_points)
                lalsimulation.SimIMRSEOBNRv4ROMTimeOfFrequency(
                        f, solar_mass_to_kg(mass1), solar_mass_to_kg(mass2),
                        float(spin1), float(spin2)) for f in track_f])
    else:
        raise ValueError('Approximant ' + approximant + ' not supported')
    return (tc - track_t, track_f)

##############################This code was taken from Andy ###########

def _energy_coeffs(m1, m2, chi1, chi2):
    """ Return the center-of-mass energy coefficients up to 3.0pN (2.5pN spin)
    """
    mtot = m1 + m2
    eta = m1*m2 / (mtot*mtot)
    chi = (m1*chi1 + m2*chi2) / mtot
    chisym = (chi1 + chi2) / 2.
    beta = (113.*chi - 76.*eta*chisym)/12.
    sigma12 = 79.*eta*chi1*chi2/8.
    sigmaqm = 81.*m1*m1*chi1*chi1/(16.*mtot*mtot) \
            + 81.*m2*m2*chi2*chi2/(16.*mtot*mtot)

    energy0 = -0.5*eta
    energy2 = -0.75 - eta/12.
    energy3 = 0.
    energy4 = -3.375 + (19*eta)/8. - pow(eta,2)/24.
    energy5 = 0.
    energy6 = -10.546875 - (155*pow(eta,2))/96. - (35*pow(eta,3))/5184. \
                + eta*(59.80034722222222 - (205*pow(lal.PI,2))/96.)

    energy3 += (32*beta)/113. + (52*chisym*eta)/113.

    energy4 += (-16*sigma12)/79. - (16*sigmaqm)/81.
    energy5 += (96*beta)/113. + ((-124*beta)/339. - (522*chisym)/113.)*eta \
                - (710*chisym*pow(eta,2))/339.

    return (energy0, energy2, energy3, energy4, energy5, energy6)

def meco_velocity(m1, m2, chi1, chi2):
    """
    Returns the velocity of the minimum energy cutoff for 3.5pN (2.5pN spin)

    m1 : float
    m2 : float
    chi1 : float
    chi2 : float

    Returns
    -------
    v : float
        Velocity (dimensionless)
    """
    _, energy2, energy3, energy4, energy5, energy6 = \
        _energy_coeffs(m1, m2, chi1, chi2)
    def eprime(v):
        return 2. + v * v * (4.*energy2 + v * (5.*energy3 \
                + v * (6.*energy4
                + v * (7.*energy5 + 8.*energy6 * v))))
    return bisect(eprime, 0.05, 1.0)

def _meco_frequency(m1, m2, chi1, chi2):
    """Returns the frequency of the minimum energy cutoff for 3.5pN (2.5pN spin)
    """
    return velocity_to_frequency(meco_velocity(m1, m2, chi1, chi2), m1+m2)

meco_frequency = numpy.vectorize(_meco_frequency)

def _dtdv_coeffs(m1, m2, chi1, chi2):
    """ Returns the dt/dv coefficients up to 3.5pN (2.5pN spin)
    """

    dtdv0 = 1. # FIXME: Wrong but doesn't matter for now.
    dtdv2 = (1./336.) * (743. + 924.*eta)
    dtdv3 = -4. * lal.PI + beta
    dtdv4 = (3058673. + 5472432.*eta + 4353552.*eta*eta)/1016064. - sigma12 - sigmaqm
    dtdv5 = (1./672.) * lal.PI * (-7729. + 1092.*eta) + (146597.*beta/18984. + 42.*beta*eta/113. - 417307.*chisym*eta/18984. - 1389.*chisym*eta*eta/226.)
    dtdv6 = 22.065 + 165.416*eta - 2.20067*eta*eta + 4.93152*eta*eta*eta
    dtdv6log = 1712./315.
    dtdv7 = (lal.PI/1016064.) * (-15419335. - 12718104.*eta + 4975824.*eta*eta)

    return (dtdv0, dtdv2, dtdv3, dtdv4, dtdv5, dtdv6, dtdv6log, dtdv7)

def _dtdv_cutoff_velocity(m1, m2, chi1, chi2):
    _, dtdv2, dtdv3, dtdv4, dtdv5, dtdv6, dtdv6log, dtdv7 = _dtdv_coeffs(m1, m2, chi1, chi2)

    def dtdv_func(v):
        x = dtdv7
        x = v * x + dtdv6 + dtdv6log * 3. * numpy.log(v)
        x = v * x + dtdv5
        x = v * x + dtdv4
        x = v * x + dtdv3
        x = v * x + dtdv2
        return v * v * x + 1.

    if dtdv_func(1.0) < 0.:
        return bisect(dtdv_func, 0.05, 1.0)
    else:
        return 1.0

def energy_coefficients(m1, m2, s1z=0, s2z=0, phase_order=-1, spin_order=-1):
    """ Return the energy coefficients. This assumes that the system has aligned spins only.
    """
    implemented_phase_order = 7
    implemented_spin_order = 7
    if phase_order > implemented_phase_order:
        raise ValueError("pN coeffiecients of that order have not been implemented")
    elif phase_order == -1:
        phase_order = implemented_phase_order

    if spin_order > implemented_spin_order:
    elif spin_order == -1:
        spin_order = implemented_spin_order

    qmdef1 = 1.0
    qmdef2 = 1.0

    M = m1 + m2
    dm = (m1-m2)/M
    m1M = m1 / M
    m2M = m2 / M

    s1z = s1z * m1M * m1M
    s2z = s2z * m2M * m2M

    _, eta = mass1_mass2_to_mchirp_eta(m1, m2)

    ecof = numpy.zeros(phase_order+1)
    # Orbital terms
    if phase_order >= 0:
        ecof[0] = 1.0
    if phase_order >= 1:
        ecof[1] = 0
    if phase_order >= 2:
        ecof[2] = -(1.0/12.0) * (9.0 + eta)
    if phase_order >= 3:
        ecof[3] = 0
    if phase_order >= 4:
        ecof[4] = (-81.0 + 57.0*eta - eta*eta) / 24.0
    if phase_order >= 5:
        ecof[5] = 0
    if phase_order >= 6:
        ecof[6] = - 675.0/64.0 + ( 34445.0/576.0    \
              - 205.0/96.0 * lal.PI * lal.PI ) * eta  \
              - (155.0/96.0) *eta * eta - 35.0/5184.0 * eta * eta
    # Spin terms

    ESO15s1 = 8.0/3.0 + 2.0*m2/m1
    ESO15s2 = 8.0/3.0 + 2.0*m1/m2

    ESS2 = 1.0 / eta
    EQM2s1 = qmdef1/2.0/m1M/m1M
    EQM2s1L = -qmdef1*3.0/2.0/m1M/m1M
    #EQM2s2 = qmdef2/2.0/m2M/m2M
    EQM2s2L = -qmdef2*3.0/2.0/m2M/m2M

    ESO25s1 = 11.0 - 61.0*eta/9.0 + (dm/m1M) * (-3.0 + 10.*eta/3.0)
    ESO25s2 = 11.0 - 61.0*eta/9.0 + (dm/m2M) * (3.0 - 10.*eta/3.0)

    ESO35s1 = 135.0/4.0 - 367.0*eta/4.0 + 29.0*eta*eta/12.0 + (dm/m1M) * (-27.0/4.0 + 39.0*eta - 5.0*eta*eta/4.0)
    ESO35s2 = 135.0/4.0 - 367.0*eta/4.0 + 29.0*eta*eta/12.0 - (dm/m2M) * (-27.0/4.0 + 39.0*eta - 5.0*eta*eta/4.0)

    if spin_order >=3:
        ecof[3] += ESO15s1 * s1z + ESO15s2 * s2z
    if spin_order >=4:
        ecof[4] += ESS2 * (s1z*s2z - 3.0*s1z*s2z)
        ecof[4] += EQM2s1*s1z*s1z + EQM2s1*s2z*s2z + EQM2s1L*s1z*s1z + EQM2s2L*s2z*s2z
    if spin_order >=5:
        ecof[5] = ESO25s1*s1z + ESO25s2*s2z
    if spin_order >=7:
        ecof[7] += ESO35s1*s1z + ESO35s2*s2z

    return ecof

def energy(v, mass1, mass2, s1z=0, s2z=0, phase_order=-1, spin_order=-1):
    ecof = energy_coefficients(mass1, mass2, s1z, s2z, phase_order, spin_order)
    _, eta = mass1_mass2_to_mchirp_eta(mass1, mass2)
    amp = - (1.0/2.0) * eta
    e = 0.0
    for i in numpy.arange(0, len(ecof), 1):
        e += v**(i+2.0) * ecof[i]

    return e * amp

def meco2(m1, m2, s1z=0, s2z=0, phase_order=-1, spin_order=-1):
    ecof = energy_coefficients(m1, m2, s1z, s2z, phase_order, spin_order)

    def test(v):
        de = 0
        for i in numpy.arange(0, len(ecof), 1):
            de += v**(i+1.0)* ecof[i] * (i + 2)

        return de

    return bisect(test, 0.001, 1.0)

def t2_cutoff_velocity(m1, m2, chi1, chi2):
    return min(meco_velocity(m1,m2,chi1,chi2), _dtdv_cutoff_velocity(m1,m2,chi1,chi2))

def t2_cutoff_frequency(m1, m2, chi1, chi2):
    return velocity_to_frequency(t2_cutoff_velocity(m1, m2, chi1, chi2), m1 + m2)

t4_cutoff_velocity = meco_velocity
t4_cutoff_frequency = meco_frequency

# Hybrid MECO in arXiv:1602.03134
# To obtain the MECO, find minimum in v of eq. (6)

def kerr_lightring(v, chi):
    """Return the function whose first root defines the Kerr light ring"""
    return 1 + chi * v**3 - 3 * v**2 * (1 - chi * v**3)**(1./3)

def kerr_lightring_velocity(chi):
    """Return the velocity at the Kerr light ring"""
    # If chi > 0.9996, the algorithm cannot solve the function
    if chi >= 0.9996:
        return brentq(kerr_lightring, 0, 0.8, args=(0.9996))
    else:
        return brentq(kerr_lightring, 0, 0.8, args=(chi))

def hybridEnergy(v, m1, m2, chi1, chi2, qm1, qm2):
    """Return hybrid MECO energy.

    Return the hybrid energy [eq. (6)] whose minimum defines the hybrid MECO
    up to 3.5PN (including the 3PN spin-spin)

    m1 : float
        Mass of the primary object in solar masses.
    m2 : float
        Mass of the secondary object in solar masses.
    chi1: float
        Dimensionless spin of the primary object.
    chi2: float
        Dimensionless spin of the secondary object.
    qm1: float
        Quadrupole-monopole term of the primary object (1 for black holes).
    qm2: float
        Quadrupole-monopole term of the secondary object (1 for black holes).

    Returns
    -------
    h_E: float
        The hybrid energy as a function of v
    """
    pi_sq = numpy.pi**2
    v2, v3, v4, v5, v6, v7 = v**2, v**3, v**4, v**5, v**6, v**7
    chi1_sq, chi2_sq = chi1**2, chi2**2
    m1, m2 = float(m1), float(m2)
    M = float(m1 + m2)
    M_2, M_4 = M**2, M**4
    eta = m1 * m2 / M_2
    eta2, eta3 = eta**2, eta**3
    m1_2, m1_4 = m1**2, m1**4
    m2_2, m2_4 = m2**2, m2**4

    chi = (chi1 * m1 + chi2 * m2) / M
    Kerr = -1. + (1. - 2. * v2 * (1. - chi * v3)**(1./3.)) / \
        numpy.sqrt((1. - chi * v3) * (1. + chi * v3 - 3. * v2 * (1 - chi * v3)**(1./3.)))

    h_E = Kerr - \
        (v2 / 2.) * \
        (
        - eta * v2 / 12. - 2 * (chi1 + chi2) * eta * v3 / 3. +
        (19. * eta / 8. - eta2 / 24. + chi1_sq * m1_2 * (1 - qm1) / M_2 +
            chi2_sq * m2_2 * (1 - qm2) / M_2) * v4
        - 1. / 9. * (120. * (chi1 + chi2) * eta2 +
            (76. * chi1 + 45. * chi2) * m1_2 * eta / M_2 +
            (45. * chi1 + 76. * chi2) * m2_2 * eta / M_2) * v5
        + (34445. * eta / 576. - 205. * pi_sq * eta / 96. - 155. * eta2 / 96. -
            35. * eta3 / 5184. +
            5. / 18. * (21. * chi1_sq * (1. - qm1) * m1_4 / M_4 +
            21. * chi2_sq * (1. - qm2) * m2_4 / M_4 +
            (chi1_sq * (56. - 27. * qm1) + 20. * chi1 * chi2) * eta * m1_2 / M_2 +
            (chi2_sq * (56. - 27. * qm2) + 20. * chi1 * chi2) * eta * m2_2 / M_2 +
            (chi1_sq * (31. - 9. * qm1) + 38. * chi1 * chi2 +
            chi2_sq * (31. - 9. * qm2)) * eta2)) * v6
        - eta / 12. * (3. * (292. * chi1 + 81. * chi2) * m1_4 / M_4 +
            3. * (81. * chi1 + 292. * chi2) * m2_4 / M_4 +
            4. * (673. * chi1 + 360. * chi2) * eta * m1_2 / M_2 +
            4. * (360. * chi1 + 673. * chi2) * eta * m2_2 / M_2 +
            3012. * eta2 * (chi1 + chi2)) * v7
        )

    return h_E

def hybrid_meco_velocity(m1, m2, chi1, chi2, qm1=None, qm2=None):
    """Return the velocity of the hybrid MECO

    m1 : float
    m2 : float
    qm1: {None, float}, optional
        If None, will be set to qm1 = 1.
    qm2: {None, float}, optional
        If None, will be set to qm2 = 1.

    Returns
    -------
    v: float
        The velocity (dimensionless) of the hybrid MECO
    """

    if qm1 is None:
        qm1 = 1
    if qm2 is None:
        qm2 = 1

    # Set bounds at 0.1 to skip v=0 and at the lightring velocity
    chi = (chi1 * m1 + chi2 * m2) / (m1 + m2)
    vmax = kerr_lightring_velocity(chi) - 0.01

    return minimize(hybridEnergy, 0.2, args=(m1, m2, chi1, chi2, qm1, qm2),
                    bounds=[(0.1, vmax)]).x.item()

def hybrid_meco_frequency(m1, m2, chi1, chi2, qm1=None, qm2=None):
    """Return the frequency of the hybrid MECO

    m1 : float
    m2 : float

    Returns
    -------
    f: float
        The frequency (in Hz) of the hybrid MECO
    """
        qm1 = 1
        qm2 = 1

    return velocity_to_frequency(hybrid_meco_velocity(m1, m2, chi1, chi2, qm1, qm2), m1 + m2)

""" Tools for creating pools of worker processes
"""
from __future__ import absolute_import
import multiprocessing.pool
import functools
from multiprocessing import TimeoutError
import types
import signal
import atexit

def is_main_process():
    """ Check if this is the main control process and may handle one time tasks
    """
    try:
        from mpi4py import MPI
        comm = MPI.COMM_WORLD
        rank = comm.Get_rank()
        return rank == 0
    except (ImportError, ValueError, RuntimeError):
        return True

# Allow the pool to be interrupted, need to disable the children processes
# from intercepting the keyboard interrupt
def _noint(init, *args):
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    if init is not None:
        return init(*args)

_process_lock = None
_numdone = None
def _lockstep_fcn(values):
    """ Wrapper to ensure that all processes execute together """
    numrequired, fcn, args = values
    with _process_lock:
        _numdone.value += 1
    # yep this is an ugly busy loop, do something better please
    # when we care about the performance of this call and not just the
    # guarantee it provides (ok... maybe never)
    while 1:
        if _numdone.value == numrequired:
            return fcn(args)

def _shutdown_pool(p):
    p.terminate()
    p.join()

class BroadcastPool(multiprocessing.pool.Pool):
    """ Multiprocessing pool with a broadcast method
    """
    def __init__(self, processes=None, initializer=None, initargs=(), **kwds):
        global _process_lock
        global _numdone
        _process_lock = multiprocessing.Lock()
        _numdone = multiprocessing.Value('i', 0)
        noint = functools.partial(_noint, initializer)
        super(BroadcastPool, self).__init__(processes, noint, initargs, **kwds)
        atexit.register(_shutdown_pool, self)

        return len(self._pool)

    def broadcast(self, fcn, args):
        """ Do a function call on every worker.

        fcn: funtion
            Function to call.
        args: tuple
            The arguments for Pool.map
        """
        results = self.map(_lockstep_fcn, [(len(self), fcn, args)] * len(self))
        _numdone.value = 0
        return results

    def allmap(self, fcn, args):
        """ Do a function call on every worker with different arguments

        """
        results = self.map(_lockstep_fcn,
                           [(len(self), fcn, arg) for arg in args])

    def map(self, func, items, chunksize=None):
        """ Catch keyboard interuppts to allow the pool to exit cleanly.

        func: function
            Function to call
        items: list of tuples
            Arguments to pass
        chunksize: int, Optional
            Number of calls for each process to handle at once
        """
        results = self.map_async(func, items, chunksize)
        while True:
            try:
                return results.get(1800)
            except TimeoutError:
                pass
            except KeyboardInterrupt:
                self.terminate()
                self.join()
                raise KeyboardInterrupt

def _dummy_broadcast(self, f, args):
    self.map(f, [args] * self.size)

class SinglePool(object):
        return self.map(fcn, [args])

    def map(self, f, items):
        return [f(a) for a in items]

def use_mpi(require_mpi=False, log=True):
    """ Get whether MPI is enabled and if so the current size and rank
    """
    use_mpi = False
    size = rank = 0
    try:
        size = comm.Get_size()
        if size > 1:
            use_mpi = True
            if log:
                logging.info('Running under mpi with size: %s, rank: %s',
                             size, rank)
    except ImportError as e:
        if require_mpi:
            print(e)
            raise ValueError("Failed to load mpi, ensure mpi4py is installed")
    return use_mpi, size, rank

def choose_pool(processes, mpi=False):
    """ Get processing pool
    """
    do_mpi, size, rank = use_mpi(require_mpi=mpi)
    if do_mpi:
        try:
            import schwimmbad
            pool = schwimmbad.choose_pool(mpi=do_mpi,
                                          processes=(size - 1))
            pool.broadcast = types.MethodType(_dummy_broadcast, pool)
            atexit.register(pool.close)

            if processes:
                logging.info('NOTE: that for MPI process size determined by '
                             'MPI launch size, not the processes argument')

            if do_mpi and not mpi:
                logging.info('NOTE: using MPI as this process was launched'
                             'under MPI')
        except ImportError:
            raise ValueError("Failed to start up an MPI pool, "
                             "install mpi4py / schwimmbad")
    elif processes == 1:
        pool = SinglePool()
    else:
        pool = BroadcastPool(processes)

    pool.size = processes
    if size:
        pool.size = size
    return pool

import bisect
from . import bin_utils

def integral_element(mu, pdf):
    '''
    Returns an array of elements of the integrand dP = p(mu) dmu
    for a density p(mu) defined at sample values mu ; samples need
    not be equally spaced.  Uses a simple trapezium rule.
    Number of dP elements is 1 - (number of mu samples).
    '''
    dmu = mu[1:] - mu[:-1]
    bin_mean = (pdf[1:] + pdf[:-1]) / 2.
    return dmu * bin_mean

def normalize_pdf(mu, pofmu):
    """
    Takes a function pofmu defined at rate sample values mu and
    normalizes it to be a suitable pdf. Both mu and pofmu must be
    arrays or lists of the same length.
    """
    if min(pofmu) < 0:
        raise ValueError("Probabilities cannot be negative, don't ask me to "
                         "normalize a function with negative values!")
    if min(mu) < 0:
        raise ValueError("Rates cannot be negative, don't ask me to "
                         "normalize a function over a negative domain!")

    dp = integral_element(mu, pofmu)
    return mu, pofmu/sum(dp)

def compute_upper_limit(mu_in, post, alpha=0.9):
    """
    Returns the upper limit mu_high of confidence level alpha for a
    posterior distribution post on the given parameter mu.
    The posterior need not be normalized.
    """
    if 0 < alpha < 1:
        dp = integral_element(mu_in, post)
        high_idx = bisect.bisect_left(dp.cumsum() / dp.sum(), alpha)
        # if alpha is in (0,1] and post is non-negative, bisect_left
        # will always return an index in the range of mu since
        # post.cumsum()/post.sum() will always begin at 0 and end at 1
        mu_high = mu_in[high_idx]
    elif alpha == 1:
        mu_high = numpy.max(mu_in[post > 0])
    else:
        raise ValueError("Confidence level must be in (0,1].")

    return mu_high

def compute_lower_limit(mu_in, post, alpha=0.9):
    """
    Returns the lower limit mu_low of confidence level alpha for a
    """
        low_idx = bisect.bisect_right(dp.cumsum() / dp.sum(), 1 - alpha)
        # if alpha is in [0,1) and post is non-negative, bisect_right
        mu_low = mu_in[low_idx]
        mu_low = numpy.min(mu_in[post > 0])
    else:

    return mu_low

def confidence_interval_min_width(mu, post, alpha=0.9):
    '''
    Returns the minimal-width confidence interval [mu_low, mu_high] of
    confidence level alpha for a posterior distribution post on the parameter mu.
    '''
    if not 0 < alpha < 1:
        raise ValueError("Confidence level must be in (0,1).")

    # choose a step size for the sliding confidence window
    alpha_step = 0.01

    # initialize the lower and upper limits
    mu_low = numpy.min(mu)
    mu_high = numpy.max(mu)

    # find the smallest window (by delta-mu) stepping by dalpha
    for ai in numpy.arange(0, 1 - alpha, alpha_step):
        ml = compute_lower_limit(mu, post, 1 - ai)
        mh = compute_upper_limit(mu, post, alpha + ai)
        if mh - ml < mu_high - mu_low:
            mu_low = ml
            mu_high = mh

    return mu_low, mu_high

def hpd_coverage(mu, pdf, thresh):
    '''
    Integrates a pdf over mu taking only bins where
    the mean over the bin is above a given threshold
    This gives the coverage of the HPD interval for
    the given threshold.
    '''
    dp = integral_element(mu, pdf)

    return dp[bin_mean > thresh].sum()

def hpd_threshold(mu_in, post, alpha, tol):
    '''
    For a PDF post over samples mu_in, find a density
    threshold such that the region having higher density
    has coverage of at least alpha, and less than alpha
    plus a given tolerance.
    '''
    norm_post = normalize_pdf(mu_in, post)
    # initialize bisection search
    p_minus = 0.0
    p_plus = max(post)
    while abs(hpd_coverage(mu_in, norm_post, p_minus) -
              hpd_coverage(mu_in, norm_post, p_plus)) >= tol:
        p_test = (p_minus + p_plus) / 2.
        if hpd_coverage(mu_in, post, p_test) >= alpha:
            # test value was too low or just right
            p_minus = p_test
        else:
            # test value was too high
            p_plus = p_test
    # p_minus never goes above the required threshold and p_plus never goes below
    # thus on exiting p_minus is at or below the required threshold and the
    # difference in coverage is within tolerance
    return p_minus

def hpd_credible_interval(mu_in, post, alpha=0.9, tolerance=1e-3):
    '''
    Returns the minimum and maximum rate values of the HPD
    (Highest Posterior Density) credible interval for a posterior
    post defined at the sample values mu_in.  Samples need not be
    uniformly spaced and posterior need not be normalized.

    Will not return a correct credible interval if the posterior
    is multimodal and the correct interval is not contiguous;
    in this case will over-cover by including the whole range from
    minimum to maximum mu.
    '''
    if alpha == 1:
        nonzero_samples = mu_in[post > 0]
        mu_low = numpy.min(nonzero_samples)
        mu_high = numpy.max(nonzero_samples)
    elif 0 < alpha < 1:
        # determine the highest PDF for which the region with
        # higher density has sufficient coverage
        pthresh = hpd_threshold(mu_in, post, alpha, tol=tolerance)
        samples_over_threshold = mu_in[post > pthresh]
        mu_low = numpy.min(samples_over_threshold)
        mu_high = numpy.max(samples_over_threshold)

# Following functions are for the old pylal volume vs mass calculations
# These were replaced by 'imr_utils' functions now contained in sensitivity.py
# and bin_utils.py

def integrate_efficiency(dbins, eff, err=0, logbins=False):

    if logbins:
        logd = numpy.log(dbins)
        dlogd = logd[1:] - logd[:-1]
        # use log midpoint of bins
        dreps = numpy.exp((numpy.log(dbins[1:]) + numpy.log(dbins[:-1])) / 2.)
        vol = numpy.sum(4.*numpy.pi * dreps**3. * eff * dlogd)
        # propagate errors in eff to errors in v
        verr = numpy.sqrt(
            numpy.sum((4.*numpy.pi * dreps**3. * err * dlogd)**2.)
        )
    else:
        dd = dbins[1:] - dbins[:-1]
        dreps = (dbins[1:] + dbins[:-1]) / 2.
        vol = numpy.sum(4. * numpy.pi * dreps**2. * eff * dd)
        # propagate errors
        verr = numpy.sqrt(numpy.sum((4.*numpy.pi * dreps**2. * err * dd)**2.))

    return vol, verr

def compute_efficiency(f_dist, m_dist, dbins):
    '''
    Compute the efficiency as a function of distance for the given sets of found
    and missed injection distances.
    Note that injections that do not fit into any dbin get lost :(
    '''
    efficiency = numpy.zeros(len(dbins) - 1)
    error = numpy.zeros(len(dbins) - 1)
    for j, dlow in enumerate(dbins[:-1]):
        dhigh = dbins[j + 1]
        found = numpy.sum((dlow <= f_dist) * (f_dist < dhigh))
        missed = numpy.sum((dlow <= m_dist) * (m_dist < dhigh))
        if found+missed == 0:
            # avoid divide by 0 in empty bins
            missed = 1.
        efficiency[j] = float(found) / (found + missed)
        error[j] = numpy.sqrt(efficiency[j] * (1 - efficiency[j]) /
                              (found + missed))

    return efficiency, error

def mean_efficiency_volume(found, missed, dbins):

    if len(found) == 0:
        # no efficiency here
        return numpy.zeros(len(dbins) - 1), numpy.zeros(len(dbins) - 1), 0, 0

    # only need distances
    f_dist = numpy.array([l.distance for l in found])
    m_dist = numpy.array([l.distance for l in missed])

    # compute the efficiency and its variance
    eff, err = compute_efficiency(f_dist, m_dist, dbins)
    vol, verr = integrate_efficiency(dbins, eff, err)

    return eff, err, vol, verr

def filter_injections_by_mass(injs, mbins, bin_num, bin_type, bin_num2=None):
    '''
    For a given set of injections (sim_inspiral rows), return the subset
    of injections that fall within the given mass range.
    '''
    if bin_type == "Mass1_Mass2":
        m1bins = numpy.concatenate((mbins.lower()[0],
                                    numpy.array([mbins.upper()[0][-1]])))
        m1lo = m1bins[bin_num]
        m1hi = m1bins[bin_num + 1]
        m2bins = numpy.concatenate((mbins.lower()[1],
                                    numpy.array([mbins.upper()[1][-1]])))
        m2lo = m2bins[bin_num2]
        m2hi = m2bins[bin_num2 + 1]
        newinjs = [l for l in injs if
                   ((m1lo <= l.mass1 < m1hi and m2lo <= l.mass2 < m2hi) or
                    (m1lo <= l.mass2 < m1hi and m2lo <= l.mass1 < m2hi))]
        return newinjs

    mbins = numpy.concatenate((mbins.lower()[0],
                               numpy.array([mbins.upper()[0][-1]])))
    mlow = mbins[bin_num]
    mhigh = mbins[bin_num + 1]
    if bin_type == "Chirp_Mass":
        newinjs = [l for l in injs if (mlow <= l.mchirp < mhigh)]
    elif bin_type == "Total_Mass":
        newinjs = [l for l in injs if (mlow <= l.mass1 + l.mass2 < mhigh)]
    elif bin_type == "Component_Mass":
        # here it is assumed that m2 is fixed
        newinjs = [l for l in injs if (mlow <= l.mass1 < mhigh)]
    elif bin_type == "BNS_BBH":
        if bin_num in [0, 2]:
            # BNS/BBH case
            newinjs = [l for l in injs if
                       (mlow <= l.mass1 < mhigh and mlow <= l.mass2 < mhigh)]
        else:
            # NSBH
            newinjs = [l for l in injs if (mbins[0] <= l.mass1 < mbins[1] and
                                           mbins[2] <= l.mass2 < mbins[3])]
            # BHNS
            newinjs += [l for l in injs if (mbins[0] <= l.mass2 < mbins[1] and
                                            mbins[2] <= l.mass1 < mbins[3])]

    return newinjs

def compute_volume_vs_mass(found, missed, mass_bins, bin_type, dbins=None):
    """
    Compute the average luminosity an experiment was sensitive to

    Assumes that luminosity is uniformly distributed in space.
    Input is the sets of found and missed injections.
    """
    # mean and std estimate for luminosity
    volArray = bin_utils.BinnedArray(mass_bins)
    vol2Array = bin_utils.BinnedArray(mass_bins)

    # found/missed stats
    foundArray = bin_utils.BinnedArray(mass_bins)
    missedArray = bin_utils.BinnedArray(mass_bins)

    # compute the mean luminosity in each mass bin
    effvmass = []
    errvmass = []
    # 2D case first
        for j, mc1 in enumerate(mass_bins.centres()[0]):
            for k, mc2 in enumerate(mass_bins.centres()[1]):
                newfound = filter_injections_by_mass(
                                              found, mass_bins, j, bin_type, k)
                newmissed = filter_injections_by_mass(
                                             missed, mass_bins, j, bin_type, k)

                foundArray[(mc1, mc2)] = len(newfound)
                missedArray[(mc1, mc2)] = len(newmissed)

                # compute the volume using this injection set
                meaneff, efferr, meanvol, volerr = mean_efficiency_volume(
                                                    newfound, newmissed, dbins)
                effvmass.append(meaneff)
                errvmass.append(efferr)
                volArray[(mc1, mc2)] = meanvol
                vol2Array[(mc1, mc2)] = volerr

        return volArray, vol2Array, foundArray, missedArray, effvmass, errvmass

    for j, mc in enumerate(mass_bins.centres()[0]):

        # filter out injections not in this mass bin
        newfound = filter_injections_by_mass(found, mass_bins, j, bin_type)
        newmissed = filter_injections_by_mass(missed, mass_bins, j, bin_type)

        foundArray[(mc, )] = len(newfound)
        missedArray[(mc, )] = len(newmissed)

        # compute the volume using this injection set
        meaneff, efferr, meanvol, volerr = mean_efficiency_volume(
        effvmass.append(meaneff)
        errvmass.append(efferr)
        volArray[(mc, )] = meanvol
        vol2Array[(mc, )] = volerr

    return volArray, vol2Array, foundArray, missedArray, effvmass, errvmass

# Copyright (C) 2014  Alex Nitz, Andrew Miller
#
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides python contexts that set the default behavior for PyCBC
objects.
"""
import os
import pycbc
from decorator import decorator
from .libutils import get_ctypes_library

try:
    _libgomp = get_ctypes_library("gomp", ['gomp'])
except:
    # Should we fail or give a warning if we cannot import
    # libgomp? Seems to work even for MKL scheme, but
    # not entirely sure why...
    _libgomp = None

class _SchemeManager(object):
    _single = None

        if _SchemeManager._single is not None:
            raise RuntimeError("SchemeManager is a private class")
        _SchemeManager._single= self

        self.state= None
        self._lock= False

    def lock(self):
        self._lock= True

    def unlock(self):

    def shift_to(self, state):
        if self._lock is False:
            self.state = state
        else:
            raise RuntimeError("The state is locked, cannot shift schemes")

# Create the global processing scheme manager
mgr = _SchemeManager()
DefaultScheme = None
default_context = None

class Scheme(object):
    """Context that sets PyCBC objects to use CPU processing. """
        if DefaultScheme is type(self):
            return
        if Scheme._single is not None:
            raise RuntimeError("Only one processing scheme can be used")
        Scheme._single = True
    def __enter__(self):
        mgr.shift_to(self)
        mgr.lock()
    def __exit__(self, type, value, traceback):
        mgr.unlock()
        mgr.shift_to(default_context)
    def __del__(self):
        if Scheme is not None:
            Scheme._single = None

_cuda_cleanup_list=[]

def register_clean_cuda(function):
    _cuda_cleanup_list.append(function)

def clean_cuda(context):
    #Before cuda context is destroyed, all item destructions dependent on cuda
    # must take place. This calls all functions that have been registered
    # with _register_clean_cuda() in reverse order
    #So the last one registered, is the first one cleaned
    _cuda_cleanup_list.reverse()
    for func in _cuda_cleanup_list:
        func()

    context.pop()
    from pycuda.tools import clear_context_caches
    clear_context_caches()

class CUDAScheme(Scheme):
    """Context that sets PyCBC objects to use a CUDA processing scheme. """
    def __init__(self, device_num=0):
        Scheme.__init__(self)
        if not pycbc.HAVE_CUDA:
            raise RuntimeError("Install PyCUDA to use CUDA processing")
        import pycuda.driver
        pycuda.driver.init()
        self.device = pycuda.driver.Device(device_num)
        self.context = self.device.make_context(flags=pycuda.driver.ctx_flags.SCHED_BLOCKING_SYNC)
        import atexit
        atexit.register(clean_cuda,self.context)

class CPUScheme(Scheme):
    def __init__(self, num_threads=1):
        if isinstance(num_threads, int):
            self.num_threads=num_threads
        elif num_threads == 'env' and "PYCBC_NUM_THREADS" in os.environ:
            self.num_threads = int(os.environ["PYCBC_NUM_THREADS"])
        else:
            import multiprocessing
            self.num_threads = multiprocessing.cpu_count()

        Scheme.__enter__(self)
        os.environ["OMP_NUM_THREADS"] = str(self.num_threads)
        if _libgomp is not None:
            _libgomp.omp_set_num_threads( int(self.num_threads) )

        os.environ["OMP_NUM_THREADS"] = "1"
            _libgomp.omp_set_num_threads(1)
        Scheme.__exit__(self, type, value, traceback)

class MKLScheme(CPUScheme):
        CPUScheme.__init__(self, num_threads)
        if not pycbc.HAVE_MKL:
            raise RuntimeError("Can't find MKL libraries")

class NumpyScheme(CPUScheme):
    pass

scheme_prefix = {
    CUDAScheme: "cuda",
    CPUScheme: "cpu",
    MKLScheme: "mkl",
    NumpyScheme: "numpy",
}
_scheme_map = {v: k for (k, v) in scheme_prefix.items()}

_default_scheme_prefix = os.getenv("PYCBC_SCHEME", "cpu")
try:
    _default_scheme_class = _scheme_map[_default_scheme_prefix]
except KeyError as exc:
    raise RuntimeError(
        "PYCBC_SCHEME={!r} not recognised, please select one of: {}".format(
            _default_scheme_prefix,
            ", ".join(map(repr, _scheme_map)),
        ),
    )

class DefaultScheme(_default_scheme_class):
    pass

default_context = DefaultScheme()
mgr.state = default_context
scheme_prefix[DefaultScheme] = _default_scheme_prefix

def current_prefix():
    return scheme_prefix[type(mgr.state)]

_import_cache = {}
def schemed(prefix):
    @decorator
    def scheming_function(fn, *args, **kwds):
        try:
            return _import_cache[mgr.state][fn](*args, **kwds)
            exc_errors = []
            for sch in mgr.state.__class__.__mro__[0:-2]:
                try:
                    backend = __import__(prefix + scheme_prefix[sch], fromlist=[fn.__name__])
                    schemed_fn = getattr(backend, fn.__name__)
                except (ImportError, AttributeError) as e:
                    exc_errors += [e]
                    continue

                if mgr.state not in _import_cache:
                    _import_cache[mgr.state] = {}

                _import_cache[mgr.state][fn] = schemed_fn

                return schemed_fn(*args, **kwds)

            err = """Failed to find implementation of (%s)
                  for %s scheme." % (str(fn), current_prefix())"""
            for emsg in exc_errors:
                err += print(emsg)
            raise RuntimeError(err)

    return scheming_function

@decorator
def cpuonly(fn, *args, **kwds):
    if not issubclass(type(mgr.state), CPUScheme):
        raise TypeError(fn.__name__ +
                        " can only be called from a CPU processing scheme.")
    else:
        return fn(*args, **kwds)

def insert_processing_option_group(parser):
    """
    Adds the options used to choose a processing scheme. This should be used
    if your program supports the ability to select the processing scheme.

    """
    processing_group = parser.add_argument_group("Options for selecting the"
                                   " processing scheme in this program.")
    processing_group.add_argument("--processing-scheme",
                      help="The choice of processing scheme. "
                           "Choices are " + str(list(set(scheme_prefix.values()))) +
                           ". (optional for CPU scheme) The number of "
                           "execution threads "
                           "can be indicated by cpu:NUM_THREADS, "
                           "where NUM_THREADS "
                           "is an integer. The default is a single thread. "
                           "If the scheme is provided as cpu:env, the number "
                           "of threads can be provided by the PYCBC_NUM_THREADS "
                           "environment variable. If the environment variable "
                           "is not set, the number of threads matches the number "
                           "of logical cores. ",
                      default="cpu")

    processing_group.add_argument("--processing-device-id",
                      help="(optional) ID of GPU to use for accelerated "
                           "processing",
                      default=0, type=int)

def from_cli(opt):
    """Parses the command line options and returns a precessing scheme.

    opt: object
        Result of parsing the CLI with OptionParser, or any object with
        the required attributes.

    Returns
    -------
    ctx: Scheme
        Returns the requested processing scheme.
    """
    scheme_str = opt.processing_scheme.split(':')
    name = scheme_str[0]

    if name == "cuda":
        logging.info("Running with CUDA support")
        ctx = CUDAScheme(opt.processing_device_id)
    elif name == "mkl":
        if len(scheme_str) > 1:
            numt = scheme_str[1]
            if numt.isdigit():
                numt = int(numt)
            ctx = MKLScheme(num_threads=numt)
        else:
            ctx = MKLScheme()
        logging.info("Running with MKL support: %s threads" % ctx.num_threads)
    else:
            ctx = CPUScheme(num_threads=numt)
        else:
            ctx = CPUScheme()
        logging.info("Running with CPU support: %s threads" % ctx.num_threads)
    return ctx

def verify_processing_options(opt, parser):
    """Parses the  processing scheme options and verifies that they are
       reasonable.

        required attributes.
    """
    scheme_types = scheme_prefix.values()
    if opt.processing_scheme.split(':')[0] not in scheme_types:
        parser.error("(%s) is not a valid scheme type.")

class ChooseBySchemeDict(dict):
    """ This class represents a dictionary whose purpose is to chose objects
    based on their processing scheme. The keys are intended to be processing
    schemes.
    """
    def __getitem__(self, scheme):
        for base in scheme.__mro__[0:-1]:
            try:
                return dict.__getitem__(self, base)
                break
            except:
                pass

""" This module contains utilities for calculating search sensitivity
"""
from pycbc.conversions import chirp_distance

def compute_search_efficiency_in_bins(
         found, total, ndbins,
         sim_to_bins_function=lambda sim: (sim.distance,)):
    """
    Calculate search efficiency in the given ndbins.

    The first dimension of ndbins must be bins over injected distance.
    sim_to_bins_function must map an object to a tuple indexing the ndbins.
    """
    bins = bin_utils.BinnedRatios(ndbins)

    # increment the numerator and denominator with found / found+missed injs
    [bins.incnumerator(sim_to_bins_function(sim)) for sim in found]
    [bins.incdenominator(sim_to_bins_function(sim)) for sim in total]

    # regularize by setting denoms to 1 to avoid nans
    bins.regularize()

    # efficiency array is the ratio
    eff = bin_utils.BinnedArray(bin_utils.NDBins(ndbins), array=bins.ratio())

    # compute binomial uncertainties in each bin
    err_arr = numpy.sqrt(eff.array * (1-eff.array)/bins.denominator.array)
    err = bin_utils.BinnedArray(bin_utils.NDBins(ndbins), array=err_arr)

    return eff, err

def compute_search_volume_in_bins(found, total, ndbins, sim_to_bins_function):
    """
    Calculate search sensitive volume by integrating efficiency in distance bins

    No cosmological corrections are applied: flat space is assumed.
    sim_to_bins_function must maps an object to a tuple indexing the ndbins.
    """
    eff, err = compute_search_efficiency_in_bins(
                                    found, total, ndbins, sim_to_bins_function)
    dx = ndbins[0].upper() - ndbins[0].lower()
    r = ndbins[0].centres()

    # volume and errors have one fewer dimension than the input NDBins
    vol = bin_utils.BinnedArray(bin_utils.NDBins(ndbins[1:]))
    errors = bin_utils.BinnedArray(bin_utils.NDBins(ndbins[1:]))

    # integrate efficiency to obtain volume
    vol.array = numpy.trapz(eff.array.T * 4. * numpy.pi * r**2, r, dx)

    # propagate errors in eff to errors in V
    errors.array = numpy.sqrt(
        ((4 * numpy.pi * r**2 * err.array.T * dx)**2).sum(axis=-1)
    )

    return vol, errors

def volume_to_distance_with_errors(vol, vol_err):
    """ Return the distance and standard deviation upper and lower bounds

    vol: float
    vol_err: float

    Returns
    -------
    dist: float
    ehigh: float
    elow: float

    """
    dist = (vol * 3.0/4.0/numpy.pi) ** (1.0/3.0)
    ehigh = ((vol + vol_err) * 3.0/4.0/numpy.pi) ** (1.0/3.0) - dist
    delta = numpy.where(vol >= vol_err, vol - vol_err, 0)
    elow = dist - (delta * 3.0/4.0/numpy.pi) ** (1.0/3.0)
    return dist, ehigh, elow

def volume_montecarlo(found_d, missed_d, found_mchirp, missed_mchirp,
                      distribution_param, distribution, limits_param,
                      min_param=None, max_param=None):
    """
    Compute sensitive volume and standard error via direct Monte Carlo integral

    Injections should be made over a range of distances such that sensitive
    volume due to signals closer than D_min is negligible, and efficiency at
    distances above D_max is negligible
    TODO : Replace this function by Collin's formula given in Usman et al. ?
    OR get that coded as a new function?

    found_d: numpy.ndarray
        The distances of found injections
    missed_d: numpy.ndarray
        The distances of missed injections
    found_mchirp: numpy.ndarray
        Chirp mass of found injections
    missed_mchirp: numpy.ndarray
        Chirp mass of missed injections
    distribution_param: string
        Parameter D of the injections used to generate a distribution over
        distance, may be 'distance', 'chirp_distance'.
    distribution: string
        form of the distribution over the parameter, may be
        'log' (uniform in log D)
        'uniform' (uniform in D)
        'distancesquared' (uniform in D**2)
        'volume' (uniform in D**3)
    limits_param: string
        Parameter Dlim specifying limits inside which injections were made
        may be 'distance', 'chirp distance'
    min_param: float
        minimum value of Dlim at which injections were made; only used for
        log distribution, then if None the minimum actually injected value
        will be used
    max_param: float
        maximum value of Dlim out to which injections were made; if None
        the maximum actually injected value will be used

    Returns
    --------
    volume: float
        Volume estimate
    volume_error: float
        The standard error in the volume
    """
    d_power = {
        'log'             : 3.,
        'uniform'         : 2.,
        'distancesquared' : 1.,
        'volume'          : 0.
    }[distribution]
    mchirp_power = {
            'log'             : 0.,
            'uniform'         : 5. / 6.,
            'distancesquared' : 5. / 3.,
            'volume'          : 15. / 6.

    # establish maximum physical distance: first for chirp distance distribution
    if limits_param == 'chirp_distance':
        mchirp_standard_bns = 1.4 * 2.**(-1. / 5.)
        all_mchirp = numpy.concatenate((found_mchirp, missed_mchirp))
        max_mchirp = all_mchirp.max()
        if max_param is not None:
            # use largest injected mchirp to convert back to distance
            max_distance = max_param * \
                                  (max_mchirp / mchirp_standard_bns)**(5. / 6.)
        else:
            max_distance = max(found_d.max(), missed_d.max())
    elif limits_param == 'distance':
            max_distance = max_param
        else:
            # if no max distance given, use max distance actually injected
    else:
        raise NotImplementedError("%s is not a recognized parameter"
                                  % limits_param)

    # volume of sphere
    montecarlo_vtot = (4. / 3.) * numpy.pi * max_distance**3.

    # arrays of weights for the MC integral
    if distribution_param == 'distance':
        found_weights = found_d ** d_power
        missed_weights = missed_d ** d_power
    elif distribution_param == 'chirp_distance':
        # weight by a power of mchirp to rescale injection density to the
        # target mass distribution
        found_weights = found_d ** d_power * \
                        found_mchirp ** mchirp_power
        missed_weights = missed_d ** d_power * \
                         missed_mchirp ** mchirp_power
    else:
        raise NotImplementedError("%s is not a recognized distance parameter"
                                  % distribution_param)

    all_weights = numpy.concatenate((found_weights, missed_weights))

    # measured weighted efficiency is w_i for a found inj and 0 for missed
    # MC integral is volume of sphere * (sum of found weights)/(sum of all weights)
    # over injections covering the sphere
    mc_weight_samples = numpy.concatenate((found_weights, 0 * missed_weights))
    mc_sum = sum(mc_weight_samples)

    if limits_param == 'distance':
        mc_norm = sum(all_weights)
    elif limits_param == 'chirp_distance':
        # if injections are made up to a maximum chirp distance, account for
        # extra missed injections that would occur when injecting up to
        # maximum physical distance : this works out to a 'chirp volume' factor
        mc_norm = sum(all_weights * (max_mchirp / all_mchirp) ** (5. / 2.))

    # take out a constant factor
    mc_prefactor = montecarlo_vtot / mc_norm

    # count the samples
        Ninj = len(mc_weight_samples)
        # find the total expected number after extending from maximum chirp
        # dist up to maximum physical distance
        if distribution == 'log':
            # only need minimum distance in this one case
            if min_param is not None:
                min_distance = min_param * \
                     (numpy.min(all_mchirp) / mchirp_standard_bns) ** (5. / 6.)
            else:
                min_distance = min(numpy.min(found_d), numpy.min(missed_d))
            logrange = numpy.log(max_distance / min_distance)
            Ninj = len(mc_weight_samples) + (5. / 6.) * \
                             sum(numpy.log(max_mchirp / all_mchirp) / logrange)
        else:
            Ninj = sum((max_mchirp / all_mchirp) ** mchirp_power)

    # sample variance of efficiency: mean of the square - square of the mean
    mc_sample_variance = sum(mc_weight_samples ** 2.) / Ninj - \
                                                          (mc_sum / Ninj) ** 2.

    # return MC integral and its standard deviation; variance of mc_sum scales
    # relative to sample variance by Ninj (Bienayme' rule)
    vol = mc_prefactor * mc_sum
    vol_err = mc_prefactor * (Ninj * mc_sample_variance) ** 0.5
    return vol, vol_err

def chirp_volume_montecarlo(
        found_d, missed_d, found_mchirp, missed_mchirp,
        distribution_param, distribution, limits_param, min_param, max_param):

    assert distribution_param == 'chirp_distance'
    assert limits_param == 'chirp_distance'

    found_dchirp = chirp_distance(found_d, found_mchirp)
    missed_dchirp = chirp_distance(missed_d, missed_mchirp)
    # treat chirp distances in MC volume estimate as physical distances
    return volume_montecarlo(found_dchirp, missed_dchirp, found_mchirp,
                             missed_mchirp, 'distance', distribution,
                             'distance', min_param, max_param)

def volume_binned_pylal(f_dist, m_dist, bins=15):
    """ Compute the sensitive volume using a distance binned efficiency estimate

    f_dist: numpy.ndarray
    m_dist: numpy.ndarray

    Returns
    --------
    """
    def sims_to_bin(sim):
        return (sim, 0)

    total = numpy.concatenate([f_dist, m_dist])
    ndbins = bin_utils.NDBins([bin_utils.LinearBins(min(total), max(total), bins),
                               bin_utils.LinearBins(0., 1, 1)])
    vol, verr = compute_search_volume_in_bins(f_dist, total, ndbins, sims_to_bin)
    return vol.array[0], verr.array[0]

def volume_shell(f_dist, m_dist):
    """ Compute the sensitive volume using sum over spherical shells.

    Returns
    --------
    """
    f_dist.sort()
    m_dist.sort()
    distances = numpy.concatenate([f_dist, m_dist])
    dist_sorting = distances.argsort()
    distances = distances[dist_sorting]
    low = 0
    vol = 0
    vol_err = 0
    for i in range(len(distances)):
        if i == len(distances) - 1:
            break

        high = (distances[i+1] + distances[i]) / 2
        bin_width = high - low

        if dist_sorting[i] < len(f_dist):
            vol += 4 * numpy.pi * distances[i]**2.0 * bin_width
            vol_err += (4 * numpy.pi * distances[i]**2.0 * bin_width)**2.0

        low = high
    vol_err = vol_err ** 0.5

# Copyright (C) 2017  Christopher M. Biwer
#
#
"""
This modules provides classes and functions for transforming parameters.
"""

import os
from six import string_types
from pycbc import coordinates
from pycbc import cosmology
from pycbc.io import record
from pycbc.waveform import parameters
from pycbc.boundaries import Bounds
from pycbc import VARARGS_DELIM

class BaseTransform(object):
    """A base class for transforming between two sets of parameters.
    """
    name = None
    inverse = None
    _inputs = []
    _outputs = []

        self.inputs = set(self._inputs)
        self.outputs = set(self._outputs)

    def __call__(self, maps):
        return self.transform(maps)

    def transform(self, maps):
        """ This function transforms from inputs to outputs.
        """
        raise NotImplementedError("Not added.")

    def inverse_transform(self, maps):
        """ The inverse conversions of transform. This function transforms from
        outputs to inputs.
        """

    def jacobian(self, maps):
        """ The Jacobian for the inputs to outputs transformation.
        """
        raise NotImplementedError("Jacobian transform not implemented.")

    def inverse_jacobian(self, maps):
        """ The Jacobian for the outputs to inputs transformation.
        """

    @staticmethod
    def format_output(old_maps, new_maps):
        """ This function takes the returned dict from `transform` and converts
        it to the same datatype as the input.

        old_maps : {FieldArray, dict}
            The mapping object to add new maps to.
        new_maps : dict
            A dict with key as parameter name and value is numpy.array.

        Returns
        -------
        {FieldArray, dict}
            The old_maps object with new keys from new_maps.
        """

        # if input is FieldArray then return FieldArray
        if isinstance(old_maps, record.FieldArray):
            keys = new_maps.keys()
            values = [new_maps[key] for key in keys]
            for key, vals in zip(keys, values):
                try:
                    old_maps = old_maps.add_fields([vals], [key])
                except ValueError:
                    old_maps[key] = vals
            return old_maps

        # if input is dict then return dict
        elif isinstance(old_maps, dict):
            out = old_maps.copy()
            out.update(new_maps)
            return out

        # else error
        else:
            raise TypeError("Input type must be FieldArray or dict.")

    @classmethod
    def from_config(cls, cp, section, outputs, skip_opts=None,
                    additional_opts=None):
        """Initializes a transform from the given section.

        cp : pycbc.workflow.WorkflowConfigParser
            A parsed configuration file that contains the transform options.
        section : str
            Name of the section in the configuration file.
        outputs : str
            The names of the parameters that are output by this transformation,
            separated by `VARARGS_DELIM`. These must appear in the "tag" part
            of the section header.
        skip_opts : list, optional
            Do not read options in the given list.
        additional_opts : dict, optional
            Any additional arguments to pass to the class. If an option is
            provided that also exists in the config file, the value provided
            will be used instead of being read from the file.

        Returns
        -------
        cls
            An instance of the class.
        """
        tag = outputs
        if skip_opts is None:
            skip_opts = []
        if additional_opts is None:
            additional_opts = {}
        else:
            additional_opts = additional_opts.copy()
        outputs = set(outputs.split(VARARGS_DELIM))
        special_args = ['name'] + skip_opts + list(additional_opts.keys())
        # get any extra arguments to pass to init
        extra_args = {}
        for opt in cp.options("-".join([section, tag])):
            if opt in special_args:
                continue
            # check if option can be cast as a float
            val = cp.get_opt_tag(section, opt, tag)
            try:
                val = float(val)
            except ValueError:
                pass
            # add option
            extra_args.update({opt:val})
        extra_args.update(additional_opts)
        out = cls(**extra_args)
        # check that the outputs matches
        if outputs-out.outputs != set() or out.outputs-outputs != set():
            raise ValueError("outputs of class do not match outputs specified "
                             "in section")
        return out

class CustomTransform(BaseTransform):
    """Allows for any transform to be defined.

    input_args : (list of) str
        The names of the input parameters.
    output_args : (list of) str
        The names of the output parameters.
    transform_functions : dict
        Dictionary mapping input args to a string giving a function call;
        e.g., ``{'q': 'q_from_mass1_mass2(mass1, mass2)'}``.
    jacobian : str, optional
        String giving a jacobian function. The function must be in terms of
        the input arguments.

    Examples
    --------
    Create a custom transform that converts mass1, mass2 to mtotal, q:

    >>> t = transforms.CustomTransform(['mass1', 'mass2'], ['mtotal', 'q'], {'mtotal': 'mass1+mass2', 'q': 'mass1/mass2'}, '(mass1 + mass2) / mass2**2')

    Evaluate a pair of masses:

    >>> t.transform({'mass1': 10., 'mass2': 5.})
    {'mass1': 10.0, 'mass2': 5.0, 'mtotal': 15.0, 'q': 2.0}

    The Jacobian for the same pair of masses:

    >>> t.jacobian({'mass1': 10., 'mass2': 5.})
    0.59999999999999998

    """
    name = "custom"

    def __init__(self, input_args, output_args, transform_functions,
                 jacobian=None):
        if isinstance(input_args, string_types):
            input_args = [input_args]
        if isinstance(output_args, string_types):
            output_args = [output_args]
        self.inputs = set(input_args)
        self.outputs = set(output_args)
        self.transform_functions = transform_functions
        self._jacobian = jacobian
        # we'll create a scratch FieldArray space to do transforms on
        # we'll default to length 1; this will be changed if a map is passed
        # with more than one value in it
        self._createscratch()

    def _createscratch(self, shape=1):
        """Creates a scratch FieldArray to use for transforms."""
        self._scratch = record.FieldArray(shape, dtype=[(p, float)
            for p in self.inputs])

    def _copytoscratch(self, maps):
        """Copies the data in maps to the scratch space.

        If the maps contain arrays that are not the same shape as the scratch
        space, a new scratch space will be created.
        """
        try:
            for p in self.inputs:
                self._scratch[p][:] = maps[p]
            # we'll get a ValueError if the scratch space isn't the same size
            # as the maps; in that case, re-create the scratch space with the
            # appropriate size and try again
            invals = maps[list(self.inputs)[0]]
            if isinstance(invals, numpy.ndarray):
                shape = invals.shape
            else:
                shape = len(invals)
            self._createscratch(shape)

    def _getslice(self, maps):
        """Determines how to slice the scratch for returning values."""
        invals =  maps[list(self.inputs)[0]]
        if not isinstance(invals, (numpy.ndarray, list)):
            getslice = 0
        else:
            getslice = slice(None, None)
        return getslice

        """Applies the transform functions to the given maps object.

        maps : dict, or FieldArray

        Returns
        -------
        dict or FieldArray
            A map object containing the transformed variables, along with the
            original variables. The type of the output will be the same as the
            input.
        """
        if self.transform_functions is None:
            raise NotImplementedError("no transform function(s) provided")
        # copy values to scratch
        self._copytoscratch(maps)
        # ensure that we return the same data type in each dict
        getslice = self._getslice(maps)
        # evaluate the functions
        out = {p: self._scratch[func][getslice]
               for p,func in self.transform_functions.items()}
        return self.format_output(maps, out)

        if self._jacobian is None:
            raise NotImplementedError("no jacobian provided")
        out = self._scratch[self._jacobian]
        if isinstance(out, numpy.ndarray):
            out = out[self._getslice(maps)]
        return out

    def from_config(cls, cp, section, outputs):
        """Loads a CustomTransform from the given config file.

        Example section:

        .. code-block:: ini

            [{section}-outvar1+outvar2]
            name = custom
            inputs = inputvar1, inputvar2
            outvar1 = func1(inputs)
            outvar2 = func2(inputs)
            jacobian = func(inputs)
        """
        inputs = map(str.strip,
                     cp.get_opt_tag(section, 'inputs', tag).split(','))
        # get the functions for each output
        transform_functions = {}
        for var in outputs:
            func = cp.get_opt_tag(section, var, tag)
            transform_functions[var] = func
        s = '-'.join([section, tag])
        if cp.has_option(s, 'jacobian'):
            jacobian = cp.get_opt_tag(section, 'jacobian', tag)
        else:
            jacobian = None
        return cls(inputs, outputs, transform_functions, jacobian=jacobian)

#
#
#                             Forward Transforms
#
#

class MchirpQToMass1Mass2(BaseTransform):
    """ Converts chirp mass and mass ratio to component masses.
    """
    name = "mchirp_q_to_mass1_mass2"

    def __init__(self, mass1_param=None, mass2_param=None, mchirp_param=None,
                 q_param=None):
        if mass1_param is None:
            mass1_param = parameters.mass1
        if mass2_param is None:
            mass2_param = parameters.mass2
        if mchirp_param is None:
            mchirp_param = parameters.mchirp
        if q_param is None:
            q_param = parameters.q
        self.mass1_param = mass1_param
        self.mass2_param = mass2_param
        self.mchirp_param = mchirp_param
        self.q_param = q_param
        self._inputs = [self.mchirp_param, self.q_param]
        self._outputs = [self.mass1_param, self.mass2_param]
        super(MchirpQToMass1Mass2, self).__init__()

        """This function transforms from chirp mass and mass ratio to component
        masses.

        maps : a mapping object

        Examples
        --------
        Convert a dict of numpy.array:

        >>> import numpy
        >>> from pycbc import transforms
        >>> t = transforms.MchirpQToMass1Mass2()
        >>> t.transform({'mchirp': numpy.array([10.]), 'q': numpy.array([2.])})
        {'mass1': array([ 16.4375183]), 'mass2': array([ 8.21875915]),
         'mchirp': array([ 10.]), 'q': array([ 2.])}

        Returns
        -------
        out : dict
            A dict with key as parameter name and value as numpy.array or float
            of transformed values.
        """
        out = {}
        out[self.mass1_param] = conversions.mass1_from_mchirp_q(
                                                maps[self.mchirp_param],
                                                maps[self.q_param])
        out[self.mass2_param] = conversions.mass2_from_mchirp_q(

        """This function transforms from component masses to chirp mass and
        mass ratio.

        Examples
        --------

        >>> t.inverse_transform({'mass1': numpy.array([16.4]), 'mass2': numpy.array([8.2])})
            {'mass1': array([ 16.4]), 'mass2': array([ 8.2]),
             'mchirp': array([ 9.97717521]), 'q': 2.0}

        Returns
        -------
        out : dict
        """
        out = {}
        m1 = maps[self.mass1_param]
        m2 = maps[self.mass2_param]
        out[self.mchirp_param] = conversions.mchirp_from_mass1_mass2(m1, m2)
        out[self.q_param] = m1 / m2

        """Returns the Jacobian for transforming mchirp and q to mass1 and
        mass2.
        """
        mchirp = maps[self.mchirp_param]
        q = maps[self.q_param]
        return mchirp * ((1.+q)/q**3.)**(2./5)

        """Returns the Jacobian for transforming mass1 and mass2 to
        mchirp and q.
        """
        return conversions.mchirp_from_mass1_mass2(m1, m2)/m2**2.

class MchirpEtaToMass1Mass2(BaseTransform):
    """ Converts chirp mass and symmetric mass ratio to component masses.
    """
    name = "mchirp_eta_to_mass1_mass2"
    _inputs = [parameters.mchirp, parameters.eta]
    _outputs = [parameters.mass1, parameters.mass2]

        """This function transforms from chirp mass and symmetric mass ratio to
        component masses.

        Examples
        --------

        >>> t = transforms.MchirpEtaToMass1Mass2()
        >>> t.transform({'mchirp': numpy.array([10.]), 'eta': numpy.array([0.25])})
         'mchirp': array([ 10.]), 'eta': array([ 0.25])}

        Returns
        -------
        out : dict
        """
        out = {}
        out[parameters.mass1] = conversions.mass1_from_mchirp_eta(
                                                maps[parameters.mchirp],
                                                maps[parameters.eta])
        out[parameters.mass2] = conversions.mass2_from_mchirp_eta(

        symmetric mass ratio.

        Examples
        --------

        >>> t.inverse_transform({'mass1': numpy.array([8.2]), 'mass2': numpy.array([8.2])})
            {'mass1': array([ 8.2]), 'mass2': array([ 8.2]),
             'mchirp': array([ 9.97717521]), 'eta': 0.25}

        Returns
        -------
        out : dict
        """
        out = {}
        m1 = maps[parameters.mass1]
        m2 = maps[parameters.mass2]
        out[parameters.mchirp] = conversions.mchirp_from_mass1_mass2(m1, m2)
        out[parameters.eta] = conversions.eta_from_mass1_mass2(m1, m2)

        """Returns the Jacobian for transforming mchirp and eta to mass1 and
        mass2.
        """
        mchirp = maps[parameters.mchirp]
        eta = maps[parameters.eta]
        m1 = conversions.mass1_from_mchirp_eta(mchirp, eta)
        m2 = conversions.mass2_from_mchirp_eta(mchirp, eta)
        return mchirp * (m1 - m2) / (m1 + m2)**3

        mchirp and eta.
        """
        mchirp = conversions.mchirp_from_mass1_mass2(m1, m2)
        eta = conversions.eta_from_mass1_mass2(m1, m2)
        return -1. * mchirp / eta**(6./5)

class ChirpDistanceToDistance(BaseTransform):
    """ Converts chirp distance to luminosity distance, given the chirp mass.
    """
    name = "chirp_distance_to_distance"
    _inputs = [parameters.chirp_distance, parameters.mchirp]
    _outputs = [parameters.distance]

    def __init__(self, ref_mass=1.4):
        self.ref_mass = ref_mass

        """This function transforms from chirp distance to luminosity distance,
        given the chirp mass.

        Examples
        --------

        >>> import numpy as np
        >>> t = transforms.ChirpDistanceToDistance()
        >>> t.transform({'chirp_distance': np.array([40.]), 'mchirp': np.array([1.2])})
        {'mchirp': array([ 1.2]), 'chirp_distance': array([ 40.]), 'distance': array([ 39.48595679])}

        Returns
        -------
        out : dict
        """
        out = {}
        out[parameters.distance] = \
                conversions.distance_from_chirp_distance_mchirp(
                                                    maps[parameters.chirp_distance],
                                                    maps[parameters.mchirp],
                                                    ref_mass=self.ref_mass)

        """This function transforms from luminosity distance to chirp distance,

        Examples
        --------

        >>> t.inverse_transform({'distance': np.array([40.]), 'mchirp': np.array([1.2])})
        {'distance': array([ 40.]), 'chirp_distance': array([ 40.52073522]), 'mchirp': array([ 1.2])}

        Returns
        -------
        out : dict
        """
        out = {}
        out[parameters.chirp_distance] = \
                conversions.chirp_distance(maps[parameters.distance],
                                            maps[parameters.mchirp], ref_mass=self.ref_mass)

        """Returns the Jacobian for transforming chirp distance to
        luminosity distance, given the chirp mass.
        """
        ref_mass=1.4
        mchirp = maps['mchirp']
        return (2.**(-1./5) * self.ref_mass / mchirp)**(-5./6)

        """Returns the Jacobian for transforming luminosity distance to
        chirp distance, given the chirp mass.
        """
        return (2.**(-1./5) * self.ref_mass / mchirp)**(5./6)

class SphericalToCartesian(BaseTransform):
    """Converts spherical coordinates to cartesian.

    x : str
        The name of the x parameter.
    y : str
        The name of the y parameter.
    z : str
        The name of the z parameter.
    radial : str
        The name of the radial parameter.
    azimuthal : str
        The name of the azimuthal angle parameter.
    polar : str
        The name of the polar angle parameter.
    """
    name = "spherical_to_cartesian"

    def __init__(self, x, y, z, radial, azimuthal, polar):
        self.x = x
        self.y = y
        self.z = z
        self.radial = radial
        self.polar = polar
        self.azimuthal = azimuthal
        self._inputs = [self.radial, self.azimuthal, self.polar]
        self._outputs = [self.x, self.y, self.z]
        super(SphericalToCartesian, self).__init__()

        """This function transforms from spherical to cartesian spins.

        Examples
        --------

        >>> t = transforms.SphericalToCartesian('x', 'y', 'z',
                                                'a', 'phi', 'theta')
        >>> t.transform({'a': numpy.array([0.1]), 'phi': numpy.array([0.1]),
                        'theta': numpy.array([0.1])})
            {'a': array([ 0.1]), 'phi': array([ 0.1]), 'theta': array([ 0.1]),
             'x': array([ 0.00993347]), 'y': array([ 0.00099667]),
             'z': array([ 0.09950042])}

        Returns
        -------
        out : dict
        """
        a = self.radial
        az = self.azimuthal
        po = self.polar
        x, y, z = coordinates.spherical_to_cartesian(maps[a], maps[az],
                                                     maps[po])
        out = {self.x: x, self.y: y, self.z: z}

        """This function transforms from cartesian to spherical spins.

        Returns
        -------
        out : dict
        """
        x = self.x
        y = self.y
        z = self.z
        a, az, po = coordinates.cartesian_to_spherical(maps[x], maps[y],
                                                       maps[z])
        out = {self.radial: a, self.azimuthal: az, self.polar: po}

class SphericalSpin1ToCartesianSpin1(SphericalToCartesian):
    """Converts spherical spin parameters (radial and two angles) to
    catesian spin parameters. This class only transforms spsins for the first
    component mass.

    **Deprecation Warning:** This will be removed in a future update. Use
    :py:class:`SphericalToCartesian` with spin-parameter names passed in
    instead.
    """
    name = "spherical_spin_1_to_cartesian_spin_1"

        logging.warning("Deprecation warning: the {} transform will be "
                        "removed in a future update. Please use {} instead, "
                        "passing spin1x, spin1y, spin1z, spin1_a, "
                        "spin1_azimuthal, spin1_polar as arguments."
                        .format(self.name, SphericalToCartesian.name))
        super(SphericalSpin1ToCartesianSpin1, self).__init__(
            "spin1x", "spin1y", "spin1z", "spin1_a", "spin1_azimuthal",
            "spin1_polar")

class SphericalSpin2ToCartesianSpin2(SphericalToCartesian):

    instead.
    """
    name = "spherical_spin_2_to_cartesian_spin_2"

                        "passing spin2x, spin2y, spin2z, spin2_a, "
                        "spin2_azimuthal, spin2_polar as arguments."
        super(SphericalSpin2ToCartesianSpin2, self).__init__(
            "spin2x", "spin2y", "spin2z", "spin2_a", "spin2_azimuthal",
            "spin2_polar")

class DistanceToRedshift(BaseTransform):
    """ Converts distance to redshift.
    """
    name = "distance_to_redshift"
    _inputs = [parameters.distance]
    _outputs = [parameters.redshift]

        """ This function transforms from distance to redshift.

        Examples
        --------

        >>> t = transforms.DistanceToRedshift()
        >>> t.transform({'distance': numpy.array([1000])})
            {'distance': array([1000]), 'redshift': 0.19650987609144363}

        Returns
        -------
        out : dict
        """
        out = {parameters.redshift : cosmology.redshift(
                                                    maps[parameters.distance])}

class AlignedMassSpinToCartesianSpin(BaseTransform):
    """ Converts mass-weighted spins to cartesian z-axis spins.
    """
    name = "aligned_mass_spin_to_cartesian_spin"
    _inputs = [parameters.mass1, parameters.mass2, parameters.chi_eff, "chi_a"]
    _outputs = [parameters.mass1, parameters.mass2,
               parameters.spin1z, parameters.spin2z]

        """ This function transforms from aligned mass-weighted spins to
        cartesian spins aligned along the z-axis.

        Returns
        -------
        out : dict
        """
        mass1 = maps[parameters.mass1]
        mass2 = maps[parameters.mass2]
        out = {}
        out[parameters.spin1z] = \
                         conversions.spin1z_from_mass1_mass2_chi_eff_chi_a(
                               mass1, mass2,
                               maps[parameters.chi_eff], maps["chi_a"])
        out[parameters.spin2z] = \
                         conversions.spin2z_from_mass1_mass2_chi_eff_chi_a(

        """ This function transforms from component masses and cartesian spins
        to mass-weighted spin parameters aligned with the angular momentum.

        Returns
        -------
        out : dict
        """
        spin1z = maps[parameters.spin1z]
        spin2z = maps[parameters.spin2z]
        out = {
            parameters.chi_eff : conversions.chi_eff(mass1, mass2,
                                                     spin1z, spin2z),
            "chi_a" : conversions.chi_a(mass1, mass2, spin1z, spin2z),
        }

class PrecessionMassSpinToCartesianSpin(BaseTransform):
    """ Converts mass-weighted spins to cartesian x-y plane spins.
    """
    name = "precession_mass_spin_to_cartesian_spin"
    _inputs = [parameters.mass1, parameters.mass2,
               "xi1", "xi2", "phi_a", "phi_s"]
                parameters.spin1x, parameters.spin1y,
                parameters.spin2x, parameters.spin2y]

        """ This function transforms from mass-weighted spins to caretsian spins
        in the x-y plane.

        Returns
        -------
        out : dict
        """

        # find primary and secondary masses
        # since functions in conversions.py map to primary/secondary masses
        m_p = conversions.primary_mass(maps["mass1"], maps["mass2"])
        m_s = conversions.secondary_mass(maps["mass1"], maps["mass2"])

        # find primary and secondary xi
        # can re-purpose spin functions for just a generic variable
        xi_p = conversions.primary_spin(maps["mass1"], maps["mass2"],
                                        maps["xi1"], maps["xi2"])
        xi_s = conversions.secondary_spin(maps["mass1"], maps["mass2"],
                                          maps["xi1"], maps["xi2"])

        # convert using convention of conversions.py that is mass1 > mass2
        spinx_p = conversions.spin1x_from_xi1_phi_a_phi_s(
                           xi_p, maps["phi_a"], maps["phi_s"])
        spiny_p = conversions.spin1y_from_xi1_phi_a_phi_s(
        spinx_s = conversions.spin2x_from_mass1_mass2_xi2_phi_a_phi_s(
                           m_p, m_s, xi_s, maps["phi_a"], maps["phi_s"])
        spiny_s = conversions.spin2y_from_mass1_mass2_xi2_phi_a_phi_s(

        # map parameters from primary/secondary to indices
        out = {}
        if isinstance(m_p, numpy.ndarray):
            mass1, mass2 = map(numpy.array, [maps["mass1"], maps["mass2"]])
            mask_mass1_gte_mass2 = mass1 >= mass2
            mask_mass1_lt_mass2 = mass1 < mass2
            out[parameters.spin1x] = numpy.concatenate((
                                        spinx_p[mask_mass1_gte_mass2],
                                        spinx_s[mask_mass1_lt_mass2]))
            out[parameters.spin1y] = numpy.concatenate((
                                        spiny_p[mask_mass1_gte_mass2],
                                        spiny_s[mask_mass1_lt_mass2]))
            out[parameters.spin2x] = numpy.concatenate((
                                        spinx_p[mask_mass1_lt_mass2],
                                        spinx_s[mask_mass1_gte_mass2]))
            out[parameters.spin2y] = numpy.concatenate((
        elif maps["mass1"] > maps["mass2"]:
            out[parameters.spin1x] = spinx_p
            out[parameters.spin1y] = spiny_p
            out[parameters.spin2x] = spinx_s
            out[parameters.spin2y] = spiny_s
        else:
            out[parameters.spin1x] = spinx_s
            out[parameters.spin1y] = spiny_s
            out[parameters.spin2x] = spinx_p
            out[parameters.spin2y] = spiny_p

        """ This function transforms from component masses and cartesian spins to
        mass-weighted spin parameters perpendicular with the angular momentum.

        Returns
        -------
        out : dict
        """

        # convert
        out = {}
        xi1 = conversions.primary_xi(
                             maps[parameters.mass1], maps[parameters.mass2],
                             maps[parameters.spin1x], maps[parameters.spin1y],
                             maps[parameters.spin2x], maps[parameters.spin2y])
        xi2 = conversions.secondary_xi(
        out["phi_a"] = conversions.phi_a(
        out["phi_s"] = conversions.phi_s(

        if isinstance(xi1, numpy.ndarray):
            mass1, mass2 = map(numpy.array, [maps[parameters.mass1],
                                             maps[parameters.mass2]])
            out["xi1"] = numpy.concatenate((
                                        xi1[mask_mass1_gte_mass2],
                                        xi2[mask_mass1_lt_mass2]))
            out["xi2"] = numpy.concatenate((
            out["xi1"] = xi1
            out["xi2"] = xi2
        else:
            out["xi1"] = xi2
            out["xi2"] = xi1

class CartesianSpinToChiP(BaseTransform):
    """Converts cartesian spins to `chi_p`.
    """
    name = "cartesian_spin_to_chi_p"
    _outputs = ["chi_p"]

        """ This function transforms from component masses and caretsian spins
        to chi_p.

        Examples
        --------

        Returns
        -------
        out : dict
        """
        out = {}
        out["chi_p"] = conversions.chi_p(

class LambdaFromTOVFile(BaseTransform):
    """Transforms mass values corresponding to Lambda values for a given EOS
    interpolating from the mass-Lambda data for that EOS read in from an
    external ASCII file.

    The interpolation of the mass-Lambda data is a one-dimensional piecewise
    linear interpolation. If the ``redshift_mass`` keyword argument is ``True``
    (the default), the mass values to be transformed are assumed to be detector
    frame masses. In that case, a distance should be provided along with the
    mass for transformation to the source frame mass before the Lambda values
    are extracted from the interpolation. If the transform is read in from a
    config file, an example code block would be:

    .. code-block:: ini

        [{section}-lambda1]
        name = lambda_from_tov_file
        mass_param = mass1
        lambda_param = lambda1
        distance = 40
        mass_lambda_file = {filepath}

    If this transform is used in a parameter estimation analysis where
    distance is a variable parameter, the distance to be used will vary
    with each draw. In that case, the example code block will be:

        mass_lambda_file = filepath

    If your prior is in terms of the source-frame masses (``srcmass``), then
    you can shut off the redshifting by adding ``do-not-redshift-mass`` to the
    config file. In this case you do not need to worry about a distance.
    Example:

        mass_param = srcmass1
        do-not-redshift-mass =

    mass_param : str
        The name of the mass parameter to transform.
    lambda_param : str
        The name of the tidal deformability parameter that mass_param is to
        be converted to interpolating from the data in the mass-Lambda file.
    mass_lambda_file : str
        Path of the mass-Lambda data file. The first column in the data file
        should contain mass values, and the second column Lambda values.
    distance : float, optional
        The distance (in Mpc) of the source. Used to redshift the mass. Needed
        if ``redshift_mass`` is True and no distance parameter exists If
        None, then a distance must be provided to the transform.
    redshift_mass : bool, optional
        Redshift the mass parameters when computing the lambdas. Default is
        False.
    file_columns : list of str, optional
        The names and order of columns in the ``mass_lambda_file``. Must
        contain at least 'mass' and 'lambda'. If not provided, will assume the
        order is ('mass', 'lambda').
    """
    name = 'lambda_from_tov_file'

    def __init__(self, mass_param, lambda_param, mass_lambda_file,
                 distance=None, redshift_mass=True, file_columns=None):
        self._mass_lambda_file = mass_lambda_file
        self._mass_param = mass_param
        self._lambda_param = lambda_param
        self.redshift_mass = redshift_mass
        self._distance = distance
        self._inputs = [mass_param, 'distance']
        self._outputs = [lambda_param]
        if file_columns is None:
            file_columns = ['mass', 'lambda']
        dtype = [(fname, float) for fname in file_columns]
        data = numpy.loadtxt(self._mass_lambda_file, dtype=dtype)
        self._data = data
        super(LambdaFromTOVFile, self).__init__()

    @property
    def mass_param(self):
        """Returns the input mass parameter."""
        return self._mass_param

    @property
    def lambda_param(self):
        """Returns the output lambda parameter."""
        return self._lambda_param

    @property
    def data(self):
        return self._data

    @property
    def mass_data(self):
        """Returns the mass data read from the mass-Lambda data file for
        an EOS.
        """
        return self._data['mass']

    @property
    def lambda_data(self):
        """Returns the Lambda data read from the mass-Lambda data file for
        an EOS.
        """
        return self._data['lambda']

    @property
    def distance(self):
        """Returns the fixed distance to transform mass samples from detector
        to source frame if one is specified.
        """
        return self._distance

    def lambda_from_tov_data(m_src, mass_data, lambda_data):
        """Returns Lambda corresponding to a given mass interpolating from the
        TOV data.

        m : float
            Value of the mass.
        mass_data : array
            Mass array from the Lambda-M curve of an EOS.
        lambda_data : array
            Lambda array from the Lambda-M curve of an EOS.

        Returns
        -------
        lambdav : float
            The Lambda corresponding to the mass `m` for the EOS considered.
        """
        if m_src > mass_data.max():
            # assume black hole
            lambdav = 0.
        else:
            lambdav = numpy.interp(m_src, mass_data, lambda_data)
        return lambdav

        """Computes the transformation of mass to Lambda.

        maps : dict or FieldArray
            A dictionary or FieldArray which provides a map between the
            parameter name of the variable to transform and its value(s).

        Returns
        -------
        out : dict or FieldArray
            A map between the transformed variable name and value(s), along
            with the original variable name and value(s).
        """
        m = maps[self._mass_param]
        if self.redshift_mass:
            if self._distance is not None:
                d = self._distance
            else:
                try:
                    d = maps['distance']
                except KeyError as e:
                    logging.warning("Either provide distance samples in the "
                                    "list of samples to be transformed, or "
                                    "provide a fixed distance value as input "
                                    "when initializing LambdaFromTOVFile.")
                    raise e
            shift = 1./(1.0 + cosmology.redshift(abs(d)))
        else:
            shift = 1.
        out = {self._lambda_param : self.lambda_from_tov_data(
            m*shift, self._data['mass'], self._data['lambda'])}

        # see if we're redshifting masses
        if cp.has_option('-'.join([section, outputs]), 'do-not-redshift-mass'):
            additional_opts = {'redshift_mass': False}
            skip_opts = ['do-not-redshift-mass']
        else:
            additional_opts = None
            skip_opts = None
        return super(LambdaFromTOVFile, cls).from_config(
            cp, section, outputs, skip_opts=skip_opts,
            additional_opts=additional_opts)

class LambdaFromMultipleTOVFiles(BaseTransform):
    """Uses multiple equation of states.

        The distance (in Mpc) of the source. Used to redshift the mass. If
        order is ('radius', 'mass', 'lambda').
    """

    name = 'lambda_from_multiple_tov_files'

    def __init__(self, mass_param, lambda_param, map_file, distance=None,
                 redshift_mass=True, file_columns=None):
        self._map_file = map_file
        self._inputs = [mass_param, 'eos', 'distance']
        # create a dictionary of the EOS files from the map_file
        self._eos_files = {}
        with open(self._map_file, 'r') as fp:
            for line in fp:
                fname = line.rstrip('\n')
                eosidx = int(os.path.basename(fname).split('.')[0])
                self._eos_files[eosidx] = os.path.abspath(fname)
        # create an eos cache for fast load later
        self._eos_cache = {}
            file_columns = ('radius', 'mass', 'lambda')
        self._file_columns = file_columns
        super(LambdaFromMultipleTOVFiles, self).__init__()

    @property

    @property

    @property
    def map_file(self):
        an EOS.
        """
        return self._map_file

    @property
        """

    def get_eos(self, eos_index):
        """Gets the EOS for the given index.

        If the index is not in range returns None.
        """
        try:
            eos = self._eos_cache[eos_index]
            try:
                fname = self._eos_files[eos_index]
                eos = LambdaFromTOVFile(mass_param=self._mass_param,
                                        lambda_param=self._lambda_param,
                                        mass_lambda_file=fname,
                                        distance=self._distance,
                                        redshift_mass=self.redshift_mass,
                                        file_columns=self._file_columns)
                self._eos_cache[eos_index] = eos
            except KeyError:
                eos = None
        return eos

    def transform(self,maps):
        """Transforms mass value and eos index into a lambda value """
        # floor
        eos_index = int(maps['eos'])
        eos = self.get_eos(eos_index)
        if eos is not None:
            return eos.transform(maps)
        else:
            # no eos, just return nan
            out = {self._lambda_param : numpy.nan}
            return self.format_output(maps, out)

        else:
        return super(LambdaFromMultipleTOVFiles, cls).from_config(

class Log(BaseTransform):
    """Applies a log transform from an `inputvar` parameter to an `outputvar`
    parameter. This is the inverse of the exponent transform.

    inputvar : str
        The name of the parameter to transform.
    outputvar : str
        The name of the transformed parameter.
    """
    name = 'log'

    def __init__(self, inputvar, outputvar):
        self._inputvar = inputvar
        self._outputvar = outputvar
        self._inputs = [inputvar]
        self._outputs = [outputvar]
        super(Log, self).__init__()

    @property
    def inputvar(self):
        """Returns the input parameter."""
        return self._inputvar

    @property
    def outputvar(self):
        """Returns the output parameter."""
        return self._outputvar

        r"""Computes :math:`\log(x)`.

        Returns
        -------
        """
        x = maps[self._inputvar]
        out = {self._outputvar : numpy.log(x)}

        r"""Computes :math:`y = e^{x}`.

        Returns
        -------
        """
        y = maps[self._outputvar]
        out = {self._inputvar : numpy.exp(y)}

        r"""Computes the Jacobian of :math:`y = \log(x)`.

        This is:

        .. math::

            \frac{\mathrm{d}y}{\mathrm{d}x} = \frac{1}{x}.

        Returns
        -------
        float
            The value of the jacobian at the given point(s).
        """
        return 1./x

        r"""Computes the Jacobian of :math:`y = e^{x}`.

        This is:

        .. math::

            \frac{\mathrm{d}y}{\mathrm{d}x} = e^{x}.

        Returns
        -------
        float
        """
        x = maps[self._outputvar]
        return numpy.exp(x)

class Logit(BaseTransform):
    """Applies a logit transform from an `inputvar` parameter to an `outputvar`
    parameter. This is the inverse of the logistic transform.

    Typically, the input of the logit function is assumed to have domain
    :math:`\in (0, 1)`. However, the `domain` argument can be used to expand
    this to any finite real interval.

    domain : tuple or distributions.bounds.Bounds, optional
        The domain of the input parameter. Can be any finite
        interval. Default is (0., 1.).
    """
    name = 'logit'

    def __init__(self, inputvar, outputvar, domain=(0., 1.)):
        self._bounds = Bounds(domain[0], domain[1],
                              btype_min='open', btype_max='open')
        # shortcuts for quick access later
        self._a = domain[0]
        self._b = domain[1]
        super(Logit, self).__init__()

    @property

    @property

    @property
    def bounds(self):
        """Returns the domain of the input parameter.
        """
        return self._bounds

    def logit(x, a=0., b=1.):
        r"""Computes the logit function with domain :math:`x \in (a, b)`.

        This is given by:

        .. math::

            \mathrm{logit}(x; a, b) = \log\left(\frac{x-a}{b-x}\right).

        Note that this is also the inverse of the logistic function with range
        :math:`(a, b)`.

        x : float
            The value to evaluate.
        a : float, optional
            The minimum bound of the domain of x. Default is 0.
        b : float, optional
            The maximum bound of the domain of x. Default is 1.

        Returns
        -------
        float
            The logit of x.
        """
        return numpy.log(x-a) - numpy.log(b-x)

    def logistic(x, a=0., b=1.):
        r"""Computes the logistic function with range :math:`\in (a, b)`.

        .. math::

            \mathrm{logistic}(x; a, b) = \frac{a + b e^x}{1 + e^x}.

        Note that this is also the inverse of the logit function with domain

        x : float
            The minimum bound of the range of the logistic function. Default
            is 0.
            The maximum bound of the range of the logistic function. Default
            is 1.

        Returns
        -------
        float
            The logistic of x.
        """
        expx = numpy.exp(x)
        return (a + b*expx)/(1. + expx)

        r"""Computes :math:`\mathrm{logit}(x; a, b)`.

        The domain :math:`a, b` of :math:`x` are given by the class's bounds.

        Returns
        -------
        """
        # check that x is in bounds
        isin = self._bounds.__contains__(x)
        if isinstance(isin, numpy.ndarray):
            isin = isin.all()
        if not isin:
            raise ValueError("one or more values are not in bounds")
        out = {self._outputvar : self.logit(x, self._a, self._b)}

        r"""Computes :math:`y = \mathrm{logistic}(x; a,b)`.

        The codomain :math:`a, b` of :math:`y` are given by the class's bounds.

        Returns
        -------
        """
        out = {self._inputvar : self.logistic(y, self._a, self._b)}

        r"""Computes the Jacobian of :math:`y = \mathrm{logit}(x; a,b)`.

        This is:

        .. math::

            \frac{\mathrm{d}y}{\mathrm{d}x} = \frac{b -a}{(x-a)(b-x)},

        where :math:`x \in (a, b)`.

        Returns
        -------
        float
        """
        if isinstance(isin, numpy.ndarray) and not isin.all():
        elif not isin:
            raise ValueError("{} is not in bounds".format(x))
        return (self._b - self._a)/((x - self._a)*(self._b - x))

        r"""Computes the Jacobian of :math:`y = \mathrm{logistic}(x; a,b)`.

        This is:

        .. math::

            \frac{\mathrm{d}y}{\mathrm{d}x} = \frac{e^x (b-a)}{(1+e^y)^2},

        where :math:`y \in (a, b)`.

        Returns
        -------
        float
        """
        return expx * (self._b - self._a) / (1. + expx)**2.

        """Initializes a Logit transform from the given section.

        The section must specify an input and output variable name. The domain
        of the input may be specified using `min-{input}`, `max-{input}`.
        Example:

            [{section}-logitq]
            name = logit
            inputvar = q
            outputvar = logitq
            min-q = 1
            max-q = 8

        Returns
        -------
        cls
        """
        # pull out the minimum, maximum values of the input variable
        inputvar = cp.get_opt_tag(section, 'inputvar', outputs)
        s = '-'.join([section, outputs])
        opt = 'min-{}'.format(inputvar)
        else:
        if cp.has_option(s, opt):
            a = cp.get_opt_tag(section, opt, outputs)
            skip_opts.append(opt)
        else:
            a = None
        opt = 'max-{}'.format(inputvar)
            b = cp.get_opt_tag(section, opt, outputs)
        else:
            b = None
        if a is None and b is not None or b is None and a is not None:
            raise ValueError("if providing a min(max)-{}, must also provide "
                             "a max(min)-{}".format(inputvar, inputvar))
        elif a is not None:
            additional_opts.update({'domain': (float(a), float(b))})
        return super(Logit, cls).from_config(cp, section, outputs, skip_opts,
                                             additional_opts)

#
#
#                             Inverse Transforms
#
#
class Mass1Mass2ToMchirpQ(MchirpQToMass1Mass2):
    """The inverse of MchirpQToMass1Mass2.
    """
    name = "mass1_mass2_to_mchirp_q"
    inverse = MchirpQToMass1Mass2
    transform = inverse.inverse_transform
    inverse_transform = inverse.transform
    jacobian = inverse.inverse_jacobian
    inverse_jacobian = inverse.jacobian

        self._inputs = [self.mass1_param, self.mass2_param]
        self._outputs = [self.mchirp_param, self.q_param]
        BaseTransform.__init__(self)

class Mass1Mass2ToMchirpEta(MchirpEtaToMass1Mass2):
    """The inverse of MchirpEtaToMass1Mass2.
    """
    name = "mass1_mass2_to_mchirp_eta"
    inverse = MchirpEtaToMass1Mass2
    _inputs = inverse._outputs
    _outputs = inverse._inputs

class DistanceToChirpDistance(ChirpDistanceToDistance):
    """The inverse of ChirpDistanceToDistance.
    """
    name = "distance_to_chirp_distance"
    inverse = ChirpDistanceToDistance
    _inputs = [parameters.distance, parameters.mchirp]
    _outputs = [parameters.chirp_distance]

class CartesianToSpherical(SphericalToCartesian):

    x : str
    y : str
    z : str
    polar : str
    """
    name = "cartesian_to_spherical"
    inverse = SphericalToCartesian

    def __init__(self, *args):
        super(CartesianToSpherical, self).__init__(*args)
        # swap inputs and outputs
        outputs = self._inputs
        inputs = self._outputs
        self._inputs = inputs
        self._outputs = outputs

class CartesianSpin1ToSphericalSpin1(CartesianToSpherical):
    """The inverse of SphericalSpin1ToCartesianSpin1.

    :py:class:`CartesianToSpherical` with spin-parameter names passed in
    instead.
    """
    name = "cartesian_spin_1_to_spherical_spin_1"

                        .format(self.name, CartesianToSpherical.name))
        super(CartesianSpin1ToSphericalSpin1, self).__init__(

class CartesianSpin2ToSphericalSpin2(CartesianToSpherical):
    """The inverse of SphericalSpin2ToCartesianSpin2.

    instead.
    """
    name = "cartesian_spin_2_to_spherical_spin_2"

        super(CartesianSpin2ToSphericalSpin2, self).__init__(

class CartesianSpinToAlignedMassSpin(AlignedMassSpinToCartesianSpin):
    """The inverse of AlignedMassSpinToCartesianSpin."""
    name = "cartesian_spin_to_aligned_mass_spin"
    inverse = AlignedMassSpinToCartesianSpin

class CartesianSpinToPrecessionMassSpin(PrecessionMassSpinToCartesianSpin):
    """The inverse of PrecessionMassSpinToCartesianSpin.
    """
    name = "cartesian_spin_to_precession_mass_spin"
    inverse = PrecessionMassSpinToCartesianSpin

class ChiPToCartesianSpin(CartesianSpinToChiP):
    """The inverse of `CartesianSpinToChiP`.
    """
    inverse = CartesianSpinToChiP

class Exponent(Log):
    """Applies an exponent transform to an `inputvar` parameter.

    This is the inverse of the log transform.

    """
    name = 'exponent'
    inverse = Log

        super(Exponent, self).__init__(outputvar, inputvar)

class Logistic(Logit):
    """Applies a logistic transform from an `input` parameter to an `output`
    parameter. This is the inverse of the logit transform.

    Typically, the output of the logistic function has range :math:`\in [0,1)`.
    However, the `codomain` argument can be used to expand this to any
    finite real interval.

    frange : tuple or distributions.bounds.Bounds, optional
        The range of the output parameter. Can be any finite
    """
    name = 'logistic'
    inverse = Logit

    def __init__(self, inputvar, outputvar, codomain=(0.,1.)):
        super(Logistic, self).__init__(outputvar, inputvar, domain=codomain)

    @property
        """Returns the range of the output parameter.
        """

        """Initializes a Logistic transform from the given section.

        The section must specify an input and output variable name. The
        codomain of the output may be specified using `min-{output}`,
        `max-{output}`. Example:

            [{section}-q]
            name = logistic
            inputvar = logitq
            outputvar = q
            min-q = 1
            max-q = 8

        Returns
        -------
        cls
        """
        # pull out the minimum, maximum values of the output variable
        outputvar = cp.get_opt_tag(section, 'output', outputs)
        else:
        opt = 'min-{}'.format(outputvar)
        else:
            a = None
        opt = 'max-{}'.format(outputvar)
        else:
            b = None
                             "a max(min)-{}".format(outputvar, outputvar))
            additional_opts.update({'codomain': (float(a), float(b))})
        return super(Logistic, cls).from_config(cp, section, outputs,
                                                skip_opts, additional_opts)

# set the inverse of the forward transforms to the inverse transforms
MchirpQToMass1Mass2.inverse = Mass1Mass2ToMchirpQ
ChirpDistanceToDistance.inverse = DistanceToChirpDistance
SphericalToCartesian.inverse = CartesianToSpherical
SphericalSpin1ToCartesianSpin1.inverse = CartesianSpin1ToSphericalSpin1
SphericalSpin2ToCartesianSpin2.inverse = CartesianSpin2ToSphericalSpin2
AlignedMassSpinToCartesianSpin.inverse = CartesianSpinToAlignedMassSpin
PrecessionMassSpinToCartesianSpin.inverse = CartesianSpinToPrecessionMassSpin
ChiPToCartesianSpin.inverse = CartesianSpinToChiP
Log.inverse = Exponent
Logit.inverse = Logistic

#
#
#                      Collections of transforms
#
#

# dictionary of all transforms
transforms = {
    CustomTransform.name : CustomTransform,
    MchirpQToMass1Mass2.name : MchirpQToMass1Mass2,
    Mass1Mass2ToMchirpQ.name : Mass1Mass2ToMchirpQ,
    MchirpEtaToMass1Mass2.name : MchirpEtaToMass1Mass2,
    Mass1Mass2ToMchirpEta.name : Mass1Mass2ToMchirpEta,
    ChirpDistanceToDistance.name : ChirpDistanceToDistance,
    DistanceToChirpDistance.name : DistanceToChirpDistance,
    SphericalToCartesian.name : SphericalToCartesian,
    CartesianToSpherical.name : CartesianToSpherical,
    SphericalSpin1ToCartesianSpin1.name : SphericalSpin1ToCartesianSpin1,
    CartesianSpin1ToSphericalSpin1.name : CartesianSpin1ToSphericalSpin1,
    SphericalSpin2ToCartesianSpin2.name : SphericalSpin2ToCartesianSpin2,
    CartesianSpin2ToSphericalSpin2.name : CartesianSpin2ToSphericalSpin2,
    DistanceToRedshift.name : DistanceToRedshift,
    AlignedMassSpinToCartesianSpin.name : AlignedMassSpinToCartesianSpin,
    CartesianSpinToAlignedMassSpin.name : CartesianSpinToAlignedMassSpin,
    PrecessionMassSpinToCartesianSpin.name : PrecessionMassSpinToCartesianSpin,
    CartesianSpinToPrecessionMassSpin.name : CartesianSpinToPrecessionMassSpin,
    ChiPToCartesianSpin.name : ChiPToCartesianSpin,
    CartesianSpinToChiP.name : CartesianSpinToChiP,
    Log.name : Log,
    Exponent.name : Exponent,
    Logit.name : Logit,
    Logistic.name : Logistic,
    LambdaFromTOVFile.name : LambdaFromTOVFile,
    LambdaFromMultipleTOVFiles.name : LambdaFromMultipleTOVFiles
}

# standard CBC transforms: these are transforms that do not require input
# arguments; they are typically used in CBC parameter estimation to transform
# to coordinates understood by the waveform generator
common_cbc_forward_transforms = [
    MchirpQToMass1Mass2(), DistanceToRedshift(),
    SphericalToCartesian(parameters.spin1x, parameters.spin1y,
                         parameters.spin1z, parameters.spin1_a,
                         parameters.spin1_azimuthal, parameters.spin1_polar),
    SphericalToCartesian(parameters.spin2x, parameters.spin2y,
                         parameters.spin2z, parameters.spin2_a,
                         parameters.spin2_azimuthal, parameters.spin2_polar),
    AlignedMassSpinToCartesianSpin(), PrecessionMassSpinToCartesianSpin(),
    ChiPToCartesianSpin(), ChirpDistanceToDistance()
]
common_cbc_inverse_transforms = [_t.inverse()
                                 for _t in common_cbc_forward_transforms
                                 if not (_t.inverse is None or
                                         _t.name == 'spherical_to_cartesian')]
common_cbc_inverse_transforms.extend([
    CartesianToSpherical(parameters.spin1x, parameters.spin1y,
    CartesianToSpherical(parameters.spin2x, parameters.spin2y,
                         parameters.spin2_azimuthal, parameters.spin2_polar)])

common_cbc_transforms = common_cbc_forward_transforms + \
                        common_cbc_inverse_transforms

def get_common_cbc_transforms(requested_params, variable_args,
                              valid_params=None):
    """Determines if any additional parameters from the InferenceFile are
    needed to get derived parameters that user has asked for.

    First it will try to add any base parameters that are required to calculate
    the derived parameters. Then it will add any sampling parameters that are
    required to calculate the base parameters needed.

    requested_params : list
        List of parameters that user wants.
    variable_args : list
        List of parameters that InferenceFile has.
    valid_params : list
        List of parameters that can be accepted.

    Returns
    -------
        Updated list of parameters that user wants.
    all_c : list
        List of BaseTransforms to apply.
    """
    variable_args = set(variable_args) if not isinstance(variable_args, set) \
                                    else variable_args

    # try to parse any equations by putting all strings together
    # this will get some garbage but ensures all alphanumeric/underscored
    # parameter names are added
    new_params = []
    for opt in requested_params:
        s = ""
        for ch in opt:
            s += ch if ch.isalnum() or ch == "_" else " "
        new_params += s.split(" ")
    requested_params = set(list(requested_params) + list(new_params))

    # can pass a list of valid parameters to remove garbage from parsing above
    if valid_params:
        valid_params = set(valid_params)
        requested_params = requested_params.intersection(valid_params)

    # find all the transforms for the requested derived parameters
    # calculated from base parameters
    from_base_c = []
    for converter in common_cbc_inverse_transforms:
        if (converter.outputs.issubset(variable_args) or
                converter.outputs.isdisjoint(requested_params)):
            continue
        intersect = converter.outputs.intersection(requested_params)
        if (not intersect or intersect.issubset(converter.inputs) or
                intersect.issubset(variable_args)):
            continue
        requested_params.update(converter.inputs)
        from_base_c.append(converter)

    # find all the tranforms for the required base parameters
    # calculated from sampling parameters
    to_base_c = []
    for converter in common_cbc_forward_transforms:
        if (converter.inputs.issubset(variable_args) and
                len(converter.outputs.intersection(requested_params)) > 0):
            requested_params.update(converter.inputs)
            to_base_c.append(converter)
            variable_args.update(converter.outputs)

    # get list of transforms that converts sampling parameters to the base
    # parameters and then converts base parameters to the derived parameters
    all_c = to_base_c + from_base_c

    return list(requested_params), all_c

def apply_transforms(samples, transforms, inverse=False):
    """Applies a list of BaseTransform instances on a mapping object.

    samples : {FieldArray, dict}
        Mapping object to apply transforms to.
    transforms : list
        List of BaseTransform instances to apply. Nested transforms are assumed
        to be in order for forward transforms.
    inverse : bool, optional
        Apply inverse transforms. In this case transforms will be applied in
        the opposite order. Default is False.

    Returns
    -------
        Mapping object with transforms applied. Same type as input.
    """
    if inverse:
        transforms = transforms[::-1]
    for t in transforms:
        try:
            if inverse:
                samples = t.inverse_transform(samples)
            else:
                samples = t.transform(samples)
        except NotImplementedError:
            continue
    return samples

def compute_jacobian(samples, transforms, inverse=False):
    """Computes the jacobian of the list of transforms at the given sample
    points.

        Mapping object specifying points at which to compute jacobians.
        Compute inverse jacobians. Default is False.

    Returns
    -------
    float :
        The product of the jacobians of all fo the transforms.
    """
    j = 1.
        for t in transforms:
            j *= t.inverse_jacobian(samples)
    else:
            j *= t.jacobian(samples)
    return j

def order_transforms(transforms):
    """Orders transforms to ensure proper chaining.

    For example, if `transforms = [B, A, C]`, and `A` produces outputs needed
    by `B`, the transforms will be re-rorderd to `[A, B, C]`.

        List of transform instances to order.

    Outputs
    -------
    list :
        List of transformed ordered such that forward transforms can be carried
        out without error.
    """
    # get a set of all inputs and all outputs
    outputs = set().union(*[t.outputs for t in transforms])
    out = []
    remaining = [t for t in transforms]
    while remaining:
        # pull out transforms that have no inputs in the set of outputs
        leftover = []
        for t in remaining:
            if t.inputs.isdisjoint(outputs):
                out.append(t)
                outputs -= t.outputs
            else:
                leftover.append(t)
        remaining = leftover
    return out

def read_transforms_from_config(cp, section="transforms"):
    """Returns a list of PyCBC transform instances for a section in the
    given configuration file.

    If the transforms are nested (i.e., the output of one transform is the
    input of another), the returned list will be sorted by the order of the
    nests.

    cp : WorflowConfigParser
        An open config file to read.
    section : {"transforms", string}
        Prefix on section names from which to retrieve the transforms.

    Returns
    -------
    list
        A list of the parsed transforms.
    """
    trans = []
    for subsection in cp.get_subsections(section):
        name = cp.get_opt_tag(section, "name", subsection)
        t = transforms[name].from_config(cp, section, subsection)
        trans.append(t)
    return order_transforms(trans)

# Copyright (C) 2012  Alex Nitz, Josh Willis
#
#
#

#
#
#                                   Preamble
#
#
"""PyCBC contains a toolkit for CBC gravitational wave analysis
"""
from __future__ import (absolute_import, print_function)
import subprocess, os, sys, tempfile, signal, warnings

# Filter annoying Cython warnings that serve no good purpose.
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")
import random
import string

try:
    # This will fail when pycbc is imported during the build process,
    # before version.py has been generated.
    from .version import git_hash
    from .version import version as pycbc_version
except:
    git_hash = 'none'
    pycbc_version = 'none'

__version__ = pycbc_version

def init_logging(verbose=False, format='%(asctime)s %(message)s'):
    """Common utility for setting up logging in PyCBC.

    Installs a signal handler such that verbosity can be activated at
    run-time by sending a SIGUSR1 to the process.

    verbose : bool or int, optional
        What level to set the verbosity level to. Accepts either a boolean
        or an integer representing the level to set. If True/False will set to
        ``logging.INFO``/``logging.WARN``. For higher logging levels, pass
        an integer representing the level to set (see the ``logging`` module
        for details). Default is ``False`` (``logging.WARN``).
    format : str, optional
        The format to use for logging messages.
    """
    def sig_handler(signum, frame):
        logger = logging.getLogger()
        log_level = logger.level
        if log_level == logging.DEBUG:
            log_level = logging.WARN
        else:
            log_level = logging.DEBUG
        logging.warn('Got signal %d, setting log level to %d',
                     signum, log_level)
        logger.setLevel(log_level)

    signal.signal(signal.SIGUSR1, sig_handler)

    if not verbose:
        initial_level = logging.WARN
    elif int(verbose) == 1:
        initial_level = logging.INFO
    else:
        initial_level = int(verbose)
    logging.getLogger().setLevel(initial_level)
    logging.basicConfig(format=format, level=initial_level)

def makedir(path):
    """
    Make the analysis directory path and any parent directories that don't
    already exist. Will do nothing if path already exists.
    """
    if path is not None and not os.path.exists(path):
        os.makedirs(path)

# PyCBC-Specific Constants

# Set the value we want any aligned memory calls to use
# N.B.: *Not* all pycbc memory will be aligned to multiples
# of this value

PYCBC_ALIGNMENT = 32

# Dynamic range factor: a large constant for rescaling
# GW strains.  This is 2**69 rounded to 17 sig.fig.

DYN_RANGE_FAC =  5.9029581035870565e+20

# String used to separate parameters in configuration file section headers.
# This is used by the distributions and transforms modules
VARARGS_DELIM = '+'

# Check for optional components of the PyCBC Package
try:
    # This is a crude check to make sure that the driver is installed
    try:
        loaded_modules = subprocess.Popen(['lsmod'], stdout=subprocess.PIPE).communicate()[0]
        loaded_modules = loaded_modules.decode()
        if 'nvidia' not in loaded_modules:
            raise ImportError("nvidia driver may not be installed correctly")
    except OSError:
        pass

    # Check that pycuda is installed and can talk to the driver
    import pycuda.driver as _pycudadrv

    HAVE_CUDA=True
    HAVE_CUDA=False

# Check for MKL capability
try:
    import pycbc.fft.mkl
    HAVE_MKL=True
    HAVE_MKL=False

# Check for openmp suppport, currently we pressume it exists, unless on
# platforms (mac) that are silly and don't use the standard gcc.
if sys.platform == 'darwin':
    HAVE_OMP = False
else:
    HAVE_OMP = True

# https://pynative.com/python-generate-random-string/
def random_string(stringLength=10):
    """Generate a random string of fixed length """
    letters = string.ascii_lowercase
    return ''.join(random.choice(letters) for i in range(stringLength))

def gps_now():
    """Return the current GPS time as a float using Astropy.
    """
    from astropy.time import Time

    return float(Time.now().gps)

# Copyright (C) 2017  Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
""" This modules contains information about the announced LIGO/Virgo
compact binary mergers
"""

# For the time being all quantities are the 1-d median value
# FIXME with posteriors when available and we can just post-process that

# LVC catalogs
base_lvc_url = "https://www.gw-openscience.org/eventapi/jsonfull/{}/"
_catalogs = {'GWTC-1-confident': 'LVC',
             'GWTC-1-marginal': 'LVC',
             'Initial_LIGO_Virgo': 'LVC',
             'O1_O2-Preliminary': 'LVC',
             'O3_Discovery_Papers': 'LVC',
             'GWTC-2': 'LVC'}

# add some aliases
_aliases = {}
_aliases['gwtc-1'] = 'GWTC-1-confident'
_aliases['gwtc-2'] = 'GWTC-2'

def list_catalogs():
    """Return a list of possible GW catalogs to query"""
    return list(_catalogs.keys())

def get_source(source):
    """Get the source data for a particular GW catalog
    """
    if source in _aliases:
        source = _aliases[source]

    if source in _catalogs:
        catalog_type = _catalogs[source]
        if catalog_type == 'LVC':
            fname = get_file(base_lvc_url.format(source), cache=True)
    else:
        raise ValueError('Unkown catalog source {}'.format(source))
    return data['events']

#
#
#

#
#
#                                   Preamble
#
#
""" This package provides information about LIGO/Virgo detections of
"""

import os
from . import catalog

_aliases['mchirp'] = 'chirp_mass_source'
_aliases['mass1'] = 'mass_1_source'
_aliases['mass2'] = 'mass_2_source'
_aliases['snr'] = 'network_matched_filter_snr'
_aliases['z'] = _aliases['redshift'] = 'redshift'
_aliases['distance'] = 'luminosity_distance'

class Merger(object):
    """Informaton about a specific compact binary merger"""
    def __init__(self, name, source='gwtc-1'):
        """ Return the information of a merger

        name: str
            The name (GW prefixed date) of the merger event.
        """
        try:
            self.data = catalog.get_source(source)[name]
            # Try common name
            data = catalog.get_source(source)
            for mname in data:
                cname = data[mname]['commonName']
                if cname == name:
                    name = mname
                    self.data = data[name]
                    break
            else:
                raise ValueError('Did not find merger matching'
                                 ' name: {}'.format(name))

        # Set some basic params from the dataset
        for key in self.data:
            setattr(self, '_raw_' + key, self.data[key])

        for key in _aliases:
            setattr(self, key, self.data[_aliases[key]])

        self.common_name = self.data['commonName']
        self.time = self.data['GPS']
        self.frame = 'source'

    def median1d(self, name, return_errors=False):
        """ Return median 1d marginalized parameters

        name: str
            The name of the parameter requested
        return_errors: Optional, {bool, False}
            If true, return a second and third parameter that represents the
            lower and upper 90% error on the parameter.

        Returns
        -------
        param: float or tuple
            The requested parameter
        """
        if name in _aliases:
            name = _aliases[name]

        try:
            if return_errors:
                mid = self.data[name]
                high = self.data[name + '_upper']
                low = self.data[name + '_lower']
                return (mid, low, high)
            else:
                return self.data[name]
            print(e)
            raise RuntimeError("Cannot get parameter {}".format(name))

    def strain(self, ifo, duration=32, sample_rate=4096):
        """ Return strain around the event

        Currently this will return the strain around the event in the smallest
        format available. Selection of other data is not yet available.

        ifo: str
            The name of the observatory you want strain for. Ex. H1, L1, V1

        Returns
        -------
        strain: pycbc.types.TimeSeries
            Strain around the event.
        """
        from pycbc.io import get_file
        from pycbc.frame import read_frame

        if sample_rate == 4096:
            sampling = "4KHz"
        elif sample_rate == 16384:
            sampling = "16KHz"

        for fdict in self.data['strain']:
            if (fdict['detector'] == ifo and fdict['duration'] == duration and
                    fdict['sampling_rate'] == sample_rate and
                    fdict['format'] == 'gwf'):
                url = fdict['url']
                break

        ver = url.split('/')[-1].split('-')[1].split('_')[-1]
        channel = "{}:GWOSC-{}_{}_STRAIN".format(ifo, sampling.upper(), ver)

        filename = get_file(url, cache=True)
        return read_frame(str(filename), str(channel))

class Catalog(object):
    """Manage a set of binary mergers"""

    def __init__(self, source='gwtc-1'):
        """ Return the set of detected mergers

        The set of detected mergers. At some point this may have some selection
        abilities.
        """
        self.data = catalog.get_source(source=source)
        self.mergers = {name: Merger(name,
                                     source=source) for name in self.data}
        self.names = self.mergers.keys()

        return len(self.mergers)

    def __getitem__(self, key):
        try:
            return self.mergers[key]
            for m in self.mergers:
                if key == self.mergers[m].common_name:
                    break
            else:
                                 ' name: {}'.format(key))
            return self.mergers[m]

        self.mergers[key] = value

    def __delitem__(self, key):
        del self.mergers[key]

        return iter(self.mergers)

    def median1d(self, param, return_errors=False):

        name: str

        Returns
        -------
        param: nump.ndarray or tuple
        """
        v = [self.mergers[m].median1d(param, return_errors=return_errors)
             for m in self.mergers]
        if return_errors:
            value, merror, perror = zip(*v)
            return numpy.array(value), numpy.array(merror), numpy.array(perror)
        else:
            return numpy.array(v)

#
#
"""
This modules provides classes for evaluating angular distributions.
"""

from six.moves.configparser import Error
from pycbc import boundaries
from pycbc.distributions import bounded
from pycbc.distributions import uniform

class UniformAngle(uniform.Uniform):
    """A uniform distribution in which the dependent variable is between
    `[0,2pi)`.

    The domain of the distribution may optionally be made cyclic using the
    `cyclic_domain` parameter.

    Bounds may be provided to limit the range for which the pdf has support.
    If provided, the parameter bounds are in radians.

    cyclic_domain : {False, bool}
        If True, cyclic bounds on [0, 2pi) are applied to all values when
        evaluating the pdf. This is done prior to any additional bounds
        specified for a parameter are applied. Default is False.
    \**params :
        The keyword arguments should provide the names of parameters and
        (optionally) their corresponding bounds, as either
        `boundaries.Bounds` instances or tuples. The bounds must be
        in [0,2PI). These are converted to radians for storage. None may also
        be passed; in that case, the domain bounds will be used.

    name : 'uniform_angle'
        The name of this distribution.
    params : list of strings
        The list of parameter names.
    bounds : dict
        A dictionary of the parameter names and their bounds, in radians.
    domain : boundaries.Bounds
        The domain of the distribution.

    Notes
    ------
    For more information, see Uniform.
    """
    name = 'uniform_angle'

    _domainbounds = (0, 2*numpy.pi)

    def __init__(self, cyclic_domain=False, **params):
        # _domain is a bounds instance used to apply cyclic conditions; this is
        # applied first, before any bounds specified in the initialization
        # are used
        self._domain = boundaries.Bounds(self._domainbounds[0],
            self._domainbounds[1], cyclic=cyclic_domain)

        for p,bnds in params.items():
            if bnds is None:
                bnds = self._domain
            elif isinstance(bnds, boundaries.Bounds):
                # convert to radians
                bnds._min = bnds._min.__class__(bnds._min)
                bnds._max = bnds._max.__class__(bnds._max)
            else:
                # create a Bounds instance from the given tuple
                bnds = boundaries.Bounds(bnds[0], bnds[1])
            # check that the bounds are in the domain
            if bnds.min < self._domain.min or bnds.max > self._domain.max:
                raise ValueError("bounds must be in [{x},{y}); "
                    "got [{a},{b})".format(x=self._domain.min,
                    y=self._domain.max, a=bnds.min,
                    b=bnds.max))

            # update
            params[p] = bnds
        super(UniformAngle, self).__init__(**params)

    @property
    def domain(self):
        """Returns the domain of the distribution."""
        return self._domain

    def apply_boundary_conditions(self, **kwargs):
        """Maps values to be in [0, 2pi) (the domain) first, before applying
        any additional boundary conditions.

        \**kwargs :
            The keyword args should be the name of a parameter and value to
            apply its boundary conditions to. The arguments need not include
            all of the parameters in self.

        Returns
        -------
        dict
            A dictionary of the parameter names and the conditioned values.
        """
        # map values to be within the domain
        kwargs = dict([[p, self._domain.apply_conditions(val)]
                      for p,val in kwargs.items() if p in self._bounds])
        # now apply additional conditions
        return super(UniformAngle, self).apply_boundary_conditions(**kwargs)

    def from_config(cls, cp, section, variable_args):
        """Returns a distribution based on a configuration file.

        The parameters for the distribution are retrieved from the section
        titled "[`section`-`variable_args`]" in the config file. By default,
        only the name of the distribution (`uniform_angle`) needs to be
        specified. This will results in a uniform prior on `[0, 2pi)`. To
        make the domain cyclic, add `cyclic_domain =`. To specify boundaries
        that are not `[0, 2pi)`, add `(min|max)-var` arguments, where `var`
        is the name of the variable.

        For example, this will initialize a variable called `theta` with a
        uniform distribution on `[0, 2pi)` without cyclic boundaries:

            [{section}-theta]
            name = uniform_angle

        This will make the domain cyclic on `[0, 2pi)`:

            cyclic_domain =

            A parsed configuration file that contains the distribution
            options.
        variable_args : str
            The names of the parameters for this distribution, separated by
            ``VARARGS_DELIM``. These must appear in the "tag" part

        Returns
        -------
        UniformAngle
            A distribution instance from the pycbc.inference.prior module.
        """
        # we'll retrieve the setting for cyclic_domain directly
        additional_opts = {'cyclic_domain': cp.has_option_tag(section,
                                            'cyclic_domain', variable_args)}
        return bounded.bounded_from_config(cls, cp, section, variable_args,
                                           bounds_required=False,
                                           additional_opts=additional_opts)

class SinAngle(UniformAngle):
    r"""A sine distribution; the pdf of each parameter `\theta` is given by:

    ..math::
        p(\theta) = \frac{\sin \theta}{\cos\theta_0 - \cos\theta_1}, \theta_0 \leq \theta < \theta_1,

    and 0 otherwise. Here, :math:`\theta_0, \theta_1` are the bounds of the
    parameter.

    The domain of this distribution is `[0, pi]`. This is accomplished by
    putting hard boundaries at `[0, pi]`. Bounds may be provided to further
    limit the range for which the pdf has support.  As with `UniformAngle`,
    these are initialized in radians.

        in [0,PI]. These are converted to radians for storage. None may also

    name : 'sin_angle'
    """
    name = 'sin_angle'
    _func = numpy.cos
    _dfunc = numpy.sin
    _arcfunc = numpy.arccos
    _domainbounds = (0, numpy.pi)

    def __init__(self, **params):
        super(SinAngle, self).__init__(**params)
        # replace the domain
            self._domainbounds[1], btype_min='closed', btype_max='closed',
            cyclic=False)
        self._lognorm = -sum([numpy.log(
            abs(self._func(bnd[1]) - self._func(bnd[0]))) \
            for bnd in self._bounds.values()])
        self._norm = numpy.exp(self._lognorm)

    def _cdfinv_param(self, arg, value):
        """Return inverse of cdf for mapping unit interval to parameter bounds.
        """
        scale = (numpy.cos(self._bounds[arg][0])
                 - numpy.cos(self._bounds[arg][1]))
        offset = 1. + numpy.cos(self._bounds[arg][1]) / scale
        new_value = numpy.arccos(-scale * (value - offset))
        return new_value

    def _pdf(self, **kwargs):
        """Returns the pdf at the given values. The keyword arguments must
        contain all of parameters in self's params. Unrecognized arguments are
        ignored.
        """
        if kwargs not in self:
            return 0.
        return self._norm * \
            self._dfunc(numpy.array([kwargs[p] for p in self._params])).prod()

    def _logpdf(self, **kwargs):
        """Returns the log of the pdf at the given values. The keyword
        arguments must contain all of parameters in self's params. Unrecognized
        arguments are ignored.
        """
            return -numpy.inf
        return self._lognorm + \
            numpy.log(self._dfunc(
                numpy.array([kwargs[p] for p in self._params]))).sum()

    def rvs(self, size=1, param=None):
        """Gives a set of random values drawn from this distribution.

        size : {1, int}
            The number of values to generate; default is 1.
        param : {None, string}
            If provided, will just return values for the given parameter.
            Otherwise, returns random values for each parameter.

        Returns
        -------
        structured array
            The random values in a numpy structured array. If a param was
            specified, the array will only have an element corresponding to the
            given parameter. Otherwise, the array will have an element for each
            parameter in self's params.
        """
        if param is not None:
            dtype = [(param, float)]
        else:
            dtype = [(p, float) for p in self.params]
        arr = numpy.zeros(size, dtype=dtype)
        for (p,_) in dtype:
            arr[p] = self._arcfunc(numpy.random.uniform(
                                    self._func(self._bounds[p][0]),
                                    self._func(self._bounds[p][1]),
                                    size=size))
        return arr

class CosAngle(SinAngle):
    r"""A cosine distribution. This is the same thing as a sine distribution,
    but with the domain shifted to `[-pi/2, pi/2]`. See SinAngle for more
    details.

        in [-PI/2, PI/2].

    ----------------
    name : 'cos_angle'
    """
    name = 'cos_angle'
    _func = numpy.sin
    _dfunc = numpy.cos
    _arcfunc = numpy.arcsin
    _domainbounds = (-numpy.pi/2, numpy.pi/2)

    def _cdfinv_param(self, param, value):
        a = self._bounds[param][0]
        b = self._bounds[param][1]
        scale = numpy.sin(b) - numpy.sin(a)
        offset = 1. - numpy.sin(b)/(numpy.sin(b) - numpy.sin(a))
        new_value = numpy.arcsin((value - offset) * scale)

class UniformSolidAngle(bounded.BoundedDist):
    """A distribution that is uniform in the solid angle of a sphere. The names
    of the two angluar parameters can be specified on initalization.

    polar_angle : {'theta', str}
        The name of the polar angle.
    azimuthal_angle : {'phi', str}
        The name of the azimuthal angle.
    polar_bounds : {None, tuple}
        Limit the polar angle to the given bounds. If None provided, the polar
        angle will vary from 0 (the north pole) to pi (the south pole). The
        bounds should be specified as factors of pi. For example, to limit
        the distribution to the northern hemisphere, set
        `polar_bounds=(0,0.5)`.
    azimuthal_bounds : {None, tuple}
        Limit the azimuthal angle to the given bounds. If None provided, the
        azimuthal angle will vary from 0 to 2pi. The
        the distribution to the one hemisphere, set `azimuthal_bounds=(0,1)`.
    azimuthal_cyclic_domain : {False, bool}
        Make the domain of the azimuthal angle be cyclic; i.e., azimuthal
        values are constrained to be in [0, 2pi) using cyclic boundaries prior
        to applying any other boundary conditions and prior to evaluating the
        pdf. Default is False.

    name : 'uniform_solidangle'
        The name of the distribution.
        The bounds on each angle. The keys are the names of the polar and
        azimuthal angles, the values are the minimum and maximum of each, in
        radians. For example, if the distribution was initialized with
        `polar_angle='theta', polar_bounds=(0,0.5)` then the bounds will have
        `'theta': 0, 1.5707963267948966` as an entry.
    params : list
        The names of the polar and azimuthal angles.
    polar_angle : str
    azimuthal_angle : str
    """
    name = 'uniform_solidangle'
    _polardistcls = SinAngle
    _azimuthaldistcls = UniformAngle
    _default_polar_angle = 'theta'
    _default_azimuthal_angle = 'phi'

    def __init__(self, polar_angle=None, azimuthal_angle=None,
                 polar_bounds=None, azimuthal_bounds=None,
                 azimuthal_cyclic_domain=False):
        if polar_angle is None:
            polar_angle = self._default_polar_angle
        if azimuthal_angle is None:
            azimuthal_angle = self._default_azimuthal_angle
        self._polardist = self._polardistcls(**{
            polar_angle: polar_bounds})
        self._azimuthaldist = self._azimuthaldistcls(**{
            azimuthal_angle: azimuthal_bounds,
            'cyclic_domain': azimuthal_cyclic_domain})
        self._polar_angle = polar_angle
        self._azimuthal_angle = azimuthal_angle
        self._bounds = self._polardist.bounds.copy()
        self._bounds.update(self._azimuthaldist.bounds)
        self._params = sorted(self._bounds.keys())

    @property
    def polar_angle(self):
        return self._polar_angle

    @property
    def azimuthal_angle(self):
        return self._azimuthal_angle

        """ Return the cdfinv for a single given parameter """
        if param == self.polar_angle:
            return self._polardist._cdfinv_param(param, value)
        elif param == self.azimuthal_angle:
            return self._azimuthaldist._cdfinv_param(param, value)

        """Maps the given values to be within the domain of the azimuthal and
        polar angles, before applying any other boundary conditions.

            The keyword args must include values for both the azimuthal and
            polar angle, using the names they were initilialized with. For
            example, if `polar_angle='theta'` and `azimuthal_angle=`phi`, then
            the keyword args must be `theta={val1}, phi={val2}`.

        Returns
        -------
        dict
        """
        polarval = kwargs[self._polar_angle]
        azval = kwargs[self._azimuthal_angle]
        # constrain each angle to its domain
        polarval = self._polardist._domain.apply_conditions(polarval)
        azval = self._azimuthaldist._domain.apply_conditions(azval)
        # apply any other boundary conditions
        polarval = self._bounds[self._polar_angle].apply_conditions(polarval)
        azval = self._bounds[self._azimuthal_angle].apply_conditions(azval)
        return {self._polar_angle: polarval, self._azimuthal_angle: azval}

        """
        Returns the pdf at the given angles.

        \**kwargs:
            The keyword arguments should specify the value for each angle,
            using the names of the polar and azimuthal angles as the keywords.
            Unrecognized arguments are ignored.

        Returns
        -------
        float
            The value of the pdf at the given values.
        """
        return self._polardist._pdf(**kwargs) * \
            self._azimuthaldist._pdf(**kwargs)

        """
        Returns the logpdf at the given angles.

        Returns
        -------
        float
        """
        return self._polardist._logpdf(**kwargs) +\
            self._azimuthaldist._logpdf(**kwargs)

        Returns
        -------
        """
        else:
            if p == self._polar_angle:
                arr[p] = self._polardist.rvs(size=size)
            elif p == self._azimuthal_angle:
                arr[p] = self._azimuthaldist.rvs(size=size)
            else:
                raise ValueError("unrecognized parameter %s" %(p))
        return arr

        The section must have the names of the polar and azimuthal angles in
        the tag part of the section header. For example:

            [prior-theta+phi]
            name = uniform_solidangle

        If nothing else is provided, the default names and bounds of the polar
        and azimuthal angles will be used. To specify a different name for
        each angle, set the `polar-angle` and `azimuthal-angle` attributes. For
        example:

            [prior-foo+bar]
            polar-angle = foo
            azimuthal-angle = bar

        Note that the names of the variable args in the tag part of the section
        name must match the names of the polar and azimuthal angles.

        Bounds may also be specified for each angle, as factors of pi. For
        example:

            polar-angle = theta
            azimuthal-angle = phi
            min-theta = 0
            max-theta = 0.5

        This will return a distribution that is uniform in the upper
        hemisphere.

        By default, the domain of the azimuthal angle is `[0, 2pi)`. To make
        this domain cyclic, add `azimuthal_cyclic_domain =`.

        cp : ConfigParser instance
            The config file.
            The name of the section.

        Returns
        -------
        UniformSolidAngle
        """
        tag = variable_args
        variable_args = variable_args.split(VARARGS_DELIM)

        # get the variables that correspond to the polar/azimuthal angles
        try:
            polar_angle = cp.get_opt_tag(section, 'polar-angle', tag)
        except Error:
            polar_angle = cls._default_polar_angle
        try:
            azimuthal_angle = cp.get_opt_tag(section, 'azimuthal-angle', tag)
            azimuthal_angle = cls._default_azimuthal_angle

        if polar_angle not in variable_args:
            raise Error("polar-angle %s is not one of the variable args (%s)"%(
                polar_angle, ', '.join(variable_args)))
        if azimuthal_angle not in variable_args:
            raise Error("azimuthal-angle %s is not one of the variable args "%(
                azimuthal_angle) + "(%s)"%(', '.join(variable_args)))

        # get the bounds, if provided
        polar_bounds = bounded.get_param_bounds_from_config(
                                                   cp, section, tag,
                                                   polar_angle)
        azimuthal_bounds = bounded.get_param_bounds_from_config(
                                                   azimuthal_angle)

        # see if the a cyclic domain is desired for the azimuthal angle
        azimuthal_cyclic_domain = cp.has_option_tag(section,
            'azimuthal_cyclic_domain', tag)

        return cls(polar_angle=polar_angle, azimuthal_angle=azimuthal_angle,
                   polar_bounds=polar_bounds,
                   azimuthal_bounds=azimuthal_bounds,
                   azimuthal_cyclic_domain=azimuthal_cyclic_domain)

__all__ = ['UniformAngle', 'SinAngle', 'CosAngle', 'UniformSolidAngle']

# Copyright (C) 2016 Miriam Cabero Mueller, Collin Capano
#
#
"""
This modules provides classes for evaluating arbitrary distributions from
a file.
"""

import h5py
import scipy.stats
import pycbc.transforms

class Arbitrary(bounded.BoundedDist):
    r"""A distribution constructed from a set of parameter values using a kde.
    Bounds may be optionally provided to limit the range.

    bounds : dict, optional
        Independent bounds on one or more parameters may be provided to limit
        the range of the kde.
    bandwidth : str, optional
        Set the bandwidth method for the KDE. See
        :py:func:`scipy.stats.gaussian_kde` for details. Default is "scott".
        The keyword arguments should provide the names of the parameters and
        a list of their parameter values. If multiple parameters are provided,
        a single kde will be produced with dimension equal to the number of
        parameters.
    """
    name = 'arbitrary'

    def __init__(self, bounds=None, bandwidth="scott", **kwargs):
        # initialize the bounds
        if bounds is None:
            bounds = {}
        bounds.update({p: None for p in kwargs if p not in bounds})
        super(Arbitrary, self).__init__(**bounds)
        # check that all parameters specified in bounds have samples
        if set(self.params) != set(kwargs.keys()):
            raise ValueError("Must provide samples for all parameters given "
                             "in the bounds dictionary")
        # if bounds are provided use logit transform to move the points
        # to +/- inifinity
        self._transforms = {}
        self._tparams = {}
        for param,bnds in self.bounds.items():
            if numpy.isfinite(bnds[1] - bnds[0]):
                tparam = 'logit'+param
                samples = kwargs[param]
                t = pycbc.transforms.Logit(param, tparam, domain=bnds)
                self._transforms[tparam] = t
                self._tparams[param] = tparam
                # remove any sample points that fall out side of the bounds
                outside = bnds.__contains__(samples)
                if outside.any():
                    samples = samples[outside]
                # transform the sample points
                kwargs[param] = t.transform({param: samples})[tparam]
            elif not (~numpy.isfinite(bnds[0]) and ~numpy.isfinite(bnds[1])):
                raise ValueError("if specifying bounds, both bounds must "
                                 "be finite")
        # build the kde
        self._kde = self.get_kde_from_arrays(*[kwargs[p] for p in self.params])
        self.set_bandwidth(bandwidth)

    @property
    def params(self):
        return self._params

    @property
    def kde(self):
        return self._kde

        ignored.
        """
        for p in self._params:
            if p not in kwargs.keys():
                raise ValueError('Missing parameter {} to construct pdf.'
                                 .format(p))
        if kwargs in self:
            # transform into the kde space
            jacobian = 1.
            for param, tparam in self._tparams.items():
                t = self._transforms[tparam]
                try:
                    samples = t.transform({param: kwargs[param]})
                except ValueError as e:
                    # can get a value error if the value is exactly == to
                    # the bounds, in which case, just return 0.
                    if kwargs[param] in self.bounds[param]:
                        return 0.
                    else:
                        raise ValueError(e)
                kwargs[param] = samples[tparam]
                # update the jacobian for the transform; if p is the pdf
                # in the params frame (the one we want) and p' is the pdf
                # in the transformed frame (the one that's calculated) then:
                # p = J * p', where J is the Jacobian of going from p to p'
                jacobian *= t.jacobian(samples)
            # for scipy < 0.15.0, gaussian_kde.pdf = gaussian_kde.evaluate
            this_pdf = jacobian * self._kde.evaluate([kwargs[p]
                                                      for p in self._params])
            if len(this_pdf) == 1:
                return float(this_pdf)
            else:
                return this_pdf
        else:
            return 0.

        arguments must contain all of parameters in self's params.
        Unrecognized arguments are ignored.
        """
        else:
            return numpy.log(self._pdf(**kwargs))

    def set_bandwidth(self, set_bw="scott"):
        self._kde.set_bandwidth(set_bw)

        """Gives a set of random values drawn from the kde.

        Returns
        -------
        """
        else:
        size = int(size)
        draws = self._kde.resample(size)
        draws = {param: draws[ii,:] for ii,param in enumerate(self.params)}
        for (param,_) in dtype:
            try:
                # transform back to param space
                tparam = self._tparams[param]
                tdraws = {tparam: draws[param]}
                draws[param] = self._transforms[tparam].inverse_transform(
                    tdraws)[param]
                pass
            arr[param] = draws[param]
        return arr

    def get_kde_from_arrays(*arrays):
        """Constructs a KDE from the given arrays.

        \*arrays :
            Each argument should be a 1D numpy array to construct the kde from.
            The resulting KDE will have dimension given by the number of
            parameters.
        """
        return scipy.stats.gaussian_kde(numpy.vstack(arrays))

        """Raises a NotImplementedError; to load from a config file, use
        `FromFile`.
        """
        raise NotImplementedError("This class does not support loading from a "
                                  "config file. Use `FromFile` instead.")

class FromFile(Arbitrary):
    r"""A distribution that reads the values of the parameter(s) from an hdf
    file, computes the kde to construct the pdf, and draws random variables
    from it.

    filename : str
        The path to an hdf file containing the values of the parameters that
        want to be used to construct the distribution. Each parameter should
        be a separate dataset in the hdf file, and all datasets should have
        the same size. For example, to give a prior for mass1 and mass2 from
        file f, f['mass1'] and f['mass2'] contain the n values for each
        parameter.
    datagroup : str, optional
        The name of the group to look in for the samples. For example, if
        ``datagroup = 'samples'``, then parameter ``param`` will be retrived
        from ``f['samples'][param]``. If none provided (the default) the data
        sets will be assumed to be in the top level directory of the file.
        The keyword arguments should provide the names of the parameters to be
        read from the file and (optionally) their bounds. If no parameters are
        provided, it will use all the parameters found in the file. To provide
        bounds, specify e.g. mass1=[10,100]. Otherwise, mass1=None.

    name : 'fromfile'
        The path to the file containing values for the parameter(s).
        Parameters to read from file.
    norm : float
        The normalization of the multi-dimensional pdf.
    lognorm : float
        The log of the normalization.
    kde :
        The kde obtained from the values in the file.
    """
    name = 'fromfile'
    def __init__(self, filename=None, datagroup=None, **params):
        if filename is None:
            raise ValueError('A file must be specified for this distribution.')
        self._filename = filename
        self.datagroup = datagroup
        # Get the parameter names to pass to get_kde_from_file
        if len(params) == 0:
            ps = None
        else:
            ps = list(params.keys())
        param_vals, bw = self.get_arrays_from_file(filename, params=ps)
        super(FromFile, self).__init__(bounds=params, bandwidth=bw,
                                       **param_vals)

    @property
    def filename(self):
        return self._filename

    def get_arrays_from_file(self, params_file, params=None):
        """Reads the values of one or more parameters from an hdf file and
        returns as a dictionary.

        params_file : str
            The hdf file that contains the values of the parameters.
        params : {None, list}
            If provided, will just retrieve the given parameter names.

        Returns
        -------
        dict
            A dictionary of the parameters mapping `param_name -> array`.
        """
        try:
            f = h5py.File(params_file, 'r')
        except:
            raise ValueError('File not found.')
        if self.datagroup is not None:
            get = f[self.datagroup]
        else:
            get = f
        if params is not None:
            if not isinstance(params, list):
                params = [params]
            for p in params:
                if p not in get.keys():
                    raise ValueError('Parameter {} is not in {}'
                                     .format(p, params_file))
        else:
            params = [str(k) for k in get.keys()]
        params_values = {p: get[p][()] for p in params}
        try:
            bandwidth = f.attrs["bandwidth"]
            bandwidth = "scott"

        f.close()
        return params_values, bandwidth

        The parameters
        for the distribution are retrieved from the section titled
        "[`section`-`variable_args`]" in the config file.

        The file to construct the distribution from must be provided by setting
        `filename`. Boundary arguments can be provided in the same way as
        described in `get_param_bounds_from_config`.

            [{section}-{tag}]
            name = fromfile
            filename = ra_prior.hdf
            min-ra = 0
            max-ra = 6.28

            options.
            `prior.VARARGS_DELIM`. These must appear in the "tag" part

        Returns
        -------
        BoundedDist
        """
                                           bounds_required=False)

__all__ = ['Arbitrary', 'FromFile']

# Copyright (C) 2016  Collin Capano, Christopher M. Biwer
#
#
"""
This modules provides classes for evaluating distributions with bounds.
"""

import warnings

#
#   Distributions for priors
#
def get_param_bounds_from_config(cp, section, tag, param):
    """Gets bounds for the given parameter from a section in a config file.

    Minimum and maximum values for bounds are specified by adding
    `min-{param}` and `max-{param}` options, where `{param}` is the name of
    the parameter. The types of boundary (open, closed, or reflected) to create
    may also be specified by adding options `btype-min-{param}` and
    `btype-max-{param}`. Cyclic conditions can be adding option
    `cyclic-{param}`. If no `btype` arguments are provided, the
    left bound will be closed and the right open.

    For example, the following will create right-open bounds for parameter
    `foo`:

        [{section}-{tag}]
        min-foo = -1
        max-foo = 1

    This would make the boundaries cyclic:

        max-foo = 1
        cyclic-foo =

    For more details on boundary types and their meaning, see
    `boundaries.Bounds`.

    If the parameter is not found in the section will just return None (in
    this case, all `btype` and `cyclic` arguments are ignored for that
    parameter).  If bounds are specified, both a minimum and maximum must be
    provided, else a Value or Type Error will be raised.

    cp : ConfigParser instance
        The config file.
    section : str
        The name of the section.
    tag : str
        Any tag in the section name. The full section name searched for in
        the config file is `{section}(-{tag})`.
    param : str
        The name of the parameter to retrieve bounds for.

    Returns
    -------
    bounds : {Bounds instance | None}
        If bounds were provided, a `boundaries.Bounds` instance
        representing the bounds. Otherwise, `None`.
    """
    try:
        minbnd = float(cp.get_opt_tag(section, 'min-'+param, tag))
    except Error:
        minbnd = None
    try:
        maxbnd = float(cp.get_opt_tag(section, 'max-'+param, tag))
        maxbnd = None
    if minbnd is None and maxbnd is None:
        bnds = None
    elif minbnd is None or maxbnd is None:
        raise ValueError("if specifying bounds for %s, " %(param) +
            "you must provide both a minimum and a maximum")
    else:
        bndargs = {'min_bound': minbnd, 'max_bound': maxbnd}
        # try to get  any other conditions, if provided
        try:
            minbtype = cp.get_opt_tag(section, 'btype-min-{}'.format(param),
                                      tag)
            minbtype = 'closed'
        try:
            maxbtype = cp.get_opt_tag(section, 'btype-max-{}'.format(param),
                                      tag)
            maxbtype = 'open'
        bndargs.update({'btype_min': minbtype, 'btype_max': maxbtype})
        cyclic = cp.has_option_tag(section, 'cyclic-{}'.format(param), tag)
        bndargs.update({'cyclic': cyclic})
        bnds = boundaries.Bounds(**bndargs)
    return bnds

def bounded_from_config(cls, cp, section, variable_args,
        bounds_required=False, additional_opts=None):
    """Returns a bounded distribution based on a configuration file. The
    parameters for the distribution are retrieved from the section titled
    "[`section`-`variable_args`]" in the config file.

    cls : pycbc.prior class
        The class to initialize with.
    cp : pycbc.workflow.WorkflowConfigParser
        A parsed configuration file that contains the distribution
        options.
        Name of the section in the configuration file.
    variable_args : str
        The names of the parameters for this distribution, separated by
        `prior.VARARGS_DELIM`. These must appear in the "tag" part
        of the section header.
    bounds_required : {False, bool}
       If True, raise a ValueError if a min and max are not provided for
       every parameter. Otherwise, the prior will be initialized with the
       parameter set to None. Even if bounds are not required, a
       ValueError will be raised if only one bound is provided; i.e.,
       either both bounds need to provided or no bounds.
    additional_opts : {None, dict}
        Provide additional options to be passed to the distribution class;
        should be a dictionary specifying option -> value. If an option is
        provided that also exists in the config file, the value provided will
        be used instead of being read from the file.

    Returns
    -------
    cls
        An instance of the given class.
    """
    tag = variable_args
    variable_args = variable_args.split(VARARGS_DELIM)

    if additional_opts is None:
        additional_opts = {}

    # list of args that are used to construct distribution
    special_args = ["name"] + \
        ['min-{}'.format(arg) for arg in variable_args] + \
        ['max-{}'.format(arg) for arg in variable_args] + \
        ['btype-min-{}'.format(arg) for arg in variable_args] + \
        ['btype-max-{}'.format(arg) for arg in variable_args] + \
        ['cyclic-{}'.format(arg) for arg in variable_args] + \
        list(additional_opts.keys())

    # get a dict with bounds as value
    dist_args = {}
    for param in variable_args:
        bounds = get_param_bounds_from_config(cp, section, tag, param)
        if bounds_required and bounds is None:
            raise ValueError("min and/or max missing for parameter %s"%(
                param))
        dist_args[param] = bounds

    # add any additional options that user put in that section
    for key in cp.options("-".join([section, tag])):

        # ignore options that are already included
        if key in special_args:
            continue

        # check if option can be cast as a float
        val = cp.get_opt_tag(section, key, tag)
        try:
            val = float(val)
            pass

        # add option
        dist_args.update({key:val})

    dist_args.update(additional_opts)

    # construction distribution and add to list
    return cls(**dist_args)

class BoundedDist(object):
    """
    A generic class for storing common properties of distributions in which
    each parameter has a minimum and maximum value.

        The keyword arguments should provide the names of parameters and their
        corresponding bounds, as either tuples or a `boundaries.Bounds`
        instance.

        A dictionary of the parameter names and their bounds.
    """
        # convert input bounds to Bounds class, if necessary
        for param,bnds in params.items():
                params[param] = boundaries.Bounds()
            elif not isinstance(bnds, boundaries.Bounds):
                params[param] = boundaries.Bounds(bnds[0], bnds[1])
            # warn the user about reflected boundaries
            if isinstance(bnds, boundaries.Bounds) and (
                    bnds.min.name == 'reflected' or
                    bnds.max.name == 'reflected'):
                warnings.warn("Param {} has one or more ".format(param) +
                              "reflected boundaries. Reflected boundaries "
                              "can cause issues when used in an MCMC.")
        self._bounds = params
        self._params = sorted(list(params.keys()))

    @property

    @property

    def __contains__(self, params):
        try:
            return all(self._bounds[p].contains_conditioned(params[p])
                       for p in self._params)
            raise ValueError("must provide all parameters [%s]" %(
                ', '.join(self._params)))

        """Applies any boundary conditions to the given values (e.g., applying
        cyclic conditions, and/or reflecting values off of boundaries). This
        is done by running `apply_conditions` of each bounds in self on the
        corresponding value. See `boundaries.Bounds.apply_conditions` for
        details.

            all of the parameters in self. Any unrecognized arguments are
            ignored.

        Returns
        -------
        dict
        """
        return dict([[p, self._bounds[p].apply_conditions(val)]
                     for p,val in kwargs.items() if p in self._bounds])

    def pdf(self, **kwargs):
        ignored. Any boundary conditions are applied to the values before the
        pdf is evaluated.
        """
        return self._pdf(**self.apply_boundary_conditions(**kwargs))

        """The underlying pdf function called by `self.pdf`. This must be set
        by any class that inherits from this class. Otherwise, a
        `NotImplementedError` is raised.
        """
        raise NotImplementedError("pdf function not set")

    def logpdf(self, **kwargs):
        Unrecognized arguments are ignored. Any boundary conditions are
        applied to the values before the pdf is evaluated.
        """
        return self._logpdf(**self.apply_boundary_conditions(**kwargs))

        """The underlying log pdf function called by `self.logpdf`. This must
        be set by any class that inherits from this class. Otherwise, a
        """

    __call__ = logpdf

        """Return the cdfinv for a single given parameter """
        raise NotImplementedError("inverse cdf not set")

    def cdfinv(self, **kwds):
        """Return the inverse cdf to map the unit interval to parameter bounds.
        You must provide a keyword for every parameter.
        """
        updated = {}
        for param in self.params:
            updated[param] = self._cdfinv_param(param, kwds[param])
        return updated

    def from_config(cls, cp, section, variable_args, bounds_required=False):
        """Returns a distribution based on a configuration file. The parameters

            options.
        bounds_required : {False, bool}
           If True, raise a ValueError if a min and max are not provided for
           every parameter. Otherwise, the prior will be initialized with the
           parameter set to None. Even if bounds are not required, a
           ValueError will be raised if only one bound is provided; i.e.,
           either both bounds need to provided or no bounds.

        Returns
        -------
            A distribution instance from the pycbc.distribution subpackage.
        """
        return bounded_from_config(cls, cp, section, variable_args,
                                    bounds_required=bounds_required)

# Copyright (C) 2017 Christopher M. Biwer
#
#
"""
This modules provides classes for evaluating multi-dimensional constraints.
"""

import scipy.spatial
from pycbc import transforms

class Constraint(object):
    """Creates a constraint that evaluates to True if parameters obey
    the constraint and False if they do not.
    """

    def __init__(self, constraint_arg, transforms=None, **kwargs):
        self.constraint_arg = constraint_arg
        self.transforms = transforms
        for kwarg in kwargs.keys():
            setattr(self, kwarg, kwargs[kwarg])

    def __call__(self, params):
        """Evaluates constraint.
        """
        # cast to FieldArray
        if isinstance(params, dict):
            params = record.FieldArray.from_kwargs(**params)
        elif not isinstance(params, record.FieldArray):
           raise ValueError("params must be dict or FieldArray instance")

        # try to evaluate; this will assume that all of the needed parameters
        # for the constraint exists in params
        try:
            out = self._constraint(params)
        except NameError:
            # one or more needed parameters don't exist; try applying the
            # transforms
            params = transforms.apply_transforms(params, self.transforms) \
                     if self.transforms else params
        if isinstance(out, record.FieldArray):
            out = out.item() if params.size == 1 else out
        return out

    def _constraint(self, params):
        """ Evaluates constraint function.
        """
        return params[self.constraint_arg]

class SupernovaeConvexHull(Constraint):
    """Pre defined constraint for core-collapse waveforms that checks
    whether a given set of coefficients lie within the convex hull of
    the coefficients of the principal component basis vectors.
    """
    name = "supernovae_convex_hull"
    required_parameters = ["coeff_0", "coeff_1"]

        super(SupernovaeConvexHull,
              self).__init__(constraint_arg, transforms=transforms, **kwargs)

        if 'principal_components_file' in kwargs:
            pc_filename = kwargs['principal_components_file']
            hull_dimention = numpy.array(kwargs['hull_dimention'])
            self.hull_dimention = int(hull_dimention)
            pc_file = h5py.File(pc_filename, 'r')
            pc_coefficients = numpy.array(pc_file.get('coefficients'))
            pc_file.close()
            hull_points = []
            for dim in range(self.hull_dimention):
                hull_points.append(pc_coefficients[:, dim])
            hull_points = numpy.array(hull_points).T
            pc_coeffs_hull = scipy.spatial.Delaunay(hull_points)
            self._hull = pc_coeffs_hull

        output_array = []
        points = numpy.array([params["coeff_0"],
                              params["coeff_1"],
                              params["coeff_2"]])
        for coeff_index in range(len(params["coeff_0"])):
            point = points[:, coeff_index][:self.hull_dimention]
            output_array.append(self._hull.find_simplex(point) >= 0)
        return numpy.array(output_array)

# list of all constraints
constraints = {
    Constraint.name : Constraint,
    SupernovaeConvexHull.name : SupernovaeConvexHull,
}

# Copyright (C) 2020 Alexander Nitz
#
#
"""
This modules provides classes for evaluating Gaussian distributions.
"""

import importlib
import numpy.random

class External(object):
    """ Distribution defined by external cdfinv and logpdf functions

    To add to an inference configuration file:

        [prior-param1+param2]
        module = custom_mod
        logpdf = custom_function_name
        cdfinv = custom_function_name2

        list of parameter names
    logpdf : function
        function which returns the logpdf
    cdfinv : function
        function which applies the invcdf

    Examples
    --------
    To instantate by hand and example of function format. You must provide
    the logpdf function, and you may either provide the rvs or cdfinv function.
    If the cdfinv is provided, but not the rvs, the random values will
    be calculated using the cdfinv function.

    >>> import numpy
    >>> params = ['x', 'y']
    >>> def logpdf(x=None, y=None):
    ...     p = numpy.ones(len(x))
    ...     return p
    >>>
    >>> def cdfinv(**kwds):
    ...     return kwds
    >>> e = External(['x', 'y'], logpdf, cdfinv=cdfinv)
    >>> e.rvs(size=10)
    """
    name = "external"

    def __init__(self, params, logpdf, rvs=None, cdfinv=None):
        self.params = params
        self.logpdf = logpdf
        self.cdfinv = cdfinv
        self._rvs = rvs

        if not (rvs or cdfinv):
            raise ValueError("Must provide either rvs or cdfinv")

    def rvs(self, size=1, **kwds):
        "Draw random value"
        if self._rvs:
            return self._rvs(size=size)

        draw = {}
            draw[param] = numpy.random.uniform(0, 1, size=size)
        return self.cdfinv(**draw)

    def apply_boundary_conditions(self, **params):
        return params

    def __call__(self, **kwds):
        return self.logpdf(**kwds)

        params = variable_args.split(VARARGS_DELIM)
        modulestr = cp.get_opt_tag(section, 'module', tag)

        mod = importlib.import_module(modulestr)
        logpdfstr = cp.get_opt_tag(section, 'logpdf', tag)
        logpdf = getattr(mod, logpdfstr)

        cdfinv = rvs = None
        if cp.has_option_tag(section, 'cdfinv', tag):
            cdfinvstr = cp.get_opt_tag(section, 'cdfinv', tag)
            cdfinv = getattr(mod, cdfinvstr)

        if cp.has_option_tag(section, 'rvs', tag):
            rvsstr = cp.get_opt_tag(section, 'rvs', tag)
            rvs = getattr(mod, rvsstr)

        return cls(params, logpdf, rvs=rvs, cdfinv=cdfinv)

__all__ = ['External']

#
#
"""
This modules provides classes for evaluating distributions based on a fixed
set of points
"""

class FixedSamples(object):
    """
    A distribution consisting of a collection of a large number of fixed points.
    Only these values can be drawn from, so the number of points may need to be
    large to properly reflect the paramter space. This distribution is intended
    to aid in using nested samplers for semi-abitrary or complicated
    distributions where it is possible to provide or draw samples but less
    straightforward to provide an analytic invcdf. This class numerically
    approximates the invcdf for 1 or 2 dimensional distributions
    (but no higher).

    params :
        This of parameters this distribution should use
    samples : dict of arrays or FieldArray
        Sampled points of the distribution. May contain transformed parameters
        which are different from the original distribution. If so, an inverse
        mapping is provided to associate points with other parameters provided.
    """

    name = "fixed_samples"

    def __init__(self, params, samples):
        self.samples = samples

        self.p1 = self.samples[params[0]]
        self.frac = len(self.p1)**0.5 / len(self.p1)
        self.sort = self.p1.argsort()
        self.p1sorted = self.p1[self.sort]

        assert len(numpy.unique(self.p1)) == len(self.p1)

        if len(params) > 2:
            raise ValueError("Only one or two parameters supported "
                             "for fixed sample distribution")

        i = numpy.random.randint(0, high=len(self.p1), size=size)
        return {p: self.samples[p][i] for p in self.params}

    def cdfinv(self, **original):
        """Map unit cube to parameters in the space"""
        new = {}

        #First dimension
        u1 = original[self.params[0]]
        i1 = int(round(u1 * len(self.p1)))
        if i1 >= len(self.p1):
            i1 = len(self.p1) - 1
        if i1 < 0:
            i1 = 0
        new[self.params[0]] = p1v = self.p1sorted[i1]
        if len(self.params) == 1:
            return new

        # possible second dimension, probably shouldn't
        # do more dimensions than this
        u2 = original[self.params[1]]
        l = numpy.searchsorted(self.p1sorted, p1v * (1 - self.frac))
        r = numpy.searchsorted(self.p1sorted, p1v * (1 + self.frac))
        if r < l:
            l, r = r, l

        region = numpy.array(self.sort[l:r], ndmin=1)
        p2 = self.samples[self.params[1]]
        p2part = numpy.array(p2[region], ndmin=1)
        l = p2part.argsort()
        p2part = numpy.array(p2part[l], ndmin=1)

        i2 = int(round(u2 * len(p2part)))
        if i2 >= len(p2part):
            i2 = len(p2part) - 1
        if i2 < 0:
            i2 = 0
        new[self.params[1]] = p2part[i2]

        p1part = numpy.array(self.p1[region[l]], ndmin=1)
        new[self.params[0]] = p1part[i2]
        return new

        """ Apply boundary conditions (none here) """

        """ Dummy function, not the actual pdf """
        return 0

    def from_config(cls, cp, section, tag):
        """ Return instance based on config file

        Return a new instance based on the config file. This will draw from
        a single distribution section provided in the config file and
        apply a single transformation section if desired. If a transformation
        is applied, an inverse mapping is also provided for use in the config
        file.
        """
        from pycbc.distributions import read_distributions_from_config
        from pycbc.transforms import (read_transforms_from_config,
                                      apply_transforms, BaseTransform)
        from pycbc.transforms import transforms as global_transforms

        params = tag.split(VARARGS_DELIM)
        subname = cp.get_opt_tag(section, 'subname', tag)
        size = cp.get_opt_tag(section, 'sample-size', tag)

        distsec = '{}_sample'.format(subname)
        dist = read_distributions_from_config(cp, section=distsec)
        if len(dist) > 1:
            raise ValueError("Fixed sample distrubtion only supports a single"
                             " distribution to sample from.")

        logging.info('Drawing samples for fixed sample distribution:%s', params)
        samples = dist[0].rvs(size=int(float(size)))
        samples = {p: samples[p] for p in samples.dtype.names}

        transec = '{}_transform'.format(subname)
        trans = read_transforms_from_config(cp, section=transec)
        if len(trans) > 0:
            trans = trans[0]
            samples = apply_transforms(samples, [trans])
            p1 = samples[params[0]]

            # We have transformed parameters, so automatically provide the
            # inverse transform for use in passing to waveform approximants
            class Thook(BaseTransform):
                name = subname
                _inputs = trans.outputs
                _outputs = trans.inputs
                p1name = params[0]
                sort = p1.argsort()
                p1sorted = p1[sort]
                def transform(self, maps):
                    idx = numpy.searchsorted(self.p1sorted, maps[self.p1name])
                    out = {p: samples[p][self.sort[idx]] for p in self.outputs}
                    return self.format_output(maps, out)
            global_transforms[Thook.name] = Thook
        return cls(params, samples)

__all__ = ['FixedSamples']

# Copyright (C) 2016  Christopher M. Biwer, Collin Capano
#
#
"""
"""

from scipy.special import erf, erfinv

class Gaussian(bounded.BoundedDist):
    r"""A Gaussian distribution on the given parameters; the parameters are
    independent of each other.

    Bounds can be provided on each parameter, in which case the distribution
    will be a truncated Gaussian distribution.  The PDF of a truncated
    Gaussian distribution is given by:

    .. math::
        p(x|a, b, \mu,\sigma) = \frac{1}{\sqrt{2 \pi \sigma^2}}\frac{e^{- \frac{\left( x - \mu \right)^2}{2 \sigma^2}}}{\Phi(b|\mu, \sigma) - \Phi(a|\mu, \sigma)},

    where :math:`\mu` is the mean, :math:`\sigma^2` is the variance,
    :math:`a,b` are the bounds, and :math:`\Phi` is the cumulative distribution
    of an unbounded normal distribution, given by:

    .. math::
        \Phi(x|\mu, \sigma) = \frac{1}{2}\left[1 + \mathrm{erf}\left(\frac{x-\mu}{\sigma \sqrt{2}}\right)\right].

    Note that if :math:`[a,b) = [-\infty, \infty)`, this reduces to a standard
    Gaussian distribution.

    Instances of this class can be called like a function. By default, logpdf
    will be called, but this can be changed by setting the class's __call__
    method to its pdf method.

        (optionally) some bounds, as either a tuple or a
        `boundaries.Bounds` instance. The mean and variance of each
        parameter can be provided by additional keyword arguments that have
        `_mean` and `_var` adding to the parameter name. For example,
        `foo=(-2,10), foo_mean=3, foo_var=2` would create a truncated Gaussian
        with mean 3 and variance 2, bounded between :math:`[-2, 10)`. If no
        mean or variance is provided, the distribution will have 0 mean and
        unit variance. If None is provided for the bounds, the distribution
        will be a normal, unbounded Gaussian (equivalent to setting the bounds
        to `[-inf, inf)`).

    name : 'guassian'

    Examples
    --------
    Create an unbounded Gaussian distribution with zero mean and unit variance:
    >>> dist = distributions.Gaussian(mass1=None)

    Create a bounded Gaussian distribution on :math:`[1,10)` with a mean of 3
    and a variance of 2:
    >>> dist = distributions.Gaussian(mass1=(1,10), mass1_mean=3, mass1_var=2)

    Create a bounded Gaussian distribution with the same parameters, but with
    cyclic boundary conditions:
    >>> dist = distributions.Gaussian(mass1=Bounds(1,10, cyclic=True), mass1_mean=3, mass1_var=2)
    """
    name = "gaussian"

        # save distribution parameters as dict
        # calculate the norm and exponential norm ahead of time
        # and save to self._norm, self._lognorm, and self._expnorm
        self._bounds = {}
        self._mean = {}
        self._var = {}
        self._norm = {}
        self._lognorm = {}
        self._expnorm = {}
        # pull out specified means, variance
        mean_args = [p for p in params if p.endswith('_mean')]
        var_args = [p for p in params if p.endswith('_var')]
        self._mean = dict([[p[:-5], params.pop(p)] for p in mean_args])
        self._var = dict([[p[:-4], params.pop(p)] for p in var_args])
        super(Gaussian, self).__init__(**params)

        # check that there are no params in mean/var that are not in params
        missing = set(self._mean.keys()) - set(params.keys())
        if any(missing):
            raise ValueError("means provided for unknow params {}".format(
                ', '.join(missing)))
        missing = set(self._var.keys()) - set(params.keys())
            raise ValueError("vars provided for unknow params {}".format(
        # set default mean/var for params not specified
        self._mean.update(dict([[p, 0.]
            for p in params if p not in self._mean]))
        self._var.update(dict([[p, 1.]
            for p in params if p not in self._var]))

        # compute norms
        for p,bnds in self._bounds.items():
            sigmasq = self._var[p]
            mu = self._mean[p]
            a,b = bnds
            invnorm = scipy.stats.norm.cdf(b, loc=mu, scale=sigmasq**0.5) \
                    - scipy.stats.norm.cdf(a, loc=mu, scale=sigmasq**0.5)
            invnorm *= numpy.sqrt(2*numpy.pi*sigmasq)
            self._norm[p] = 1./invnorm
            self._lognorm[p] = numpy.log(self._norm[p])
            self._expnorm[p] = -1./(2*sigmasq)

    @property
    def mean(self):
        return self._mean

    @property
    def var(self):
        return self._var

    def _normalcdf(self, param, value):
        """The CDF of the normal distribution, without bounds."""
        mu = self._mean[param]
        var = self._var[param]
        return 0.5*(1. + erf((value - mu)/(2*var)**0.5))

    def cdf(self, param, value):
        """Returns the CDF of the given parameter value."""
        a, b = self._bounds[param]
        if a != -numpy.inf:
            phi_a = self._normalcdf(param, a)
        else:
            phi_a = 0.
        if b != numpy.inf:
            phi_b = self._normalcdf(param, b)
        else:
            phi_b = 1.
        phi_x = self._normalcdf(param, value)
        return (phi_x - phi_a)/(phi_b - phi_a)

    def _normalcdfinv(self, param, p):
        """The inverse CDF of the normal distribution, without bounds."""
        return mu + (2*var)**0.5 * erfinv(2*p - 1.)

    def _cdfinv_param(self, param, p):
        """Return inverse of the CDF.
        """
        else:
            phi_a = 0.
        else:
            phi_b = 1.
        adjusted_p = phi_a + p * (phi_b - phi_a)
        return self._normalcdfinv(param, adjusted_p)

        ignored.
        """
        return numpy.exp(self._logpdf(**kwargs))

        """
            return sum([self._lognorm[p] +
                        self._expnorm[p]*(kwargs[p]-self._mean[p])**2.
                        for p in self._params])
        else:

        Returns
        -------
        """
        else:
            sigma = numpy.sqrt(self._var[p])
            a,b = self._bounds[p]
            arr[p][:] = scipy.stats.truncnorm.rvs((a-mu)/sigma, (b-mu)/sigma,
                loc=self._mean[p], scale=sigma, size=size)
        return arr

        """Returns a Gaussian distribution based on a configuration file. The
        parameters for the distribution are retrieved from the section titled

        Boundary arguments should be provided in the same way as described in
        `get_param_bounds_from_config`. In addition, the mean and variance of
        each parameter can be specified by setting `{param}_mean` and
        `{param}_var`, respectively. For example, the following would create a
        truncated Gaussian distribution between 0 and 6.28 for a parameter
        called `phi` with mean 3.14 and variance 0.5 that is cyclic:

            min-phi = 0
            max-phi = 6.28
            phi_mean = 3.14
            phi_var = 0.5
            cyclic =

            options.

        Returns
        -------
        Gaussain
        """
                                                  bounds_required=False)

__all__ = ['Gaussian']

# Copyright (C)  2017 Collin Capano, Christopher M. Biwer, Alex Nitz
#
#
""" This module provides classes to describe joint distributions
"""
from pycbc.io.record import FieldArray

class JointDistribution(object):
    """
    Callable class that calculates the joint distribution built from a set of
    distributions.

        A list of strings that contain the names of the variable parameters and
        the order they are expected when the class is called.
    \*distributions :
        The rest of the arguments must be instances of distributions describing
        the individual distributions on the variable parameters.
        A single distribution may contain
        multiple parameters. The set of all params across the distributions
        (retrieved from the distributions' params attribute) must be the same
        as the set of variable_args provided.
    \*\*kwargs :
        Valid keyword arguments include:
        `constraints` : a list of functions that accept a dict of parameters
        with the parameter name as the key. If the constraint is satisfied the
        function should return True, if the constraint is violated, then the
        function should return False.
        `n_test_samples` : number of random draws used to fix pdf normalization
        factor after applying constraints.

    variable_args : tuple
        The parameters expected when the evaluator is called.
    distributions : list
        The distributions for the parameters.
    constraints : list
        A list of functions to test if parameter values obey multi-dimensional
        constraints.

    Examples
    --------
    An example of creating a joint distribution with constraint that total mass must
    be below 30.

    >>> from pycbc.distributions import Uniform, JointDistribution
    >>> def mtotal_lt_30(params):
        ...    return params["mass1"] + params["mass2"] < 30
    >>> mass_lim = (2, 50)
    >>> uniform_prior = Uniform(mass1=mass_lim, mass2=mass_lim)
    >>> prior_eval = JointDistribution(["mass1", "mass2"], uniform_prior,
        ...                               constraints=[mtotal_lt_30])
    >>> print(prior_eval(mass1=20, mass2=1))

    """
    name = 'joint'

    def __init__(self, variable_args, *distributions, **kwargs):

        # store the names of the parameters defined in the distributions
        self.variable_args = tuple(variable_args)

        # store the distributions
        self.distributions = distributions

        # store the constraints on the parameters defined inside the
        # distributions list
        self._constraints = kwargs["constraints"] \
                                  if "constraints" in kwargs.keys() else []

        # check that all of the supplied parameters are described by the given
        # distributions
        distparams = set()
        for dist in distributions:
            distparams.update(set(dist.params))

        varset = set(self.variable_args)
        missing_params = distparams - varset
        if missing_params:
            raise ValueError("provided variable_args do not include "
                "parameters %s" %(','.join(missing_params)) + " which are "
                "required by the provided distributions")
        extra_params = varset - distparams
        if extra_params:
            raise ValueError("variable_args %s " %(','.join(extra_params)) +
                "are not in any of the provided distributions")

        # if there are constraints then find the renormalization factor
        # since a constraint will cut out part of the space
        # do this by random sampling the full space and find the percent
        # of samples rejected
        n_test_samples = kwargs["n_test_samples"] \
                             if "n_test_samples" in kwargs else int(1e6)
        if self._constraints:
            logging.info("Renormalizing distribution for constraints")

            # draw samples
            samples = {}
            for dist in self.distributions:
                draw = dist.rvs(n_test_samples)
                for param in dist.params:
                    samples[param] = draw[param]
            samples = FieldArray.from_kwargs(**samples)

            # evaluate constraints
            result = self.contains(samples)

            # set new scaling factor for prior to be
            # the fraction of acceptances in random sampling of entire space
            self._pdf_scale = result.sum() / float(n_test_samples)
            if self._pdf_scale == 0.0:
                raise ValueError("None of the random draws for pdf "
                    "renormalization satisfied the constraints. "
                    " You can try increasing the 'n_test_samples' keyword.")

        else:
            self._pdf_scale = 1.0

        # since Distributions will return logpdf we keep the scale factor
        # in log scale as well for self.__call__
        self._logpdf_scale = numpy.log(self._pdf_scale)

        """Applies each distributions' boundary conditions to the given list
        of parameters, returning a new list with the conditions applied.

        **params :
            Keyword arguments should give the parameters to apply the
            conditions to.

        Returns
        -------
        dict
            A dictionary of the parameters after each distribution's
            `apply_boundary_conditions` function has been applied.
        """
        for dist in self.distributions:
            params.update(dist.apply_boundary_conditions(**params))

    def _ensure_fieldarray(params):
        """Ensures the given params are a ``FieldArray``.

        params : dict, FieldArray, numpy.record, or numpy.ndarray
            If the given object is a dict, it will be converted to a
            FieldArray.

        Returns
        -------
        params : FieldArray
            The given values as a FieldArray.
        return_atomic : bool
            Whether or not functions run on the parameters should be returned
            as atomic types or not.
        """
            return_atomic = not any(isinstance(val, numpy.ndarray)
                                    for val in params.values())
            params = FieldArray.from_kwargs(**params)
        elif isinstance(params, numpy.record):
            return_atomic = True
            params = FieldArray.from_records(tuple(params),
                                             names=params.dtype.names)
        elif isinstance(params, numpy.ndarray):
            return_atomic = False
            params = params.view(type=FieldArray)
        elif isinstance(params, FieldArray):
        else:
            raise ValueError("params must be either dict, FieldArray, "
                             "record, or structured array")
        return params, return_atomic

    def contains(self, params):
        """Evaluates whether the given parameters satisfy the constraints.

            The parameter values to evaluate.

        Returns
        -------
        (array of) bool :
            If params was an array, or if params a dictionary and one or more
            of the parameters are arrays, will return an array of booleans.
            Otherwise, a boolean.
        """
        params, return_atomic = self._ensure_fieldarray(params)
        # convert params to a field array if it isn't one
        result = numpy.ones(params.shape, dtype=bool)
        for constraint in self._constraints:
            result &= constraint(params)
        if return_atomic:
            result = result.item()
        return result

    def __call__(self, **params):
        """Evaluate joint distribution for parameters.
        """
        # check if statisfies constraints
        if len(self._constraints) != 0:
            parray, return_atomic = self._ensure_fieldarray(params)
            isin = self.contains(parray)
            if not isin.any():
                if return_atomic:
                    out = -numpy.inf
                else:
                    out = numpy.full(parray.shape, -numpy.inf)
                return out

        # evaluate
        # note: this step may fail if arrays of values were provided, as
        # not all distributions are vectorized currently
        logps = numpy.array([d(**params) for d in self.distributions])
        logp = logps.sum(axis=0)

            logp += numpy.log(isin.astype(float))

        if numpy.isscalar(logp):
            logp = logp.item()

        return logp - self._logpdf_scale

    def rvs(self, size=1):
        """ Rejection samples the parameter space.
        """
        # create output FieldArray
        dtype = [(arg, float) for arg in self.variable_args]
        out = FieldArray(size, dtype=dtype)
        # loop until enough samples accepted
        remaining = size
        ndraw = size
        while remaining:
            # scratch space for evaluating constraints
            scratch = FieldArray(ndraw, dtype=dtype)
                # drawing samples from the distributions is generally faster
                # then evaluating constrants, so we'll always draw the full
                # size, even if that gives us more points than we need
                draw = dist.rvs(size=ndraw)
                    scratch[param] = draw[param]
            # apply any constraints
            keep = self.contains(scratch)
            nkeep = keep.sum()
            kmin = size - remaining
            kmax = min(nkeep, remaining)
            out[kmin:kmin+kmax] = scratch[keep][:kmax]
            remaining = max(0, remaining - nkeep)
            # to try to speed up next go around, we'll increase the draw
            # size by the fraction of values that were kept, but cap at 1e6
            ndraw = int(min(1e6, ndraw * numpy.ceil(ndraw / (nkeep + 1.))))
        return out

    @property
    def well_reflected(self):
        """ Get list of which parameters are well reflected
        """
        reflect = []
        bounds = self.bounds
        for param in bounds:
            if bounds[param].reflected == 'well':
                reflect.append(param)
        return reflect

    @property
        """ Get list of which parameters are cyclic
        """
        cyclic = []
            if bounds[param].cyclic:
                cyclic.append(param)
        return cyclic

    @property
        """ Get the dict of boundaries
        """
        bnds = {}
            if hasattr(dist, 'bounds'):
                bnds.update(dist.bounds)
        return bnds

        """ Apply the inverse cdf to the array of values [0, 1]. Every
        variable parameter must be given as a keyword argument.
        """
            updated.update(dist.cdfinv(**original))

# Copyright (C) 2021 Yifan Wang
#
#

"""This modules provides classes for evaluating distributions for mchirp and 
q (i.e., mass ratio) from uniform component mass.
"""

from scipy.interpolate import interp1d
from scipy.special import hyp2f1
from pycbc.distributions import power_law

class MchirpfromUniformMass1Mass2(power_law.UniformPowerLaw):
    r""" A distribution for chirp mass from uniform component mass + 
    constraints given by chirp mass. This is a special case for UniformPowerLaw 
    with index 1. For more details see UniformPowerLaw.
    
    The parameters are independent of each other. Instances of this class can 
    be called like a function. By default, logpdf will be called, but this can 
    be changed by setting the class's __call__ method to its pdf method.

    Since

    .. math::

        P(m_1,m_2)dm_1dm_2 = P(\mathcal{M}_c,q)d\mathcal{M}_cdq

    Where :math:`\mathcal{M}_c` is chirp mass and math:`q` is mass ratio,
    math:`m_1` and math:`m_2` are component masses. The jacobian to transform 
    chirp mass and mass ratio to component masses is 

    .. math::

        \frac{\partial(m_1,m_2)}{\partial(\mathcal{M}_c,q)} = \
        \mathcal{M}_c \left(\frac{1+q}{q^3}\right)^{2/5} 

    (cf:https://github.com/gwastro/pycbc/blob/master/pycbc/transforms.py#L416.)

    Because :math:`P(m_1,m_2) = const', then 
    :math:`P(\mathcal{M}_c,q) =
    P(\mathcal{M}_c)P(q) \propto 
    \mathcal{M}_c \left(\frac{1+q}{q^3}\right)^{2/5}`.
    Therefore, 

    .. math::
        P(\mathcal{M}_c) \propto \mathcal{M}_c

    and 

    .. math::
        P(q) \propto \left(\frac{1+q}{q^3}\right)^{2/5}

        instance.

    name : 'mchirp_from_uniform_mass1_mass2'

    Examples
    --------

    Generate 10000 random numbers from this distribution in [5,100]

    >>> from pycbc import distributions as dist
    >>> minmc = 5, maxmc = 100, size = 10000
    >>> mc = dist.MchirpfromUniformMass1Mass2(value=(minmc,maxmc)).rvs(size)

    The settings in the configuration file for pycbc_inference should be

    [variable_params]
    mchirp =
    [prior-mchirp]
    name = mchirp_from_uniform_mass1_mass2
    min-mchirp = 10
    max-mchirp = 80

    """

    name = "mchirp_from_uniform_mass1_mass2"

    def __init__(self, dim=2, **params):
        super(MchirpfromUniformMass1Mass2, self).__init__(dim=2, **params)

class QfromUniformMass1Mass2(bounded.BoundedDist):
    """A distribution for mass ratio (i.e., q) from uniform component mass
    + constraints given by q.

    The parameters are independent of each other. Instances of this class can
    be called like a function. By default, logpdf will be called, but this can

    For mathematical derivation see the documentation in class
    "MchirpfromUniformMass1Mass2".

        instance.

    name : 'q_from_uniform_mass1_mass2'

    Examples
    --------

    Generate 10000 random numbers from this distribution in [1,8]

    >>> minq = 1, maxq = 8, size = 10000
    >>> q = dist.QfromUniformMass1Mass2(value=(minq,maxq)).rvs(size)

    q = 
    [prior-q]
    name = q_from_uniform_mass1_mass2
    min-q = 1
    max-q = 8

    """

    name = 'q_from_uniform_mass1_mass2'

        super(QfromUniformMass1Mass2, self).__init__(**params)
        self._norm = 1.0
        self._lognorm = 0.0
            self._norm /= self._cdf_param(p, self._bounds[p][1]) - \
                self._cdf_param(p, self._bounds[p][0])
            self._lognorm += numpy.log(self._norm)

    @property
    def norm(self):
        return self._norm

    @property
    def lognorm(self):
        return self._lognorm

        ignored.
        """
                raise ValueError(
                    'Missing parameter {} to construct pdf.'.format(p))
            pdf = self._norm * \
                numpy.prod([(1.+kwargs[p])**(2./5)/kwargs[p]**(6./5)
                            for p in self._params])
            return float(pdf)
        else:
            return 0.0

        """
                    'Missing parameter {} to construct logpdf.'.format(p))
        else:

    def _cdf_param(self, param, value):
        """>>> from sympy import *
           >>> x = Symbol('x')
           >>> integrate((1+x)**(2/5)/x**(6/5))
           Output:
                             _                        
                      -0.2  |_  /-0.4, -0.2 |    I*pi\
                -5.0*x    * |   |           | x*e    |
                           2  1 \   0.8     |        /
        """
        if param in self._params:
            return -5. * value**(-1./5) * hyp2f1(-2./5, -1./5, 4./5, -value)
        else:
            raise ValueError('{} is not contructed yet.'.format(param))

        """
            lower_bound = self._bounds[param][0]
            upper_bound = self._bounds[param][1]
            q_array = numpy.linspace(lower_bound, upper_bound, 1000)
            q_invcdf_interp = interp1d(self._cdf_param(param, q_array), q_array,
                                       kind='cubic', bounds_error=False,
                                       fill_value=(lower_bound, upper_bound))
            return q_invcdf_interp(value)
        else:

        Returns
        -------
        """
        else:
        for (p, _) in dtype:
            uniformcdf = numpy.random.uniform(self._cdf_param(p, self._bounds[p][0]),
                                              self._cdf_param(
                                                  p, self._bounds[p][1]),
                                              size=size)
            arr[p] = self._cdfinv_param(p, uniformcdf)
        return arr

            options.

        Returns
        -------
        Uniform
        """
        return super(QfromUniformMass1Mass2, cls).from_config(cp, section,
                                                              variable_args,
                                                              bounds_required=True)

#
#
"""
This modules provides classes for evaluating distributions where the
probability density function is a power law.
"""

class UniformPowerLaw(bounded.BoundedDist):
    r"""
    For a uniform distribution in power law. The parameters are
    independent of each other. Instances of this class can be called like
    a function. By default, logpdf will be called, but this can be changed
    by setting the class's __call__ method to its pdf method.

    The cumulative distribution function (CDF) will be the ratio of volumes:

    .. math::

        F(r) = \frac{V(r)}{V(R)}

    Where :math:`R` is the radius of the sphere. So we can write our
    probability density function (PDF) as:

    .. math::

        f(r) = c r^n

    For generality we use :math:`n` for the dimension of the volume element,
    eg. :math:`n=2` for a 3-dimensional sphere. And use
    :math:`c` as a general constant.

    So now we calculate the CDF in general for this type of PDF:

    .. math::

        F(r) = \int f(r) dr = \int c r^n dr = \frac{1}{n + 1} c r^{n + 1} + k

    Now with the definition of the CDF at radius :math:`r_{l}` is equal to 0
    and at radius :math:`r_{h}` is equal to 1 we find that the constant from
    integration from this system of equations:

    .. math::

        1 = \frac{1}{n + 1} c ((r_{h})^{n + 1} - (r_{l})^{n + 1}) + k

    Can see that :math:`c = (n + 1) / ((r_{h})^{n + 1} - (r_{l})^{n + 1}))`.
    And :math:`k` is:

    .. math::

        k = - \frac{r_{l}^{n + 1}}{(r_{h})^{n + 1} - (r_{l})^{n + 1}}

    Can see that :math:`c= \frac{n + 1}{R^{n + 1}}`. So can see that the CDF is:

    .. math::

        F(r) = \frac{1}{(r_{h})^{n + 1} - (r_{l})^{n + 1}} r^{n + 1} - \frac{r_{l}^{n + 1}}{(r_{h})^{n + 1} - (r_{l})^{n + 1}}

    And the PDF is the derivative of the CDF:

    .. math::

        f(r) = \frac{(n + 1)}{(r_{h})^{n + 1} - (r_{l})^{n + 1}} (r)^n

    Now we use the probabilty integral transform method to get sampling on
    uniform numbers from a continuous random variable. To do this we find
    the inverse of the CDF evaluated for uniform numbers:

    .. math::

        F(r) = u = \frac{1}{(r_{h})^{n + 1} - (r_{l})^{n + 1}} r^{n + 1} - \frac{r_{l}^{n + 1}}{(r_{h})^{n + 1} - (r_{l})^{n + 1}}

    And find :math:`F^{-1}(u)` gives:

    .. math::

        u = \frac{1}{n + 1} \frac{(r_{h})^{n + 1} - (r_{l})^{n + 1}} - \frac{r_{l}^{n + 1}}{(r_{h})^{n + 1} - (r_{l})^{n + 1}}

    And solving for :math:`r` gives:

    .. math::

        r = ( ((r_{h})^{n + 1} - (r_{l})^{n + 1}) u + (r_{l})^{n + 1})^{\frac{1}{n + 1}}

    Therefore the radius can be sampled by taking the n-th root of uniform
    numbers and multiplying by the radius offset by the lower bound radius.

        instance.

    name : 'uniform_radius'
    dim : int
        The dimension of volume space. In the notation above `dim`
        is :math:`n+1`. For a 3-dimensional sphere this is 3.

    """
    name = "uniform_power_law"
    def __init__(self, dim=None, **params):
        super(UniformPowerLaw, self).__init__(**params)
        self.dim = dim
            self._norm *= self.dim  / \
                                   (self._bounds[p][1]**(self.dim) -
                                    self._bounds[p][0]**(self.dim))

    @property

    @property

        Returns
        -------
        """
        else:
            offset = numpy.power(self._bounds[p][0], self.dim)
            factor = numpy.power(self._bounds[p][1], self.dim) - \
                                      numpy.power(self._bounds[p][0], self.dim)
            arr[p] = numpy.random.uniform(0.0, 1.0, size=size)
            arr[p] = numpy.power(factor * arr[p] + offset, 1.0 / self.dim)
        return arr

        """Return inverse of cdf to map unit interval to parameter bounds.
        """
        n = self.dim - 1
        r_l = self._bounds[param][0]
        r_h = self._bounds[param][1]
        new_value = ((r_h**(n+1) - r_l**(n+1))*value + r_l**(n+1))**(1./(n+1))

        ignored.
        """
                            'Missing parameter {} to construct pdf.'.format(p))
                  numpy.prod([(kwargs[p])**(self.dim - 1)
                              for p in self._params])
        else:
            return 0.0

        """
            log_pdf = self._lognorm + \
                      (self.dim - 1) * \
                      numpy.log([kwargs[p] for p in self._params]).sum()
            return log_pdf
        else:

            options.

        Returns
        -------
        Uniform
        """
        return super(UniformPowerLaw, cls).from_config(cp, section,
                                                       variable_args,
                                                       bounds_required=True)

class UniformRadius(UniformPowerLaw):
    """ For a uniform distribution in volume using spherical coordinates, this
    is the distriubtion to use for the radius.

    For more details see UniformPowerLaw.
    """
    name = "uniform_radius"
    def __init__(self, dim=3, **params):
        super(UniformRadius, self).__init__(dim=3, **params)

__all__ = ["UniformPowerLaw", "UniformRadius"]

# Copyright (C) 2018 Miriam Cabero, Collin Capano
#
#

import re
from pycbc import conversions, boundaries
from . import uniform, bounded

class UniformF0Tau(uniform.Uniform):
    """A distribution uniform in QNM frequency and damping time.

    Constraints may be placed to exclude frequencies and damping times
    corresponding to specific masses and spins.

    To ensure a properly normalized pdf that accounts for the constraints
    on final mass and spin, a renormalization factor is calculated upon
    initialization. This is calculated numerically: f0 and tau are drawn
    randomly, then the norm is scaled by the fraction of points that yield
    final masses and spins within the constraints. The `norm_tolerance` keyword
    arguments sets the error on the estimate of the norm from this numerical
    method. If this value is too large, such that no points are found in
    the allowed region, a ValueError is raised.

    f0 : tuple or boundaries.Bounds
        The range of QNM frequencies (in Hz).
    tau : tuple or boundaries.Bounds
        The range of QNM damping times (in s).
    final_mass : tuple or boundaries.Bounds, optional
        The range of final masses to allow. Default is [0,inf).
    final_spin : tuple or boundaries.Bounds, optional
        The range final spins to allow. Must be in [-0.996, 0.996], which is
        the default.
    rdfreq : str, optional
        Use the given string as the name for the f0 parameter. Default is 'f0'.
    damping_time : str, optional
        Use the given string as the name for the tau parameter. Default is
        'tau'.
    norm_tolerance : float, optional
        The tolerance on the estimate of the normalization. Default is 1e-3.
    norm_seed : int, optional
        Seed to use for the random number generator when estimating the norm.
        Default is 0. After the norm is estimated, the random number generator
        is set back to the state it was in upon initialization.

    Examples
    --------

    Create a distribution:

    >>> dist = UniformF0Tau(f0=(10., 2048.), tau=(1e-4,1e-2))

    Check that all random samples drawn from the distribution yield final
    masses > 1:

    >>> from pycbc import conversions
    >>> samples = dist.rvs(size=1000)
    >>> (conversions.final_mass_from_f0_tau(samples['f0'],
            samples['tau']) > 1.).all()
    True

    Create a distribution with tighter bounds on final mass and spin:

    >>> dist = UniformF0Tau(f0=(10., 2048.), tau=(1e-4,1e-2),
            final_mass=(20., 200.), final_spin=(0,0.996))

    Check that all random samples drawn from the distribution are in the
    final mass and spin constraints:

            samples['tau']) >= 20.).all()
    True
            samples['tau']) < 200.).all()
    True
    >>> (conversions.final_spin_from_f0_tau(samples['f0'],
            samples['tau']) >= 0.).all()
    True
            samples['tau']) < 0.996).all()
    True

    """

    name = 'uniform_f0_tau'

    def __init__(self, f0=None, tau=None, final_mass=None, final_spin=None,
                 rdfreq='f0', damping_time='tau', norm_tolerance=1e-3,
                 norm_seed=0):
        if f0 is None:
            raise ValueError("must provide a range for f0")
        if tau is None:
            raise ValueError("must provide a range for tau")
        self.rdfreq = rdfreq
        self.damping_time = damping_time
        parent_args = {rdfreq: f0, damping_time: tau}
        super(UniformF0Tau, self).__init__(**parent_args)
        if final_mass is None:
            final_mass = (0., numpy.inf)
        if final_spin is None:
            final_spin = (-0.996, 0.996)
        self.final_mass_bounds = boundaries.Bounds(
            min_bound=final_mass[0], max_bound=final_mass[1])
        self.final_spin_bounds = boundaries.Bounds(
            min_bound=final_spin[0], max_bound=final_spin[1])
        # Re-normalize to account for cuts: we'll do this by just sampling
        # a large number of spaces f0 taus, and seeing how many are in the
        # desired range.
        # perseve the current random state
        s = numpy.random.get_state()
        numpy.random.seed(norm_seed)
        nsamples = int(1./norm_tolerance**2)
        draws = super(UniformF0Tau, self).rvs(size=nsamples)
        # reset the random state
        numpy.random.set_state(s)
        num_in = self._constraints(draws).sum()
        # if num_in is 0, than the requested tolerance is too large
        if num_in == 0:
            raise ValueError("the normalization is < then the norm_tolerance; "
                             "try again with a smaller nrom_tolerance")
        self._lognorm += numpy.log(num_in) - numpy.log(nsamples)

        isin = super(UniformF0Tau, self).__contains__(params)
        if isin:
            isin &= self._constraints(params)
        return isin

    def _constraints(self, params):
        f0 = params[self.rdfreq]
        tau = params[self.damping_time]
        # check if we need to specify a particular mode (l,m) != (2,2)
        if re.match(r'f_\d{3}', self.rdfreq):
            mode = self.rdfreq.strip('f_')
            l, m = int(mode[0]), int(mode[1])
        else:
            l, m = 2, 2
        # temporarily silence invalid warnings... these will just be ruled out
        # automatically
        orig = numpy.seterr(invalid='ignore')
        mf = conversions.final_mass_from_f0_tau(f0, tau, l=l, m=m)
        sf = conversions.final_spin_from_f0_tau(f0, tau, l=l, m=m)
        isin = (self.final_mass_bounds.__contains__(mf)) & (
                self.final_spin_bounds.__contains__(sf))
        numpy.seterr(**orig)

        """Draw random samples from this distribution.

        size : int, optional
            The number of draws to do. Default is 1.

        Returns
        -------
        array
            A structured array of the random draws.
        """
        dtype = [(p, float) for p in self.params]
        keepidx = 0
            draws = super(UniformF0Tau, self).rvs(size=remaining)
            mask = self._constraints(draws)
            addpts = mask.sum()
            arr[keepidx:keepidx+addpts] = draws[mask]
            keepidx += addpts
            remaining = size - keepidx
        return arr

        """Initialize this class from a config file.

        Bounds on ``f0``, ``tau``, ``final_mass`` and ``final_spin`` should
        be specified by providing ``min-{param}`` and ``max-{param}``. If
        the ``f0`` or ``tau`` param should be renamed, ``rdfreq`` and
        ``damping_time`` should be provided; these must match
        ``variable_args``. If ``rdfreq`` and ``damping_time`` are not
        provided, ``variable_args`` are expected to be ``f0`` and ``tau``.

        Only ``min/max-f0`` and ``min/max-tau`` need to be provided.

        Example:

            [{section}-f0+tau]
            name = uniform_f0_tau
            min-f0 = 10
            max-f0 = 2048
            min-tau = 0.0001
            max-tau = 0.010
            min-final_mass = 10

            WorkflowConfigParser instance to read.
            The name of the section to read.
            The name of the variable args. These should be separated by
            ``pycbc.VARARGS_DELIM``.

        Returns
        -------
        UniformF0Tau :
            This class initialized with the parameters provided in the config
            file.
        """
        variable_args = set(variable_args.split(pycbc.VARARGS_DELIM))
        # get f0 and tau
        f0 = bounded.get_param_bounds_from_config(cp, section, tag, 'f0')
        tau = bounded.get_param_bounds_from_config(cp, section, tag, 'tau')
        # see if f0 and tau should be renamed
        if cp.has_option_tag(section, 'rdfreq', tag):
            rdfreq = cp.get_opt_tag(section, 'rdfreq', tag)
        else:
            rdfreq = 'f0'
        if cp.has_option_tag(section, 'damping_time', tag):
            damping_time = cp.get_opt_tag(section, 'damping_time', tag)
        else:
            damping_time = 'tau'
        # check that they match whats in the variable args
        if not variable_args == set([rdfreq, damping_time]):
            raise ValueError("variable args do not match rdfreq and "
                             "damping_time names")
        # get the final mass and spin values, if provided
        final_mass = bounded.get_param_bounds_from_config(
            cp, section, tag, 'final_mass')
        final_spin = bounded.get_param_bounds_from_config(
            cp, section, tag, 'final_spin')
        extra_opts = {}
        if cp.has_option_tag(section, 'norm_tolerance', tag):
            extra_opts['norm_tolerance'] = float(
                cp.get_opt_tag(section, 'norm_tolerance', tag))
        if cp.has_option_tag(section, 'norm_seed', tag):
            extra_opts['norm_seed'] = int(
                cp.get_opt_tag(section, 'norm_seed', tag))
        return cls(f0=f0, tau=tau,
                   final_mass=final_mass, final_spin=final_spin,
                   rdfreq=rdfreq, damping_time=damping_time,
                   **extra_opts)

#
#
"""This modules provides classes for evaluating sky distributions in
right acension and declination.
"""

from pycbc.distributions import angular

class UniformSky(angular.UniformSolidAngle):
    """A distribution that is uniform on the sky. This is the same as
    UniformSolidAngle, except that the polar angle varies from pi/2 (the north
    pole) to -pi/2 (the south pole) instead of 0 to pi. Also, the default
    names are "dec" (declination) for the polar angle and "ra" (right
    ascension) for the azimuthal angle, instead of "theta" and "phi".
    """
    name = 'uniform_sky'
    _polardistcls = angular.CosAngle
    _default_polar_angle = 'dec'
    _default_azimuthal_angle = 'ra'

__all__ = ['UniformSky']

# Copyright (C) 2017 Collin Capano, Chris Biwer
#
#
"""
This modules provides spin distributions of CBCs.
"""

from pycbc.distributions.uniform import Uniform
from pycbc.distributions.angular import UniformAngle
from pycbc.distributions.power_law import UniformPowerLaw
from pycbc.distributions.arbitrary import Arbitrary
from pycbc.distributions.bounded import get_param_bounds_from_config, \
    VARARGS_DELIM, BoundedDist

class IndependentChiPChiEff(Arbitrary):
    r"""A distribution such that :math:`\chi_{\mathrm{eff}}` and
    :math:`\chi_p` are uniform and independent of each other.

    To ensure constraints are applied correctly, this distribution produces all
    three components of both spins as well as the component masses.

    mass1 : BoundedDist, Bounds, or tuple
        The distribution or bounds to use for mass1. Must be either a
        BoundedDist giving the distribution on mass1, or bounds (as
        either a Bounds instance or a tuple) giving the minimum and maximum
        values to use for mass1. If the latter, a Uniform distribution will
        be used.
    mass2 : BoundedDist, Bounds, or tuple
        The distribution or bounds to use for mass2. Syntax is the same as
        mass1.
    chi_eff : BoundedDist, Bounds, or tuple; optional
        The distribution or bounds to use for :math:`chi_eff`. Syntax is the
        same as mass1, except that None may also be passed. In that case,
        `(-1, 1)` will be used for the bounds. Default is None.
    chi_a : BoundedDist, Bounds, or tuple; optional
        The distribution or bounds to use for :math:`chi_a`. Syntax is the
    xi_bounds : Bounds or tuple, optional
        The bounds to use for :math:`\xi_1` and :math:`\xi_2`. Must be
        :math:`\in (0, 1)`. If None (the default), will be `(0, 1)`.
    nsamples : int, optional
        The number of samples to use for the internal kde. The larger the
        number of samples, the more accurate the pdf will be, but the longer
        it will take to evaluate. Default is 10000.
    seed : int, optional
        Seed value to use for the number generator for the kde. The current
        random state of numpy will be saved prior to setting the seed. After
        the samples are generated, the state will be set back to what it was.
        If None provided, will use 0.
    """
    name = "independent_chip_chieff"
    _params = ['mass1', 'mass2', 'xi1', 'xi2', 'chi_eff', 'chi_a',
               'phi_a', 'phi_s']

    def __init__(self, mass1=None, mass2=None, chi_eff=None, chi_a=None,
                 xi_bounds=None, nsamples=None, seed=None):

        if isinstance(mass1, BoundedDist):
            self.mass1_distr = mass1
        else:
            self.mass1_distr = Uniform(mass1=mass1)
        if isinstance(mass2, BoundedDist):
            self.mass2_distr = mass2
        else:
            self.mass2_distr = Uniform(mass2=mass2)
        # chi eff
        if isinstance(chi_eff, BoundedDist):
            self.chieff_distr = chi_eff
        else:
            if chi_eff is None:
                chi_eff = (-1., 1.)
            self.chieff_distr = Uniform(chi_eff=chi_eff)
        if isinstance(chi_a, BoundedDist):
            self.chia_distr = chi_a
        else:
            if chi_a is None:
                chi_a = (-1., 1.)
            self.chia_distr = Uniform(chi_a=chi_a)
        # xis
        if xi_bounds is None:
            xi_bounds = (0, 1.)
        if (xi_bounds[0] > 1. or xi_bounds[0] < 0.) or (
            xi_bounds[1] > 1. or xi_bounds[1] < 0.):
            raise ValueError("xi bounds must be in [0, 1)")
        self.xi1_distr = UniformPowerLaw(dim=0.5, xi1=xi_bounds)
        self.xi2_distr = UniformPowerLaw(dim=0.5, xi2=xi_bounds)
        # the angles
        self.phia_distr = UniformAngle(phi_a=(0,2))
        self.phis_distr = UniformAngle(phi_s=(0,2))
        self.distributions = {'mass1': self.mass1_distr,
                              'mass2': self.mass2_distr,
                              'xi1': self.xi1_distr,
                              'xi2': self.xi2_distr,
                              'chi_eff': self.chieff_distr,
                              'chi_a': self.chia_distr,
                              'phi_a': self.phia_distr,
                              'phi_s': self.phis_distr}
        # create random variables for the kde
        if nsamples is None:
            nsamples = 1e4
        # save the current random state
        rstate = numpy.random.get_state()
        # set the seed
        if seed is None:
            seed = 0
        numpy.random.seed(seed)
        rvals = self.rvs(size=int(nsamples))
        # reset the random state back to what it was
        numpy.random.set_state(rstate)
        bounds = dict(b for distr in self.distributions.values()
                        for b in distr.bounds.items())
        super(IndependentChiPChiEff, self).__init__(mass1=rvals['mass1'],
            mass2=rvals['mass2'], xi1=rvals['xi1'], xi2=rvals['xi2'],
            chi_eff=rvals['chi_eff'], chi_a=rvals['chi_a'],
            phi_a=rvals['phi_a'], phi_s=rvals['phi_s'],
            bounds=bounds)

    def _constraints(self, values):
        """Applies physical constraints to the given parameter values.

        values : {arr or dict}
            A dictionary or structured array giving the values.

        Returns
        -------
        bool
            Whether or not the values satisfy physical
        """
        mass1, mass2, phi_a, phi_s, chi_eff, chi_a, xi1, xi2, _ = \
            conversions.ensurearray(values['mass1'], values['mass2'],
                                    values['phi_a'], values['phi_s'],
                                    values['chi_eff'], values['chi_a'],
                                    values['xi1'], values['xi2'])
        s1x = conversions.spin1x_from_xi1_phi_a_phi_s(xi1, phi_a, phi_s)
        s2x = conversions.spin2x_from_mass1_mass2_xi2_phi_a_phi_s(mass1, mass2,
            xi2, phi_a, phi_s)
        s1y = conversions.spin1y_from_xi1_phi_a_phi_s(xi1, phi_a, phi_s)
        s2y = conversions.spin2y_from_mass1_mass2_xi2_phi_a_phi_s(mass1, mass2,
        s1z = conversions.spin1z_from_mass1_mass2_chi_eff_chi_a(mass1, mass2,
            chi_eff, chi_a)
        s2z = conversions.spin2z_from_mass1_mass2_chi_eff_chi_a(mass1, mass2,
        test = ((s1x**2. + s1y**2. + s1z**2.) < 1.) & \
               ((s2x**2. + s2y**2. + s2z**2.) < 1.)
        return test

        """Determines whether the given values are in each parameter's bounds
        and satisfy the constraints.
        """
        isin = all([params in dist for dist in self.distributions.values()])
            return False
        # in the individual distributions, apply constrains
        return self._constraints(params)

    def _draw(self, size=1, **kwargs):
        """Draws random samples without applying physical constrains.
        """
        # draw masses
        try:
            mass1 = kwargs['mass1']
            mass1 = self.mass1_distr.rvs(size=size)['mass1']
        try:
            mass2 = kwargs['mass2']
            mass2 = self.mass2_distr.rvs(size=size)['mass2']
        # draw angles
        try:
            phi_a = kwargs['phi_a']
            phi_a = self.phia_distr.rvs(size=size)['phi_a']
        try:
            phi_s = kwargs['phi_s']
            phi_s = self.phis_distr.rvs(size=size)['phi_s']
        # draw chi_eff, chi_a
        try:
            chi_eff = kwargs['chi_eff']
            chi_eff = self.chieff_distr.rvs(size=size)['chi_eff']
        try:
            chi_a = kwargs['chi_a']
            chi_a = self.chia_distr.rvs(size=size)['chi_a']
        # draw xis
        try:
            xi1 = kwargs['xi1']
            xi1 = self.xi1_distr.rvs(size=size)['xi1']
        try:
            xi2 = kwargs['xi2']
            xi2 = self.xi2_distr.rvs(size=size)['xi2']
        arr['mass1'] = mass1
        arr['mass2'] = mass2
        arr['phi_a'] = phi_a
        arr['phi_s'] = phi_s
        arr['chi_eff'] = chi_eff
        arr['chi_a'] = chi_a
        arr['xi1'] = xi1
        arr['xi2'] = xi2
        return arr

        return kwargs

    def rvs(self, size=1, **kwargs):
        """Returns random values for all of the parameters.
        """
        keepidx = 0
            draws = self._draw(size=remaining, **kwargs)
        return arr

            options.

        Returns
        -------
        IndependentChiPChiEff
            A distribution instance.
        """
        if not set(variable_args) == set(cls._params):
            raise ValueError("Not all parameters used by this distribution "
                             "included in tag portion of section name")
        # get the bounds for the setable parameters
        mass1 = get_param_bounds_from_config(cp, section, tag, 'mass1')
        mass2 = get_param_bounds_from_config(cp, section, tag, 'mass2')
        chi_eff = get_param_bounds_from_config(cp, section, tag, 'chi_eff')
        chi_a = get_param_bounds_from_config(cp, section, tag, 'chi_a')
        xi_bounds = get_param_bounds_from_config(cp, section, tag, 'xi_bounds')
        if cp.has_option('-'.join([section, tag]), 'nsamples'):
            nsamples = int(cp.get('-'.join([section, tag]), 'nsamples'))
        else:
            nsamples = None
        return cls(mass1=mass1, mass2=mass2, chi_eff=chi_eff, chi_a=chi_a,
                   xi_bounds=xi_bounds, nsamples=nsamples)

#
#
"""
This modules provides classes for evaluating uniform distributions.
"""

class Uniform(bounded.BoundedDist):
    """
    A uniform distribution on the given parameters. The parameters are

        instance.

    name : 'uniform'

    Examples
    --------
    Create a 2 dimensional uniform distribution:

    >>> from pycbc import distributions
    >>> dist = distributions.Uniform(mass1=(10.,50.), mass2=(10.,50.))

    Get the log of the pdf at a particular value:

    >>> dist.logpdf(mass1=25., mass2=10.)
        -7.3777589082278725

    Do the same by calling the distribution:

    >>> dist(mass1=25., mass2=10.)

    Generate some random values:

    >>> dist.rvs(size=3)
        array([(36.90885758394699, 51.294212757995254),
               (39.109058546060346, 13.36220145743631),
               (34.49594465315212, 47.531953033719454)],
              dtype=[('mass1', '<f8'), ('mass2', '<f8')])

    Initialize a uniform distribution using a boundaries.Bounds instance,
    with cyclic bounds:

    >>> dist = distributions.Uniform(phi=Bounds(10, 50, cyclic=True))

    Apply boundary conditions to a value:

    >>> dist.apply_boundary_conditions(phi=60.)
        {'mass1': array(20.0)}

    The boundary conditions are applied to the value before evaluating the pdf;
    note that the following returns a non-zero pdf. If the bounds were not
    cyclic, the following would return 0:

    >>> dist.pdf(phi=60.)
        0.025
    """
    name = 'uniform'
        super(Uniform, self).__init__(**params)
        # compute the norm and save
        # temporarily suppress numpy divide by 0 warning
        numpy.seterr(divide='ignore')
        self._lognorm = -sum([numpy.log(abs(bnd[1]-bnd[0]))
                                    for bnd in self._bounds.values()])
        numpy.seterr(divide='warn')

    @property

    @property

        """
        lower_bound = self._bounds[param][0]
        upper_bound = self._bounds[param][1]
        return (upper_bound - lower_bound) * value + lower_bound

        ignored.
        """
            return self._norm
        else:
            return 0.

        """
            return self._lognorm
        else:

        Returns
        -------
        """
        else:
            arr[p] = numpy.random.uniform(self._bounds[p][0],
                                        self._bounds[p][1],
                                        size=size)
        return arr

            options.

        Returns
        -------
        Uniform
        """
        return super(Uniform, cls).from_config(cp, section, variable_args,
                     bounds_required=True)

__all__ = ['Uniform']

#
#
""" This modules provides classes for evaluating distributions whose logarithm
are uniform.
"""

class UniformLog10(uniform.Uniform):
    """ A uniform distribution on the log base 10 of the given parameters.
    be called like a function. By default, logpdf will be called.

        instance.

    name : "uniform_log10"
    """
    name = "uniform_log10"

        super(UniformLog10, self).__init__(**params)
        self._norm = numpy.prod([numpy.log10(bnd[1]) - numpy.log10(bnd[0])
                                   for bnd in self._bounds.values()])
        self._lognorm = numpy.log(self._norm)

        lower_bound = numpy.log10(self._bounds[param][0])
        upper_bound = numpy.log10(self._bounds[param][1])
        return 10. ** ((upper_bound - lower_bound) * value + lower_bound)

        Returns
        -------
        """

        else:
            log_high = numpy.log10(self._bounds[p][0])
            log_low = numpy.log10(self._bounds[p][1])
            arr[p] = 10.0**(numpy.random.uniform(log_low, log_high, size=size))
        return arr

        ignored.
        """
            vals = numpy.array([numpy.log(10) * self._norm * kwargs[param]
                                for param in kwargs.keys()])
            return 1.0 / numpy.prod(vals)
        else:
            return 0.

        """
        else:

__all__ = ["UniformLog10"]

# Copyright (C)  2016 Collin Capano, Christopher M. Biwer, Alex Nitz
#
#
"""
This modules provides classes and functions for drawing and calculating the
probability density function of distributions.
"""
# imports needed for functions below
from six.moves import configparser as _ConfigParser
from pycbc.distributions import constraints
from pycbc import VARARGS_DELIM as _VARARGS_DELIM

# Promote some classes/functions to the distributions name space
from pycbc.distributions.angular import UniformAngle, SinAngle, CosAngle, \
                                        UniformSolidAngle
from pycbc.distributions.arbitrary import Arbitrary, FromFile
from pycbc.distributions.gaussian import Gaussian
from pycbc.distributions.power_law import UniformPowerLaw, UniformRadius
from pycbc.distributions.sky_location import UniformSky
from pycbc.distributions.uniform_log import UniformLog10
from pycbc.distributions.spins import IndependentChiPChiEff
from pycbc.distributions.qnm import UniformF0Tau
from pycbc.distributions.joint import JointDistribution
from pycbc.distributions.external import External
from pycbc.distributions.fixedsamples import FixedSamples
from pycbc.distributions.mass import MchirpfromUniformMass1Mass2, \
                                     QfromUniformMass1Mass2

# a dict of all available distributions
distribs = {
    IndependentChiPChiEff.name : IndependentChiPChiEff,
    Arbitrary.name : Arbitrary,
    FromFile.name : FromFile,
    Gaussian.name : Gaussian,
    UniformPowerLaw.name : UniformPowerLaw,
    UniformRadius.name : UniformRadius,
    Uniform.name : Uniform,
    UniformAngle.name : UniformAngle,
    CosAngle.name : CosAngle,
    SinAngle.name : SinAngle,
    UniformSolidAngle.name : UniformSolidAngle,
    UniformSky.name : UniformSky,
    UniformLog10.name : UniformLog10,
    UniformF0Tau.name : UniformF0Tau,
    External.name: External,
    FixedSamples.name: FixedSamples,
    MchirpfromUniformMass1Mass2.name: MchirpfromUniformMass1Mass2,
    QfromUniformMass1Mass2.name: QfromUniformMass1Mass2
}

def read_distributions_from_config(cp, section="prior"):
    """Returns a list of PyCBC distribution instances for a section in the

    section : {"prior", string}
        Prefix on section names from which to retrieve the distributions.

    Returns
    -------
    list
        A list of the parsed distributions.
    """
    dists = []
    variable_args = []
        dist = distribs[name].from_config(cp, section, subsection)
        if set(dist.params).isdisjoint(variable_args):
            dists.append(dist)
            variable_args += dist.params
        else:
            raise ValueError("Same parameter in more than one distribution.")
    return dists

def _convert_liststring_to_list(lstring):
    """Checks if an argument of the configuration file is a string of a list
    and returns the corresponding list (of strings).

    The argument is considered to be a list if it starts with '[' and ends
    with ']'. List elements should be comma separated. For example, passing
    `'[foo bar, cat]'` will result in `['foo bar', 'cat']` being returned. If
    the argument does not start and end with '[' and ']', the argument will
    just be returned as is.
    """
    if lstring[0]=='[' and lstring[-1]==']':
        lstring = [str(lstring[1:-1].split(',')[n].strip().strip("'"))
                      for n in range(len(lstring[1:-1].split(',')))]
    return lstring

def read_params_from_config(cp, prior_section='prior',
                            vargs_section='variable_args',
                            sargs_section='static_args'):
    """Loads static and variable parameters from a configuration file.

    cp : WorkflowConfigParser
        An open config parser to read from.
    prior_section : str, optional
        Check that priors exist in the given section. Default is 'prior.'
    vargs_section : str, optional
        The section to get the parameters that will be varied/need priors
        defined for them. Default is 'variable_args'.
    sargs_section : str, optional
        The section to get the parameters that will remain fixed. Default is
        'static_args'.

    Returns
    -------
        The names of the parameters to vary in the PE run.
    static_args : dict
        Dictionary of names -> values giving the parameters to keep fixed.
    """
    # sanity check that each parameter in [variable_args] has a priors section
    variable_args = cp.options(vargs_section)
    subsections = cp.get_subsections(prior_section)
    tags = set([p for tag in subsections for p in tag.split('+')])
    missing_prior = set(variable_args) - tags
    if any(missing_prior):
        raise KeyError("You are missing a priors section in the config file "
                       "for parameter(s): {}".format(', '.join(missing_prior)))
    # sanity check that each parameter with a priors section is in
    # [variable_args]
    missing_variable = tags - set(variable_args)
    if any(missing_variable):
        raise KeyError("Prior section found for parameter(s) {} but not "
                       "listed as variable parameter(s)."
                       .format(', '.join(missing_variable)))
    # get static args
    try:
        static_args = dict([(key, cp.get_opt_tags(sargs_section, key, []))
                           for key in cp.options(sargs_section)])
    except _ConfigParser.NoSectionError:
        static_args = {}
    # sanity check that each parameter in [variable_args]
    # is not repeated in [static_args]
    for arg in variable_args:
        if arg in static_args:
            raise KeyError("Parameter {} found both in static_args and in "
                           "variable_args sections.".format(arg))
    # try converting values to float
    for key in static_args:
        val = static_args[key]
        try:
            # the following will raise a ValueError if it cannot be cast to
            # float (as we would expect for string arguments)
            static_args[key] = float(val)
            # try converting to a list of strings; this function will just
            # return val if it does not begin (end) with [ (])
            static_args[key] = _convert_liststring_to_list(val)
    return variable_args, static_args

def read_constraints_from_config(cp, transforms=None,
                                 constraint_section='constraint'):
    """Loads parameter constraints from a configuration file.

    transforms : list, optional
        List of transforms to apply to parameters before applying constraints.
    constraint_section : str, optional
        The section to get the constraints from. Default is 'constraint'.

    Returns
    -------
    list
        List of ``Constraint`` objects. Empty if no constraints were provided.
    """
    cons = []
    for subsection in cp.get_subsections(constraint_section):
        name = cp.get_opt_tag(constraint_section, "name", subsection)
        constraint_arg = cp.get_opt_tag(
            constraint_section, "constraint_arg", subsection)
        # get any other keyword arguments
        kwargs = {}
        section = constraint_section + "-" + subsection
        extra_opts = [key for key in cp.options(section)
                      if key not in ["name", "constraint_arg"]]
        for key in extra_opts:
            val = cp.get(section, key)
            if key == "required_parameters":
                val = val.split(_VARARGS_DELIM)
            else:
                try:
                    val = float(val)
                    pass
            kwargs[key] = val
        cons.append(constraints.constraints[name](constraint_arg,
                                                  transforms=transforms,
                                                  **kwargs))

    return cons

# Copyright (C) 2015 Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
""" This modules contains functions for calculating and manipulating
coincident triggers.
"""

import numpy, logging, pycbc.pnutils, copy, lal

def background_bin_from_string(background_bins, data):
    """ Return template ids for each bin as defined by the format string

    bins: list of strings
        List of strings which define how a background bin is taken from the
        list of templates.
    data: dict of numpy.ndarrays
        Dict with parameter key values and numpy.ndarray values which define
        the parameters of the template bank to bin up.

    Returns
    -------
    bins: dict
        Dictionary of location indices indexed by a bin name
    """
    used = numpy.array([], dtype=numpy.uint32)
    bins = {}
    for mbin in background_bins:
        name, bin_type, boundary = tuple(mbin.split(':'))

        if boundary[0:2] == 'lt':
            member_func = lambda vals, bd=boundary : vals < float(bd[2:])
        elif boundary[0:2] == 'gt':
            member_func = lambda vals, bd=boundary : vals > float(bd[2:])
        else:
            raise RuntimeError("Can't parse boundary condition! Must begin "
                               "with 'lt' or 'gt'")

        if bin_type == 'component' and boundary[0:2] == 'lt':
            # maximum component mass is less than boundary value
            vals = numpy.maximum(data['mass1'], data['mass2'])
        if bin_type == 'component' and boundary[0:2] == 'gt':
            # minimum component mass is greater than bdary
            vals = numpy.minimum(data['mass1'], data['mass2'])
        elif bin_type == 'total':
            vals = data['mass1'] + data['mass2']
        elif bin_type == 'chirp':
            vals = pycbc.pnutils.mass1_mass2_to_mchirp_eta(
                                               data['mass1'], data['mass2'])[0]
        elif bin_type == 'SEOBNRv2Peak':
            vals = pycbc.pnutils.get_freq('fSEOBNRv2Peak',
                  data['mass1'], data['mass2'], data['spin1z'], data['spin2z'])
        elif bin_type == 'SEOBNRv4Peak':
            vals = pycbc.pnutils.get_freq('fSEOBNRv4Peak', data['mass1'],
                                          data['mass2'], data['spin1z'],
                                          data['spin2z'])
        elif bin_type == 'SEOBNRv2duration':
            vals = pycbc.pnutils.get_imr_duration(data['mass1'], data['mass2'],
                               data['spin1z'], data['spin2z'], data['f_lower'],
                                                        approximant='SEOBNRv2')
        else:
            raise ValueError('Invalid bin type %s' % bin_type)

        locs = member_func(vals)
        del vals

        # make sure we don't reuse anything from an earlier bin
        locs = numpy.where(locs)[0]
        locs = numpy.delete(locs, numpy.where(numpy.in1d(locs, used))[0])
        used = numpy.concatenate([used, locs])
        bins[name] = locs

    return bins

def calculate_n_louder(bstat, fstat, dec, skip_background=False):
    """ Calculate for each foreground event the number of background events
    that are louder than it.

    bstat: numpy.ndarray
        Array of the background statistic values
    fstat: numpy.ndarray or scalar
        Array of the foreground statistic values or single value
    dec: numpy.ndarray
        Array of the decimation factors for the background statistics
    skip_background: optional, {boolean, False}
        Skip calculating cumulative numbers for background triggers

    Returns
    -------
    cum_back_num: numpy.ndarray
        The cumulative array of background triggers. Does not return this
        argument if skip_background == True
    fore_n_louder: numpy.ndarray
        The number of background triggers above each foreground trigger
    """
    sort = bstat.argsort()
    bstat = bstat[sort]
    dec = dec[sort]

    # calculate cumulative number of triggers louder than the trigger in
    # a given index. We need to subtract the decimation factor, as the cumsum
    # includes itself in the first sum (it is inclusive of the first value)
    n_louder = dec[::-1].cumsum()[::-1] - dec

    # Determine how many values are louder than the foreground ones
    # We need to subtract one from the index, to be consistent with the definition
    # of n_louder, as here we do want to include the background value at the
    # found index
    idx = numpy.searchsorted(bstat, fstat, side='left') - 1

    # If the foreground are *quieter* than the background or at the same value
    # then the search sorted algorithm will choose position -1, which does not exist
    # We force it back to zero.
    if isinstance(idx, numpy.ndarray):  # Case where our input is an array
        idx[idx < 0] = 0
    else:  # Case where our input is just a scalar value
        if idx < 0:
            idx = 0

    fore_n_louder = n_louder[idx]

    if not skip_background:
        unsort = sort.argsort()
        back_cum_num = n_louder[unsort]
        return back_cum_num, fore_n_louder
    else:
        return fore_n_louder

def timeslide_durations(start1, start2, end1, end2, timeslide_offsets):
    """ Find the coincident time for each timeslide.

    Find the coincident time for each timeslide, where the first time vector
    is slid to the right by the offset in the given timeslide_offsets vector.

    start1: numpy.ndarray
        Array of the start of valid analyzed times for detector 1
    start2: numpy.ndarray
        Array of the start of valid analyzed times for detector 2
    end1: numpy.ndarray
        Array of the end of valid analyzed times for detector 1
    end2: numpy.ndarray
        Array of the end of valid analyzed times for detector 2
    timseslide_offset: numpy.ndarray
        Array of offsets (in seconds) for each timeslide

    Returns
    --------
    durations: numpy.ndarray
        Array of coincident time for each timeslide in the offset array
    """
    from . import veto
    durations = []
    seg2 = veto.start_end_to_segments(start2, end2)
    for offset in timeslide_offsets:
        seg1 = veto.start_end_to_segments(start1 + offset, end1 + offset)
        durations.append(abs((seg1 & seg2).coalesce()))
    return numpy.array(durations)

def time_coincidence(t1, t2, window, slide_step=0):
    """ Find coincidences by time window

    t1 : numpy.ndarray
        Array of trigger times from the first detector
    t2 : numpy.ndarray
        Array of trigger times from the second detector
    window : float
        Coincidence window maximum time difference, arbitrary units (usually s)
    slide_step : float (default 0)
        If calculating background coincidences, the interval between background
        slides, arbitrary units (usually s)

    Returns
    -------
    idx1 : numpy.ndarray
        Array of indices into the t1 array for coincident triggers
    idx2 : numpy.ndarray
        Array of indices into the t2 array
    slide : numpy.ndarray
        Array of slide ids
    """
    if slide_step:
        fold1 = t1 % slide_step
        fold2 = t2 % slide_step
    else:
        fold1 = t1
        fold2 = t2

    sort1 = fold1.argsort()
    sort2 = fold2.argsort()
    fold1 = fold1[sort1]
    fold2 = fold2[sort2]

        # FIXME explain this
        fold2 = numpy.concatenate([fold2 - slide_step, fold2, fold2 + slide_step])
        sort2 = numpy.concatenate([sort2, sort2, sort2])

    left = numpy.searchsorted(fold2, fold1 - window)
    right = numpy.searchsorted(fold2, fold1 + window)

    idx1 = numpy.repeat(sort1, right - left)
    idx2 = [sort2[l:r] for l, r in zip(left, right)]

    if len(idx2) > 0:
        idx2 = numpy.concatenate(idx2)
    else:
        idx2 = numpy.array([], dtype=numpy.int64)

        diff = ((t1 / slide_step)[idx1] - (t2 / slide_step)[idx2])
        slide = numpy.rint(diff)
    else:
        slide = numpy.zeros(len(idx1))

    return idx1.astype(numpy.uint32), idx2.astype(numpy.uint32), slide.astype(numpy.int32)

def time_multi_coincidence(times, slide_step=0, slop=.003,
                           pivot='H1', fixed='L1'):
    """ Find multi detector coincidences.

    times: dict of numpy.ndarrays
        Dictionary keyed by ifo of single ifo trigger times
    slide_step: float
        Interval between time slides
    slop: float
        The amount of time to add to the TOF between detectors for coincidence
    pivot: str
        The ifo to which time shifts are applied in first stage coincidence
    fixed: str
        The other ifo used in first stage coincidence, subsequently used as a
        time reference for additional ifos. All other ifos are not time shifted
        relative to this ifo

    Returns
    -------
    ids: dict of arrays of int
        Dictionary keyed by ifo with ids of trigger times forming coincidences.
        Coincidence is tested for every pair of ifos that can be formed from
        the input dict: only those tuples of times passing all tests are
        recorded
    slide: array of int
        Slide ids of coincident triggers in pivot ifo
    """
    def win(ifo1, ifo2):
        d1 = Detector(ifo1)
        d2 = Detector(ifo2)
        return d1.light_travel_time_to_detector(d2) + slop

    # Find coincs between the 'pivot' and 'fixed' detectors as in 2-ifo case
    pivot_id, fix_id, slide = time_coincidence(times[pivot], times[fixed],
                                               win(pivot, fixed),
                                               slide_step=slide_step)

    # Additional detectors do not slide independently of the 'fixed' one
    # Each trigger in an additional detector must be concident with both
    # triggers in an existing coincidence

    # Slide 'pivot' trigger times to be coincident with trigger times in
    # 'fixed' detector
    fixed_time = times[fixed][fix_id]
    pivot_time = times[pivot][pivot_id] - slide_step * slide
    ctimes = {fixed: fixed_time, pivot: pivot_time}
    ids = {fixed: fix_id, pivot: pivot_id}

    dep_ifos = [ifo for ifo in times.keys() if ifo != fixed and ifo != pivot]
    for ifo1 in dep_ifos:
        # FIXME - make this loop into a function?

        # otime is extra ifo time in original trigger order
        otime = times[ifo1]
        # tsort gives ordering from original order to time sorted order
        tsort = otime.argsort()
        time1 = otime[tsort]

        # Find coincidences between dependent ifo triggers and existing coincs
        # - Cycle over fixed and pivot
        # - At the 1st iteration, the fixed and pivot triggers are reduced to
        #  those for which the first out of fixed/pivot forms a coinc with ifo1
        # - At the 2nd iteration, we are left with triggers for which both
        #  fixed and pivot are coincident with ifo1
        # - If there is more than 1 dependent ifo, ones that were previously
        #  tested against fixed and pivot are now present for testing with new
        #  dependent ifos
        for ifo2 in ids:
            logging.info('added ifo %s, testing against %s' % (ifo1, ifo2))
            w = win(ifo1, ifo2)
            left = numpy.searchsorted(time1, ctimes[ifo2] - w)
            right = numpy.searchsorted(time1, ctimes[ifo2] + w)
            # Any times within time1 coincident with the time in ifo2 have
            # indices between 'left' and 'right'
            # 'nz' indexes into times in ifo2 which have coincidences with ifo1
            # times
            nz = (right - left).nonzero()
            if len(right - left):
                rlmax = (right - left).max()
            if len(nz[0]) and rlmax > 1:
                # We expect at most one coincident time in ifo1, assuming
                #  trigger spacing in ifo1 > time window.
                # However there are rare corner cases at starts/ends of inspiral
                #  jobs. For these, arbitrarily keep the first trigger and
                #  discard the second (and any subsequent ones).
                where = right - left == rlmax
                logging.warning('Triggers in %s are closer than coincidence '
                                'window, 1 or more coincs will be discarded. '
                                'This is a warning, not an error.' % ifo1)
                print([float(ti) for ti in
                       time1[left[where][0]:right[where][0]]])
            # identify indices of times in ifo1 that form coincs with ifo2
            dep_ids = left[nz]
            # slide is array of slide ids attached to pivot ifo
            slide = slide[nz]

            for ifo in ctimes:
                # cycle over fixed and pivot & any previous additional ifos
                # reduce times and IDs to just those forming a coinc with ifo1
                ctimes[ifo] = ctimes[ifo][nz]
                ids[ifo] = ids[ifo][nz]

        # undo time sorting on indices of ifo1 triggers, add ifo1 ids and times
        # to dicts for testing against any additional detectrs
        ids[ifo1] = tsort[dep_ids]
        ctimes[ifo1] = otime[ids[ifo1]]

    return ids, slide

def cluster_coincs(stat, time1, time2, timeslide_id, slide, window, argmax=numpy.argmax):
    """Cluster coincident events for each timeslide separately, across
    templates, based on the ranking statistic

    stat: numpy.ndarray
        vector of ranking values to maximize
    time1: numpy.ndarray
        first time vector
    time2: numpy.ndarray
        second time vector
    timeslide_id: numpy.ndarray
        vector that determines the timeslide offset
    slide: float
        length of the timeslides offset interval
    window: float
        length to cluster over

    Returns
    -------
    cindex: numpy.ndarray
        The set of indices corresponding to the surviving coincidences.
    """

    logging.info('clustering coinc triggers over %ss window' % window)

    if len(time1) == 0 or len(time2) == 0:
        logging.info('No coinc triggers in one, or both, ifos.')
        return numpy.array([])

    if numpy.isfinite(slide):
        # for a time shifted coinc, time1 is greater than time2 by approximately timeslide_id*slide
        # adding this quantity gives a mean coinc time located around time1
        time = (time1 + time2 + timeslide_id * slide) / 2
    else:
        time = 0.5 * (time2 + time1)

    tslide = timeslide_id.astype(numpy.float128)
    time = time.astype(numpy.float128)

    span = (time.max() - time.min()) + window * 10
    time = time + span * tslide
    cidx = cluster_over_time(stat, time, window, argmax)
    return cidx

def cluster_coincs_multiifo(stat, time_coincs, timeslide_id, slide, window, argmax=numpy.argmax):

    time_coincs: tuple of numpy.ndarrays
        trigger times for each ifo, or -1 if an ifo does not participate in a coinc
        duration of clustering window in seconds

    Returns
    -------
        The set of indices corresponding to the surviving coincidences
    """
    time_coinc_zip = list(zip(*time_coincs))
    if len(time_coinc_zip) == 0:
        logging.info('No coincident triggers.')

    time_avg_num = []
    #find number of ifos and mean time over participating ifos for each coinc
    for tc in time_coinc_zip:
        time_avg_num.append(mean_if_greater_than_zero(tc))

    time_avg, num_ifos = zip(*time_avg_num)

    time_avg = numpy.array(time_avg)
    num_ifos = numpy.array(num_ifos)

    # shift all but the pivot ifo by (num_ifos-1) * timeslide_id * slide
    # this leads to a mean coinc time located around pivot time
        nifos_minusone = (num_ifos - numpy.ones_like(num_ifos))
        time_avg = time_avg + (nifos_minusone * timeslide_id * slide)/num_ifos

    time_avg = time_avg.astype(numpy.float128)

    span = (time_avg.max() - time_avg.min()) + window * 10
    time_avg = time_avg + span * tslide
    cidx = cluster_over_time(stat, time_avg, window, argmax)

def mean_if_greater_than_zero(vals):
    """ Calculate mean over numerical values, ignoring values less than zero.
    E.g. used for mean time over coincident triggers when timestamps are set
    to -1 for ifos not included in the coincidence.

    vals: iterator of numerical values
        values to be mean averaged

    Returns
    -------
    mean: float
        The mean of the values in the original vector which are
        greater than zero
    num_above_zero: int
        The number of entries in the vector which are above zero
    """
    vals = numpy.array(vals)
    above_zero = vals > 0
    return vals[above_zero].mean(), above_zero.sum()

def cluster_over_time(stat, time, window, argmax=numpy.argmax):
    """Cluster generalized transient events over time via maximum stat over a
    symmetric sliding window

    time: numpy.ndarray
        time to use for clustering
    argmax: function
        the function used to calculate the maximum value

    Returns
    -------
    """
    logging.info('Clustering events over %s s window', window)

    indices = []
    time_sorting = time.argsort()
    stat = stat[time_sorting]
    time = time[time_sorting]

    left = numpy.searchsorted(time, time - window)
    right = numpy.searchsorted(time, time + window)
    indices = numpy.zeros(len(left), dtype=numpy.uint32)

    # i is the index we are inspecting, j is the next one to save
    i = 0
    j = 0
    while i < len(left):
        l = left[i]
        r = right[i]

        # If there are no other points to compare it is obviously the max
        if (r - l) == 1:
            indices[j] = i
            j += 1
            i += 1
            continue

        # Find the location of the maximum within the time interval around i
        max_loc = argmax(stat[l:r]) + l

        # If this point is the max, we can skip to the right boundary
        if max_loc == i:
            i = r
            j += 1

        # If the max is later than i, we can skip to it
        elif max_loc > i:
            i = max_loc

        elif max_loc < i:
            i += 1

    indices = indices[:j]

    logging.info('%d triggers remaining', len(indices))
    return time_sorting[indices]

class MultiRingBuffer(object):
    """Dynamic size n-dimensional ring buffer that can expire elements."""

    def __init__(self, num_rings, max_time, dtype):
        """
        num_rings: int
            The number of ring buffers to create. They all will have the same
            intrinsic size and will expire at the same time.
        max_time: int
            The maximum "time" an element can exist in each ring.
        dtype: numpy.dtype
            The type of each element in the ring buffer.
        """
        self.max_time = max_time
        self.buffer = []
        self.buffer_expire = []
        for _ in range(num_rings):
            self.buffer.append(numpy.zeros(0, dtype=dtype))
            self.buffer_expire.append(numpy.zeros(0, dtype=int))
        self.time = 0

    @property
    def filled_time(self):
        return min(self.time, self.max_time)

    def num_elements(self):
        return sum([len(a) for a in self.buffer])

    @property
    def nbytes(self):
        return sum([a.nbytes for a in self.buffer])

    def discard_last(self, indices):
        """Discard the triggers added in the latest update"""
        for i in indices:
            self.buffer_expire[i] = self.buffer_expire[i][:-1]
            self.buffer[i] = self.buffer[i][:-1]

    def advance_time(self):
        """Advance the internal time increment by 1, expiring any triggers that
        are now too old.
        """
        self.time += 1

    def add(self, indices, values):
        """Add triggers in 'values' to the buffers indicated by the indices
        """
        for i, v in zip(indices, values):
            self.buffer[i] = numpy.append(self.buffer[i], v)
            self.buffer_expire[i] = numpy.append(self.buffer_expire[i], self.time)
        self.advance_time()

    def expire_vector(self, buffer_index):
        """Return the expiration vector of a given ring buffer """
        return self.buffer_expire[buffer_index]

    def data(self, buffer_index):
        """Return the data vector for a given ring buffer"""
        # Check for expired elements and discard if they exist
        expired = self.time - self.max_time
        exp = self.buffer_expire[buffer_index]
        j = 0
        while j < len(exp):
            # Everything before this j must be expired
            if exp[j] >= expired:
                self.buffer_expire[buffer_index] = exp[j:].copy()
                self.buffer[buffer_index] = self.buffer[buffer_index][j:].copy()
                break
            j += 1

        return self.buffer[buffer_index]

class CoincExpireBuffer(object):
    """Unordered dynamic sized buffer that handles
    multiple expiration vectors.
    """

    def __init__(self, expiration, ifos,
                       initial_size=2**20, dtype=numpy.float32):
        """
        expiration: int
            The 'time' in arbitrary integer units to allow to pass before
            removing an element.
        ifos: list of strs
            List of strings to identify the multiple data expiration times.
        initial_size: int, optional
            The initial size of the buffer.
            The dtype of each element of the buffer.
        """

        self.expiration = expiration
        self.buffer = numpy.zeros(initial_size, dtype=dtype)
        self.index = 0
        self.ifos = ifos

        self.time = {}
        self.timer = {}
        for ifo in self.ifos:
            self.time[ifo] = 0
            self.timer[ifo] = numpy.zeros(initial_size, dtype=numpy.int32)

        return self.index

    @property
        return self.buffer.nbytes

    def increment(self, ifos):
        """Increment without adding triggers"""
        self.add([], [], ifos)

    def remove(self, num):
        """Remove the the last 'num' elements from the buffer"""
        self.index -= num

    def add(self, values, times, ifos):
        """Add values to the internal buffer

        values: numpy.ndarray
            Array of elements to add to the internal buffer.
        times: dict of arrays
            The current time to use for each element being added.
            The set of timers to be incremented.
        """

        for ifo in ifos:
            self.time[ifo] += 1

        # Resize the internal buffer if we need more space
        if self.index + len(values) >= len(self.buffer):
            newlen = len(self.buffer) * 2
            for ifo in self.ifos:
                self.timer[ifo].resize(newlen)
            self.buffer.resize(newlen, refcheck=False)

        self.buffer[self.index:self.index+len(values)] = values
        if len(values) > 0:
                self.timer[ifo][self.index:self.index+len(values)] = times[ifo]

            self.index += len(values)

        # Remove the expired old elements
        keep = None
            kt = self.timer[ifo][:self.index] >= self.time[ifo] - self.expiration
            keep = numpy.logical_and(keep, kt) if keep is not None else kt

        self.buffer[:keep.sum()] = self.buffer[:self.index][keep]
            self.timer[ifo][:keep.sum()] = self.timer[ifo][:self.index][keep]
        self.index = keep.sum()

    def num_greater(self, value):
        """Return the number of elements larger than 'value'"""
        return (self.buffer[:self.index] > value).sum()

    @property
        """Return the array of elements"""
        return self.buffer[:self.index]

class LiveCoincTimeslideBackgroundEstimator(object):
    """Rolling buffer background estimation."""

    def __init__(self, num_templates, analysis_block, background_statistic,
                 sngl_ranking, stat_files, ifos,
                 ifar_limit=100,
                 timeslide_interval=.035,
                 coinc_threshold=.002,
                 return_background=False,
                 **kwargs):
        """
        num_templates: int
            The size of the template bank
        analysis_block: int
            The number of seconds in each analysis segment
        background_statistic: str
            The name of the statistic to rank coincident events.
        sngl_ranking: str
            The single detector ranking to use with the background statistic
        stat_files: list of strs
            List of filenames that contain information used to construct
            various coincident statistics.
            List of ifo names that are being analyzed. At the moment this must
            be two items such as ['H1', 'L1'].
        ifar_limit: float
            The largest inverse false alarm rate in years that we would like to
            calculate.
        timeslide_interval: float
            The time in seconds between consecutive timeslide offsets.
        coinc_threshold: float
            Amount of time allowed to form a coincidence in addition to the
            time of flight in seconds.
        return_background: boolean
            If true, background triggers will also be included in the file
            output.
        kwargs: dict
            Additional options for the statistic to use. See stat.py
            for more details on statistic options.
        """
        from . import stat
        self.num_templates = num_templates
        self.analysis_block = analysis_block

        stat_class = stat.get_statistic(background_statistic)
        self.stat_calculator = stat_class(
            sngl_ranking,
            stat_files,
            ifos=ifos,
            **kwargs
        )

        self.timeslide_interval = timeslide_interval
        self.return_background = return_background

        if len(self.ifos) != 2:
            raise ValueError("Only a two ifo analysis is supported at this time")

        self.lookback_time = (ifar_limit * lal.YRJUL_SI * timeslide_interval) ** 0.5
        self.buffer_size = int(numpy.ceil(self.lookback_time / analysis_block))

        det0, det1 = Detector(ifos[0]), Detector(ifos[1])
        self.time_window = det0.light_travel_time_to_detector(det1) + coinc_threshold
        self.coincs = CoincExpireBuffer(self.buffer_size, self.ifos)

        self.singles = {}

    def pick_best_coinc(cls, coinc_results):
        """Choose the best two-ifo coinc by ifar first, then statistic if needed.

        This function picks which of the available double-ifo coincs to use.
        It chooses the best (highest) ifar. The ranking statistic is used as
        a tie-breaker.
        A trials factor is applied if multiple types of coincs are possible
        at this time given the active ifos.

        coinc_results: list of coinc result dicts
            Dictionary by detector pair of coinc result dicts.

        Returns
        -------
        best: coinc results dict
            If there is a coinc, this will contain the 'best' one. Otherwise
            it will return the provided dict.
        """
        mstat = 0
        mifar = 0
        mresult = None

        # record the trials factor from the possible coincs we could
        # maximize over
        trials = 0
        for result in coinc_results:
            # Check that a coinc was possible. See the 'add_singles' method
            # to see where this flag was added into the results dict
            if 'coinc_possible' in result:
                trials += 1

                # Check that a coinc exists
                if 'foreground/ifar' in result:
                    ifar = result['foreground/ifar']
                    stat = result['foreground/stat']
                    if ifar > mifar or (ifar == mifar and stat > mstat):
                        mifar = ifar
                        mstat = stat
                        mresult = result

        # apply trials factor for the best coinc
        if mresult:
            mresult['foreground/ifar'] = mifar / float(trials)
            logging.info('Found %s coinc with ifar %s',
                         mresult['foreground/type'],
                         mresult['foreground/ifar'])
            return mresult
        # If no coinc, just return one of the results dictionaries. They will
        # all contain the same results (i.e. single triggers) in this case.
        else:
            return coinc_results[0]

    def from_cli(cls, args, num_templates, analysis_chunk, ifos):

        # Allow None inputs
        stat_files = args.statistic_files or []
        stat_keywords = args.statistic_keywords or []

        # flatten the list of lists of filenames to a single list (may be empty)
        stat_files = sum(stat_files, [])

        kwargs = stat.parse_statistic_keywords_opt(stat_keywords)

        return cls(num_templates, analysis_chunk,
                   args.ranking_statistic,
                   args.sngl_ranking,
                   stat_files,
                   return_background=args.store_background,
                   ifar_limit=args.background_ifar_limit,
                   timeslide_interval=args.timeslide_interval,
                   ifos=ifos,
                   **kwargs)

    def insert_args(parser):

        stat.insert_statistic_option_group(parser)

        group = parser.add_argument_group('Coincident Background Estimation')
        group.add_argument('--store-background', action='store_true',
            help="Return background triggers with zerolag coincidencs")
        group.add_argument('--background-ifar-limit', type=float,
            help="The limit on inverse false alarm rate to calculate "
                 "background in years", default=100.0)
        group.add_argument('--timeslide-interval', type=float,
            help="The interval between timeslides in seconds", default=0.1)
        group.add_argument('--ifar-remove-threshold', type=float,
            help="NOT YET IMPLEMENTED", default=100.0)

    @property
    def background_time(self):
        """Return the amount of background time that the buffers contain"""
        time = 1.0 / self.timeslide_interval
        for ifo in self.singles:
            time *= self.singles[ifo].filled_time * self.analysis_block
        return time

    def save_state(self, filename):
        """Save the current state of the background buffers"""
        from six.moves import cPickle
        cPickle.dump(self, filename)

    def restore_state(filename):
        """Restore state of the background buffers from a file"""
        return cPickle.load(filename)

    def ifar(self, coinc_stat):
        """Return the far that would be associated with the coincident given.
        """
        n = self.coincs.num_greater(coinc_stat)
        return self.background_time / lal.YRJUL_SI / (n + 1)

    def set_singles_buffer(self, results):
        """Create the singles buffer

        This creates the singles buffer for each ifo. The dtype is determined
        by a representative sample of the single triggers in the results.

        restuls: dict of dict
            Dict indexed by ifo and then trigger column.
        """
        # Determine the dtype from a sample of the data.
        self.singles_dtype = []
        data = False
            if ifo in results and results[ifo] is not False \
                    and len(results[ifo]['snr']):
                data = results[ifo]
                break

        if data is False:
            return

        for key in data:
            self.singles_dtype.append((key, data[key].dtype))

        if 'stat' not in data:
            self.singles_dtype.append(('stat', self.stat_calculator.single_dtype))

        # Create a ring buffer for each template ifo combination
            self.singles[ifo] = MultiRingBuffer(self.num_templates,
                                            self.buffer_size,
                                            self.singles_dtype)

    def _add_singles_to_buffer(self, results, ifos):
        """Add single detector triggers to the internal buffer

        results: dict of arrays
            Dictionary of dictionaries indexed by ifo and keys such as 'snr',
            'chisq', etc. The specific format it determined by the
            LiveBatchMatchedFilter class.

        Returns
        -------
        updated_singles: dict of numpy.ndarrays
            Array of indices that have been just updated in the internal
            buffers of single detector triggers.
        """
        if len(self.singles.keys()) == 0:
            self.set_singles_buffer(results)
        # If this *still* didn't work, no triggers in first set, try next time
            return {}

        # convert to single detector trigger values
        # FIXME Currently configured to use pycbc live output
        # where chisq is the reduced chisq and chisq_dof is the actual DOF
        logging.info("adding singles to the background estimate...")
        updated_indices = {}
            trigs = results[ifo]

            if len(trigs['snr'] > 0):
                trigsc = copy.copy(trigs)
                trigsc['chisq'] = trigs['chisq'] * trigs['chisq_dof']
                trigsc['chisq_dof'] = (trigs['chisq_dof'] + 2) / 2
                single_stat = self.stat_calculator.single(trigsc)
            else:
                single_stat = numpy.array([], ndmin=1,
                              dtype=self.stat_calculator.single_dtype)
            trigs['stat'] = single_stat

            # add each single detector trigger to the and advance the buffer
            data = numpy.zeros(len(single_stat), dtype=self.singles_dtype)
            for key, value in trigs.items():
                data[key] = value

            self.singles[ifo].add(trigs['template_id'], data)
            updated_indices[ifo] = trigs['template_id']
        return updated_indices

    def _find_coincs(self, results, ifos):
        """Look for coincs within the set of single triggers

        Returns
        -------
        coinc_results: dict of arrays
            A dictionary of arrays containing the coincident results.
        """
        # for each single detector trigger find the allowed coincidences
        # Record which template and the index of the single trigger
        # that forms each coincident trigger
        cstat = [[]]
        offsets = []
        ctimes = {self.ifos[0]:[], self.ifos[1]:[]}
        single_expire = {self.ifos[0]:[], self.ifos[1]:[]}
        template_ids = [[]]
        trigger_ids = {self.ifos[0]:[[]], self.ifos[1]:[[]]}

        # Calculate all the permutations of coincident triggers for each
        # new single detector trigger collected

            oifo = self.ifos[1] if self.ifos[0] == ifo else self.ifos[0]

            for i in range(len(trigs['end_time'])):
                trig_stat = trigs['stat'][i]
                trig_time = trigs['end_time'][i]
                template = trigs['template_id'][i]

                times = self.singles[oifo].data(template)['end_time']
                stats = self.singles[oifo].data(template)['stat']

                i1, _, slide = time_coincidence(times,
                                 numpy.array(trig_time, ndmin=1,
                                 dtype=numpy.float64),
                                 self.time_window,
                                 self.timeslide_interval)
                trig_stat = numpy.resize(trig_stat, len(i1))
                sngls_list = [[ifo, trig_stat],
                              [oifo, stats[i1]]]
                # This can only use 2-det coincs at present
                c = self.stat_calculator.rank_stat_coinc(
                    sngls_list,
                    slide,
                    self.timeslide_interval,
                    [0, -1]
                )
                offsets.append(slide)
                cstat.append(c)
                ctimes[oifo].append(times[i1])
                ctimes[ifo].append(numpy.zeros(len(c), dtype=numpy.float64))
                ctimes[ifo][-1].fill(trig_time)

                single_expire[oifo].append(self.singles[oifo].expire_vector(template)[i1])
                single_expire[ifo].append(numpy.zeros(len(c),
                                          dtype=numpy.int32))
                single_expire[ifo][-1].fill(self.singles[ifo].time - 1)

                # save the template and trigger ids to keep association
                # to singles. The trigger was just added so it must be in
                # the last position we mark this with -1 so the
                # slicing picks the right point
                template_ids.append(numpy.zeros(len(c)) + template)
                trigger_ids[oifo].append(i1)
                trigger_ids[ifo].append(numpy.zeros(len(c)) - 1)

        cstat = numpy.concatenate(cstat)
        template_ids = numpy.concatenate(template_ids).astype(numpy.int32)
            trigger_ids[ifo] = numpy.concatenate(trigger_ids[ifo]).astype(numpy.int32)

        # cluster the triggers we've found
        # (both zerolag and non handled together)
        num_zerolag = 0
        num_background = 0

        logging.info('%s background and zerolag coincs', len(cstat))
        if len(cstat) > 0:
            offsets = numpy.concatenate(offsets)
            ctime0 = numpy.concatenate(ctimes[self.ifos[0]]).astype(numpy.float64)
            ctime1 = numpy.concatenate(ctimes[self.ifos[1]]).astype(numpy.float64)

            cidx = cluster_coincs(cstat, ctime0, ctime1, offsets,
                                      self.timeslide_interval,
                                      self.analysis_block)
            offsets = offsets[cidx]
            zerolag_idx = (offsets == 0)
            bkg_idx = (offsets != 0)

                single_expire[ifo] = numpy.concatenate(single_expire[ifo])
                single_expire[ifo] = single_expire[ifo][cidx][bkg_idx]

            self.coincs.add(cstat[cidx][bkg_idx], single_expire, ifos)
            num_zerolag = zerolag_idx.sum()
            num_background = bkg_idx.sum()
        elif len(ifos) > 0:
            self.coincs.increment(ifos)

        ####################################Collect coinc results for saving
        coinc_results = {}
        # Save information about zerolag triggers
        if num_zerolag > 0:
            zerolag_results = {}
            idx = cidx[zerolag_idx][0]
            zerolag_cstat = cstat[cidx][zerolag_idx]
            zerolag_results['foreground/ifar'] = self.ifar(zerolag_cstat)
            zerolag_results['foreground/stat'] = zerolag_cstat
            template = template_ids[idx]
                trig_id = trigger_ids[ifo][idx]
                single_data = self.singles[ifo].data(template)[trig_id]
                for key in single_data.dtype.names:
                    path = 'foreground/%s/%s' % (ifo, key)
                    zerolag_results[path] = single_data[key]

            zerolag_results['foreground/type'] = '-'.join(self.ifos)

            coinc_results.update(zerolag_results)

        # Save some summary statistics about the background
        coinc_results['background/time'] = numpy.array([self.background_time])
        coinc_results['background/count'] = len(self.coincs.data)

        # Save all the background triggers
        if self.return_background:
            coinc_results['background/stat'] = self.coincs.data
        return num_background, coinc_results

    def backout_last(self, updated_singles, num_coincs):
        """Remove the recently added singles and coincs

        num_coincs: int
            The number of coincs that were just added to the internal buffer
            of coincident triggers
        """
        for ifo in updated_singles:
            self.singles[ifo].discard_last(updated_singles[ifo])
        self.coincs.remove(num_coincs)

    def add_singles(self, results):
        """Add singles to the background estimate and find candidates

        Returns
        -------
        """
        # Let's see how large everything is
        logging.info('BKG Coincs %s stored %s bytes',
                     len(self.coincs), self.coincs.nbytes)

        # If there are no results just return
        valid_ifos = [k for k in results.keys() if results[k] and k in self.ifos]
        if len(valid_ifos) == 0: return {}

        # Add single triggers to the internal buffer
        self._add_singles_to_buffer(results, ifos=valid_ifos)

        # Calculate zerolag and background coincidences
        _, coinc_results = self._find_coincs(results, ifos=valid_ifos)

        # record if a coinc is possible in this chunk
        if len(valid_ifos) == 2:
            coinc_results['coinc_possible'] = True

        return coinc_results

#
#
#                                   Preamble
#
#
""" This module contains functions for calculating expected rates of noise
    and signal coincidences.
"""

import itertools
import pycbc.detector

def multiifo_noise_lograte(log_rates, slop):
    """
    Calculate the expected rate of noise coincidences for multiple
    combinations of detectors

    log_rates: dict
        Key: ifo string, Value: sequence of log single-detector trigger rates,
        units assumed to be Hz
        time added to maximum time-of-flight between detectors to account
        for timing error

    Returns
    -------
    expected_log_rates: dict
        Key: ifo combination string
        Value: expected log coincidence rate in the combination, units log Hz
    """
    expected_log_rates = {}

    # Order of ifos must be stable in output dict keys, so sort them
    ifos = sorted(list(log_rates.keys()))
    ifostring = ' '.join(ifos)

    # Calculate coincidence for all-ifo combination
    expected_log_rates[ifostring] = \
        combination_noise_lograte(log_rates, slop)

    # If more than one possible coincidence type exists,
    # calculate coincidence for subsets through recursion
    if len(ifos) > 2:
        # Calculate rate for each 'miss-one-out' detector combination
        subsets = itertools.combinations(ifos, len(ifos) - 1)
        for subset in subsets:
            rates_subset = {}
            for ifo in subset:
                rates_subset[ifo] = log_rates[ifo]
            sub_coinc_rates = multiifo_noise_lograte(rates_subset, slop)
            # add these sub-coincidences to the overall dictionary
            for sub_coinc in sub_coinc_rates:
                expected_log_rates[sub_coinc] = sub_coinc_rates[sub_coinc]

    return expected_log_rates

def combination_noise_rate(rates, slop):
    """
    Calculate the expected rate of noise coincidences for a combination of
    detectors
    WARNING: for high stat values, numerical underflow can occur

    rates: dict
        Key: ifo string, Value: sequence of single-detector trigger rates,

    Returns
    -------
    numpy array
        Expected coincidence rate in the combination, units Hz
    """
    logging.warning('combination_noise_rate() is liable to numerical '
                    'underflows, use combination_noise_lograte '
                    'instead')
    log_rates = {k: numpy.log(r) for (k, r) in rates.items()}
    # exp may underflow
    return numpy.exp(combination_noise_lograte(log_rates, slop))

def combination_noise_lograte(log_rates, slop):
    """
    detectors given log of single detector noise rates

    Returns
    -------
        Expected log coincidence rate in the combination, units Hz
    """
    # multiply product of trigger rates by the overlap time
    allowed_area = multiifo_noise_coincident_area(list(log_rates), slop)
    # list(dict.values()) is python-3-proof
    rateprod = numpy.sum(list(log_rates.values()), axis=0)
    return numpy.log(allowed_area) + rateprod

def multiifo_noise_coincident_area(ifos, slop):
    """
    Calculate the total extent of time offset between 2 detectors,
    or area of the 2d space of time offsets for 3 detectors, for
    which a coincidence can be generated
    Cannot yet handle more than 3 detectors.

    ifos: list of strings
        list of interferometers
        extra time to add to maximum time-of-flight for timing error

    Returns
    -------
    allowed_area: float
        area in units of seconds^(n_ifos-1) that coincident values can fall in
    """
    # set up detector objects
    dets = {}
    for ifo in ifos:
        dets[ifo] = pycbc.detector.Detector(ifo)
    n_ifos = len(ifos)

    if n_ifos == 2:
        allowed_area = 2. * \
            (dets[ifos[0]].light_travel_time_to_detector(dets[ifos[1]]) + slop)
    elif n_ifos == 3:
        tofs = numpy.zeros(n_ifos)
        ifo2_num = []

        # calculate travel time between detectors (plus extra for timing error)
        # TO DO: allow for different timing errors between different detectors
        for i, ifo in enumerate(ifos):
            ifo2_num.append(int(numpy.mod(i + 1, n_ifos)))
            det0 = dets[ifo]
            det1 = dets[ifos[ifo2_num[i]]]
            tofs[i] = det0.light_travel_time_to_detector(det1) + slop

        # combine these to calculate allowed area
        allowed_area = 0
        for i, _ in enumerate(ifos):
            allowed_area += 2 * tofs[i] * tofs[ifo2_num[i]] - tofs[i]**2
    else:
        raise NotImplementedError("Not able to deal with more than 3 ifos")

    return allowed_area

def multiifo_signal_coincident_area(ifos):
    """
    Calculate the area in which signal time differences are physically allowed

    Returns
    -------
        area in units of seconds^(n_ifos-1) that coincident signals will occupy
    """

        det0 = pycbc.detector.Detector(ifos[0])
        det1 = pycbc.detector.Detector(ifos[1])
        allowed_area = 2 * det0.light_travel_time_to_detector(det1)
        dets = {}
        # set up detector objects
            dets[ifo] = pycbc.detector.Detector(ifo)

        # calculate travel time between detectors
            tofs[i] = det0.light_travel_time_to_detector(det1)

        # calculate allowed area
        phi_12 = numpy.arccos((tofs[0]**2 + tofs[1]**2 - tofs[2]**2)
                              / (2 * tofs[0] * tofs[1]))
        allowed_area = numpy.pi * tofs[0] * tofs[1] * numpy.sin(phi_12)
    else:

# self.option) any later version.
#
#

#
#
#                                   Preamble
#
#
"""This modules defines functions for clustering and thresholding timeseries to
produces event triggers
"""
import numpy, copy, os.path
from six.moves import cPickle

from pycbc.types import Array
from pycbc.scheme import schemed

from . import coinc, ranking

from .eventmgr_cython import findchirp_cluster_over_window_cython

@schemed("pycbc.events.threshold_")
def threshold(series, value):
    """Return list of values and indices values over threshold in series.
    """
    err_msg = "This function is a stub that should be overridden using the "
    err_msg += "scheme. You shouldn't be seeing this error!"
    raise ValueError(err_msg)

def threshold_only(series, value):
    """Return list of values and indices whose values in series are
       larger (in absolute value) than value
    """

# FIXME: This should be under schemed, but I don't understand that yet!
def threshold_real_numpy(series, value):
    arr = series.data
    locs = numpy.where(arr > value)[0]
    vals = arr[locs]
    return locs, vals

def threshold_and_cluster(series, threshold, window):
    """

def _threshold_cluster_factory(series):
    err_msg = "This class is a stub that should be overridden using the "

class ThresholdCluster(object):
    """Create a threshold and cluster engine

    series : complex64
      Input pycbc.types.Array (or subclass); it will be searched for
      points above threshold that are then clustered
    """
    def __new__(cls, *args, **kwargs):
        real_cls = _threshold_cluster_factory(*args, **kwargs)
        return real_cls(*args, **kwargs) # pylint:disable=not-callable

# The class below should serve as the parent for all schemed classes.
# The intention is that this class serves simply as the location for
# all documentation of the class and its methods, though that is not
# yet implemented.  Perhaps something along the lines of:
#
#    http://stackoverflow.com/questions/2025562/inherit-docstrings-in-python-class-inheritance
#
# will work? Is there a better way?
class _BaseThresholdCluster(object):
    def threshold_and_cluster(self, threshold, window):
        """
        Threshold and cluster the memory specified at instantiation with the
        threshold and window size specified at creation.

        -----------
        threshold : float32
          The minimum absolute value of the series given at object initialization
          to return when thresholding and clustering.
        window : uint32
          The size (in number of samples) of the window over which to cluster

        Returns:
        --------
        event_vals : complex64
          Numpy array, complex values of the clustered events
        event_locs : uint32
          Numpy array, indices into series of location of events
        """
        pass

def findchirp_cluster_over_window(times, values, window_length):
    """ Reduce the events by clustering over a window using
    the FindChirp clustering algorithm

    indices: Array
        The list of indices of the SNR values
    snr: Array
        The list of SNR value
    window_size: int
        The size of the window in integer samples. Must be positive.

    Returns
    -------
        The reduced list of indices of the SNR values
    """
    assert window_length > 0, 'Clustering window length is not positive'

    indices = numpy.zeros(len(times), dtype=numpy.int32)
    tlen = len(times)
    absvalues = numpy.array(abs(values), copy=False)
    times = numpy.array(times, dtype=numpy.int32, copy=False)
    k = findchirp_cluster_over_window_cython(times, absvalues, window_length,
                                             indices, tlen)

    return indices[0:k+1]

def cluster_reduce(idx, snr, window_size):
    """ Reduce the events by clustering over a window

    snr: Array
        The size of the window in integer samples.

    Returns
    -------
    snr: Array
        The list of SNR values
    """
    ind = findchirp_cluster_over_window(idx, snr, window_size)
    return idx.take(ind), snr.take(ind)

class EventManager(object):
    def __init__(self, opt, column, column_types, **kwds):
        self.opt = opt
        self.global_params = kwds

        self.event_dtype = [('template_id', int)]
        for col, coltype in zip(column, column_types):
            self.event_dtype.append((col, coltype))

        self.events = numpy.array([], dtype=self.event_dtype)
        self.accumulate = [self.events]
        self.template_params = []
        self.template_index = -1
        self.template_events = numpy.array([], dtype=self.event_dtype)
        self.write_performance = False

    def save_state(self, tnum_finished, filename):
        from pycbc.io.hdf import dump_state

        self.tnum_finished = tnum_finished
        logging.info('Writing checkpoint file at template %s', tnum_finished)
        fp = h5py.File(filename, 'w')
        dump_state(self, fp, protocol=cPickle.HIGHEST_PROTOCOL)
        fp.close()

        from pycbc.io.hdf import load_state

        fp = h5py.File(filename, 'r')
        try:
            mgr = load_state(fp)
            fp.close()
            raise e
        next_template = mgr.tnum_finished + 1
        logging.info('Restoring with checkpoint at template %s', next_template)
        return mgr.tnum_finished + 1, mgr

    def from_multi_ifo_interface(cls, opt, ifo, column, column_types, **kwds):
        """
        To use this for a single ifo from the multi ifo interface requires
        some small fixing of the opt structure. This does that. As we edit the
        opt structure the process_params table will not be correct.
        """
        opt = copy.deepcopy(opt)
        opt_dict = vars(opt)
        for arg, value in opt_dict.items():
            if isinstance(value, dict):
                setattr(opt, arg, getattr(opt, arg)[ifo])
        return cls(opt, column, column_types, **kwds)

    def chisq_threshold(self, value, num_bins, delta=0):
        remove = []
        for i, event in enumerate(self.events):
            xi = event['chisq'] / (event['chisq_dof'] +
                                   delta * event['snr'].conj() * event['snr'])
            if xi > value:
                remove.append(i)
        self.events = numpy.delete(self.events, remove)

    def newsnr_threshold(self, threshold):
        """ Remove events with newsnr smaller than given threshold
        """
        if not self.opt.chisq_bins:
            raise RuntimeError('Chi-square test must be enabled in order to '
                               'use newsnr threshold')

        nsnrs = ranking.newsnr(abs(self.events['snr']),
                               self.events['chisq'] / self.events['chisq_dof'])
        remove_idxs = numpy.where(nsnrs < threshold)[0]
        self.events = numpy.delete(self.events, remove_idxs)

    def keep_near_injection(self, window, injections):
        from pycbc.events.veto import indices_within_times
        if len(self.events) == 0:
            return

        inj_time = numpy.array(injections.end_times())
        gpstime = self.events['time_index'].astype(numpy.float64)
        gpstime = gpstime / self.opt.sample_rate + self.opt.gps_start_time
        i = indices_within_times(gpstime, inj_time - window, inj_time + window)
        self.events = self.events[i]

    def keep_loudest_in_interval(self, window, num_keep, statname="newsnr",
                                 log_chirp_width=None):
            return

        e_copy = self.events.copy()

        # Here self.events['snr'] is the complex SNR
        e_copy['snr'] = abs(e_copy['snr'])
        # Messy step because pycbc inspiral's internal 'chisq_dof' is 2p-2
        # but stat.py / ranking.py functions use 'chisq_dof' = p
        e_copy['chisq_dof'] = e_copy['chisq_dof'] / 2 + 1
        statv = ranking.get_sngls_ranking_from_trigs(e_copy, statname)

        # Convert trigger time to integer bin number
        # NB time_index and window are in units of samples
        wtime = (e_copy['time_index'] / window).astype(numpy.int32)
        bins = numpy.unique(wtime)

        if log_chirp_width:
            from pycbc.conversions import mchirp_from_mass1_mass2
            m1 = numpy.array([p['tmplt'].mass1 for p in self.template_params])
            m2 = numpy.array([p['tmplt'].mass2 for p in self.template_params])
            mc = mchirp_from_mass1_mass2(m1, m2)[e_copy['template_id']]

            # convert chirp mass to integer bin number
            imc = (numpy.log(mc) / log_chirp_width).astype(numpy.int32)
            cbins = numpy.unique(imc)

        keep = []
        for b in bins:
            if log_chirp_width:
                for b2 in cbins:
                    bloc = numpy.where((wtime == b) & (imc == b2))[0]
                    bloudest = statv[bloc].argsort()[-num_keep:]
                    keep.append(bloc[bloudest])
            else:
                bloc = numpy.where((wtime == b))[0]
                bloudest = statv[bloc].argsort()[-num_keep:]
                keep.append(bloc[bloudest])

        keep = numpy.concatenate(keep)
        self.events = self.events[keep]

    def add_template_events(self, columns, vectors):
        """ Add a vector indexed """
        # initialize with zeros - since vectors can be None, look for the
        # longest one that isn't
        new_events = None
        for v in vectors:
            if v is not None:
                new_events = numpy.zeros(len(v), dtype=self.event_dtype)
                break
        # they shouldn't all be None
        assert new_events is not None
        new_events['template_id'] = self.template_index
        for c, v in zip(columns, vectors):
                if isinstance(v, Array):
                    new_events[c] = v.numpy()
                else:
                    new_events[c] = v
        self.template_events = numpy.append(self.template_events, new_events)

    def cluster_template_events(self, tcolumn, column, window_size):
        """ Cluster the internal events over the named column
        """
        cvec = self.template_events[column]
        tvec = self.template_events[tcolumn]
        if window_size == 0:
            indices = numpy.arange(len(tvec))
        else:
            indices = findchirp_cluster_over_window(tvec, cvec, window_size)
        self.template_events = numpy.take(self.template_events, indices)

    def new_template(self, **kwds):
        self.template_params.append(kwds)
        self.template_index += 1

    def add_template_params(self, **kwds):
        self.template_params[-1].update(kwds)

    def finalize_template_events(self):
        self.accumulate.append(self.template_events)

    def consolidate_events(self, opt, gwstrain=None):
        self.events = numpy.concatenate(self.accumulate)
        logging.info("We currently have %d triggers", len(self.events))
        if opt.chisq_threshold and opt.chisq_bins:
            logging.info("Removing triggers with poor chisq")
            self.chisq_threshold(opt.chisq_threshold, opt.chisq_bins,
                                 opt.chisq_delta)
            logging.info("%d remaining triggers", len(self.events))

        if opt.newsnr_threshold and opt.chisq_bins:
            logging.info("Removing triggers with NewSNR below threshold")
            self.newsnr_threshold(opt.newsnr_threshold)

        if opt.keep_loudest_interval:
            logging.info("Removing triggers not within the top %s "
                         "loudest of a %s second interval by %s",
                         opt.keep_loudest_num, opt.keep_loudest_interval,
                         opt.keep_loudest_stat)
            self.keep_loudest_in_interval\
                (opt.keep_loudest_interval * opt.sample_rate,
                 opt.keep_loudest_num, statname=opt.keep_loudest_stat,
                 log_chirp_width=opt.keep_loudest_log_chirp_window)

        if opt.injection_window and hasattr(gwstrain, 'injections'):
            logging.info("Keeping triggers within %s seconds of injection",
                         opt.injection_window)
            self.keep_near_injection(opt.injection_window,
                                     gwstrain.injections)

    def finalize_events(self):

    def make_output_dir(self, outname):
        path = os.path.dirname(outname)
        if path != '':
            if not os.path.exists(path) and path is not None:
                os.makedirs(path)

    def save_performance(self, ncores, nfilters, ntemplates, run_time,
                         setup_time):
        """
        Calls variables from pycbc_inspiral to be used in a timing calculation
        """
        self.run_time = run_time
        self.setup_time = setup_time
        self.ncores = ncores
        self.nfilters = nfilters
        self.ntemplates = ntemplates
        self.write_performance = True

    def write_events(self, outname):
        """ Write the found events to a sngl inspiral table
        """
        self.make_output_dir(outname)

        if '.hdf' in outname:
            self.write_to_hdf(outname)
        else:
            raise ValueError('Cannot write to this format')

    def write_to_hdf(self, outname):
        class fw(object):
            def __init__(self, name, prefix):
                self.f = h5py.File(name, 'w')
                self.prefix = prefix

            def __setitem__(self, name, data):
                col = self.prefix + '/' + name
                self.f.create_dataset(col, data=data,
                                      compression='gzip',
                                      compression_opts=9,
                                      shuffle=True)

        self.events.sort(order='template_id')
        th = numpy.array([p['tmplt'].template_hash for p in
                          self.template_params])
        tid = self.events['template_id']
        f = fw(outname, self.opt.channel_name[0:2])

        if len(self.events):
            f['snr'] = abs(self.events['snr'])
            try:
                # Precessing
                f['u_vals'] = self.events['u_vals']
                f['coa_phase'] = self.events['coa_phase']
                f['hplus_cross_corr'] = self.events['hplus_cross_corr']
            except Exception:
                # Not precessing
                f['coa_phase'] = numpy.angle(self.events['snr'])
            f['chisq'] = self.events['chisq']
            f['bank_chisq'] = self.events['bank_chisq']
            f['bank_chisq_dof'] = self.events['bank_chisq_dof']
            f['cont_chisq'] = self.events['cont_chisq']
            f['end_time'] = self.events['time_index'] / \
                              float(self.opt.sample_rate) \
                            + self.opt.gps_start_time
            try:
                template_sigmasq_plus = numpy.array(
                             [t['sigmasq_plus'] for t in self.template_params],
                                                    dtype=numpy.float32)
                f['sigmasq_plus'] = template_sigmasq_plus[tid]
                template_sigmasq_cross = numpy.array(
                            [t['sigmasq_cross'] for t in self.template_params],
                                                     dtype=numpy.float32)
                f['sigmasq_cross'] = template_sigmasq_cross[tid]
                # FIXME: I want to put something here, but I haven't yet
                #        figured out what it should be. I think we would also
                #        need information from the plus and cross correlation
                #        (both real and imaginary(?)) to get this.
                f['sigmasq'] = template_sigmasq_plus[tid]
                template_sigmasq = numpy.array(
                                  [t['sigmasq'] for t in self.template_params],
                                               dtype=numpy.float32)
                f['sigmasq'] = template_sigmasq[tid]

            template_durations = [p['tmplt'].template_duration for p in
                                  self.template_params]
            f['template_duration'] = numpy.array(template_durations,
                                                 dtype=numpy.float32)[tid]

            # FIXME: Can we get this value from the autochisq instance?
            cont_dof = self.opt.autochi_number_points
            if self.opt.autochi_onesided is None:
                cont_dof = cont_dof * 2
            if self.opt.autochi_two_phase:
            if self.opt.autochi_max_valued_dof:
                cont_dof = self.opt.autochi_max_valued_dof
            f['cont_chisq_dof'] = numpy.repeat(cont_dof, len(self.events))

            if 'chisq_dof' in self.events.dtype.names:
                f['chisq_dof'] = self.events['chisq_dof'] / 2 + 1
            else:
                f['chisq_dof'] = numpy.zeros(len(self.events))

            f['template_hash'] = th[tid]

            if 'sg_chisq' in self.events.dtype.names:
                f['sg_chisq'] = self.events['sg_chisq']

            if self.opt.psdvar_segment is not None:
                f['psd_var_val'] = self.events['psd_var_val']

        if self.opt.trig_start_time:
            f['search/start_time'] = numpy.array([self.opt.trig_start_time])
            search_start_time = float(self.opt.trig_start_time)
        else:
            f['search/start_time'] = numpy.array([self.opt.gps_start_time +
                                                  self.opt.segment_start_pad])
            search_start_time = float(self.opt.gps_start_time +
                                      self.opt.segment_start_pad)
        if self.opt.trig_end_time:
            f['search/end_time'] = numpy.array([self.opt.trig_end_time])
            search_end_time = float(self.opt.trig_end_time)
        else:
            f['search/end_time'] = numpy.array([self.opt.gps_end_time -
                                                self.opt.segment_end_pad])
            search_end_time = float(self.opt.gps_end_time -
                                    self.opt.segment_end_pad)

        if self.write_performance:
            self.analysis_time = search_end_time - search_start_time
            time_ratio = numpy.array(
                [float(self.analysis_time) / float(self.run_time)])
            temps_per_core = float(self.ntemplates) / float(self.ncores)
            filters_per_core = float(self.nfilters) / float(self.ncores)
            f['search/templates_per_core'] = \
                numpy.array([float(temps_per_core) * float(time_ratio)])
            f['search/filter_rate_per_core'] = \
                numpy.array([filters_per_core / float(self.run_time)])
            f['search/setup_time_fraction'] = \
                numpy.array([float(self.setup_time) / float(self.run_time)])
            f['search/run_time'] = numpy.array([float(self.run_time)])

        if 'q_trans' in self.global_params:
            qtrans = self.global_params['q_trans']
            for key in qtrans:
                if key == 'qtiles':
                    for seg in qtrans[key]:
                        for q in qtrans[key][seg]:
                            f['qtransform/%s/%s/%s' % (key, seg, q)] = \
                                                            qtrans[key][seg][q]
                elif key == 'qplanes':
                        f['qtransform/%s/%s' % (key, seg)] = qtrans[key][seg]

        if 'gating_info' in self.global_params:
            gating_info = self.global_params['gating_info']
            for gate_type in ['file', 'auto']:
                if gate_type in gating_info:
                    f['gating/' + gate_type + '/time'] = \
                     numpy.array([float(g[0]) for g in gating_info[gate_type]])
                    f['gating/' + gate_type + '/width'] = \
                            numpy.array([g[1] for g in gating_info[gate_type]])
                    f['gating/' + gate_type + '/pad'] = \
                            numpy.array([g[2] for g in gating_info[gate_type]])

class EventManagerMultiDetBase(EventManager):
    def __init__(self, opt, ifos, column, column_types, psd=None, **kwargs):
        self.global_params = kwargs
        if psd is not None:
            self.global_params['psd'] = psd[ifos[0]]

        # The events array does not like holding the ifo as string,
        # so create a mapping dict and hold as an int
        self.ifo_dict = {}
        self.ifo_reverse = {}
            self.ifo_dict[ifo] = i
            self.ifo_reverse[i] = ifo

        self.event_dtype = [('template_id', int), ('event_id', int)]

        self.event_id_map = {}
        self.template_event_dict = {}
        self.coinc_list = []
            self.template_event_dict[ifo] = \
                numpy.array([], dtype=self.event_dtype)

    def add_template_events_to_ifo(self, ifo, columns, vectors):
        # Just call through to the standard function
        self.template_events = self.template_event_dict[ifo]
        self.add_template_events(columns, vectors)
        self.template_event_dict[ifo] = self.template_events
        self.template_events = None

class EventManagerCoherent(EventManagerMultiDetBase):
    def __init__(self, opt, ifos, column, column_types, network_column,
                 network_column_types, psd=None, **kwargs):
        super(EventManagerCoherent, self).__init__(
            opt, ifos, column, column_types, psd=None, **kwargs)
        self.network_event_dtype = \
            [(ifo + '_event_id', int) for ifo in self.ifos]
        self.network_event_dtype.append(('template_id', int))
        self.network_event_dtype.append(('event_id', int))
        for col, coltype in zip(network_column, network_column_types):
            self.network_event_dtype.append((col, coltype))
        self.network_events = numpy.array([], dtype=self.network_event_dtype)
        self.event_index = {}
            self.event_index[ifo] = 0
        self.event_index['network'] = 0
        self.template_event_dict['network'] = \
                                numpy.array([], dtype=self.network_event_dtype)

    def cluster_template_network_events(self, tcolumn, column, window_size):
        """
        cvec = self.template_event_dict['network'][column]
        tvec = self.template_event_dict['network'][tcolumn]
        else:
        for key in self.template_event_dict:
            self.template_event_dict[key] = numpy.take(
                self.template_event_dict[key], indices)

    def add_template_network_events(self, columns, vectors):
        new_events = numpy.zeros(
            max([len(v) for v in vectors if v is not None]),
            dtype=self.network_event_dtype
        )
                else:

    def add_template_events_to_network(self, columns, vectors):
        self.template_events = self.template_event_dict['network']
        self.add_template_network_events(columns, vectors)
        self.template_event_dict['network'] = self.template_events

            def __init__(self, name):

        f = fw(outname)
        # Output network stuff
        f.prefix = 'network'
        network_events = numpy.array([e for e in self.network_events],
                                     dtype=self.network_event_dtype)
        f['event_id'] = network_events['event_id']
        f['coherent_snr'] = network_events['coherent_snr']
        f['reweighted_snr'] = network_events['reweighted_snr']
        f['null_snr'] = network_events['null_snr']
        f['end_time_gc'] = network_events['time_index'] / \
                float(self.opt.sample_rate[self.ifos[0].lower()]) + \
                        self.opt.gps_start_time[self.ifos[0].lower()]
        f['nifo'] = network_events['nifo']
        f['latitude'] = network_events['latitude']
        f['longitude'] = network_events['longitude']
        f['template_id'] = network_events['template_id']
            # First add the ifo event ids to the network branch
            f[ifo + '_event_id'] = network_events[ifo + '_event_id']
        # Individual ifo stuff
        for i, ifo in enumerate(self.ifos):
            tid = self.events['template_id'][self.events['ifo'] == i]
            f.prefix = ifo
            ifo_events = numpy.array([e for e in self.events
                    if e['ifo'] == self.ifo_dict[ifo]], dtype=self.event_dtype)
            if len(ifo_events):
                ifo_str = ifo.lower()[0] if ifo != 'H1' else ifo.lower()
                f['snr_%s' % ifo_str] = abs(ifo_events['snr'])
                f['event_id'] = ifo_events['event_id']
                try:
                    # Precessing
                    f['u_vals'] = ifo_events['u_vals']
                    f['coa_phase'] = ifo_events['coa_phase']
                    f['hplus_cross_corr'] = ifo_events['hplus_cross_corr']
                except Exception:
                    f['coa_phase'] = numpy.angle(ifo_events['snr'])
                f['chisq'] = ifo_events['chisq']
                f['bank_chisq'] = ifo_events['bank_chisq']
                f['bank_chisq_dof'] = ifo_events['bank_chisq_dof']
                f['cont_chisq'] = ifo_events['cont_chisq']
                f['end_time'] = ifo_events['time_index'] / \
                        float(self.opt.sample_rate[ifo_str]) + \
                        self.opt.gps_start_time[ifo_str]
                f['time_index'] = ifo_events['time_index']
                try:
                    template_sigmasq_plus = numpy.array(
                        [t['sigmasq_plus'] for t in self.template_params],
                        dtype=numpy.float32
                    )
                    f['sigmasq_plus'] = template_sigmasq_plus[tid]
                    template_sigmasq_cross = numpy.array(
                        [t['sigmasq_cross'] for t in self.template_params],
                    )
                    f['sigmasq_cross'] = template_sigmasq_cross[tid]
                    # FIXME: I want to put something here, but I haven't yet
                    #      figured out what it should be. I think we would also
                    #      need information from the plus and cross correlation
                    #      (both real and imaginary(?)) to get this.
                    f['sigmasq'] = template_sigmasq_plus[tid]
                    # Not precessing
                    template_sigmasq = numpy.array(
                             [t['sigmasq'][ifo] for t in self.template_params],
                                                   dtype=numpy.float32)
                    f['sigmasq'] = template_sigmasq[tid]

                template_durations = [p['tmplt'].template_duration for p in
                                      self.template_params]
                f['template_duration'] = numpy.array(template_durations,
                                                     dtype=numpy.float32)[tid]

                # FIXME: Can we get this value from the autochisq instance?
                # cont_dof = self.opt.autochi_number_points
                # if self.opt.autochi_onesided is None:
                #     cont_dof = cont_dof * 2
                # if self.opt.autochi_two_phase:
                # if self.opt.autochi_max_valued_dof:
                #     cont_dof = self.opt.autochi_max_valued_dof
                # f['cont_chisq_dof'] = numpy.repeat(cont_dof, len(ifo_events))

                if 'chisq_dof' in ifo_events.dtype.names:
                    f['chisq_dof'] = ifo_events['chisq_dof'] / 2 + 1
                else:
                    f['chisq_dof'] = numpy.zeros(len(ifo_events))

                f['template_hash'] = th[tid]

            if self.opt.trig_start_time:
                f['search/start_time'] = numpy.array([
                             self.opt.trig_start_time[ifo]], dtype=numpy.int32)
                search_start_time = float(self.opt.trig_start_time[ifo])
            else:
                    self.opt.gps_start_time[ifo] +
                    self.opt.segment_start_pad[ifo]], dtype=numpy.int32)
                search_start_time = float(self.opt.gps_start_time[ifo] +
                                          self.opt.segment_start_pad[ifo])
            if self.opt.trig_end_time:
                f['search/end_time'] = numpy.array([
                        self.opt.trig_end_time[ifo]], dtype=numpy.int32)
                search_end_time = float(self.opt.trig_end_time[ifo])
            else:
                f['search/end_time'] = numpy.array(
                    [self.opt.gps_end_time[ifo] -
                     self.opt.segment_end_pad[ifo]], dtype=numpy.int32)
                search_end_time = float(self.opt.gps_end_time[ifo] -
                                        self.opt.segment_end_pad[ifo])

            if self.write_performance:
                self.analysis_time = search_end_time - search_start_time
                time_ratio = numpy.array([float(self.analysis_time) /
                                          float(self.run_time)])
                temps_per_core = float(self.ntemplates) / float(self.ncores)
                filters_per_core = float(self.nfilters) / float(self.ncores)
                f['search/templates_per_core'] = \
                    numpy.array([float(temps_per_core) * float(time_ratio)])
                f['search/filter_rate_per_core'] = \
                    numpy.array([filters_per_core / float(self.run_time)])
                f['search/setup_time_fraction'] = \
                   numpy.array([float(self.setup_time) / float(self.run_time)])

            if 'gating_info' in self.global_params:
                gating_info = self.global_params['gating_info']
                for gate_type in ['file', 'auto']:
                    if gate_type in gating_info:
                        f['gating/' + gate_type + '/time'] = numpy.array(
                                 [float(g[0]) for g in gating_info[gate_type]])
                        f['gating/' + gate_type + '/width'] = numpy.array(
                                        [g[1] for g in gating_info[gate_type]])
                        f['gating/' + gate_type + '/pad'] = numpy.array(
                                        [g[2] for g in gating_info[gate_type]])

        # Check that none of the template events have the same time index as an
        # existing event in events. I.e. don't list the same ifo event multiple
        # times when looping over sky points and time slides.
        existing_times = {}
        new_times = {}
        existing_template_id = {}
        new_template_id = {}
        existing_events_mask = {}
        new_template_event_mask = {}
        existing_template_event_mask = {}
            ifo_events = numpy.where(self.events['ifo'] == i)
            existing_times[ifo] = self.events['time_index'][ifo_events]
            new_times[ifo] = self.template_event_dict[ifo]['time_index']
            existing_template_id[ifo] = self.events['template_id'][ifo_events]
            new_template_id[ifo] = self.template_event_dict[ifo]['template_id']
            # This is true for each existing event that has the same time index
            # and template id as a template trigger.
            existing_events_mask[ifo] = numpy.argwhere(
                numpy.logical_and(
                    numpy.isin(existing_times[ifo], new_times[ifo]),
                    numpy.isin(existing_template_id[ifo], new_template_id[ifo])
                                 )).reshape(-1,)
            # This is true for each template event that has either a new
            # trigger time or a new template id.
            new_template_event_mask[ifo] = numpy.argwhere(
                numpy.logical_or(
                   ~numpy.isin(new_times[ifo], existing_times[ifo]),
                   ~numpy.isin(new_template_id[ifo], existing_template_id[ifo])
                                )).reshape(-1,)
            # This is true for each template event that has the same time index
            # and template id as an exisitng event trigger.
            existing_template_event_mask[ifo] = numpy.argwhere(
                    numpy.isin(new_times[ifo], existing_times[ifo]),
                    numpy.isin(new_template_id[ifo], existing_template_id[ifo])
            # Set ids (These show how each trigger in the single ifo trigger
            # list correspond to the network triggers)
            num_events = len(new_template_event_mask[ifo])
            new_event_ids = numpy.arange(self.event_index[ifo],
                                         self.event_index[ifo] + num_events)
            # Every template event that corresponds to a new trigger gets a new
            # id. Triggers that have been found before are not saved.
            self.template_event_dict[ifo]['event_id'][
                new_template_event_mask[ifo]] = new_event_ids
            self.template_event_dict['network'][ifo + '_event_id'][
            # Template events that have been found before get the event id of
            # the first time they were found.
                  existing_template_event_mask[ifo]] = \
                self.events[self.events['ifo'] == i][
                  existing_events_mask[ifo]]['event_id']
            self.event_index[ifo] = self.event_index[ifo] + num_events

        # Add the network event ids for the events with this template.
        num_events = len(self.template_event_dict['network'])
        new_event_ids = numpy.arange(self.event_index['network'],
                                     self.event_index['network'] + num_events)
        self.event_index['network'] = self.event_index['network'] + num_events
        self.template_event_dict['network']['event_id'] = new_event_ids
        # Move template events for each ifo to the events list
            self.events = numpy.append(
                self.events,
                self.template_event_dict[ifo][new_template_event_mask[ifo]]
            )
                                        numpy.array([], dtype=self.event_dtype)
        # Move the template events for the network to the network events list
        self.network_events = numpy.append(self.network_events,
                                   self.template_event_dict['network'])

class EventManagerMultiDet(EventManagerMultiDetBase):
        super(EventManagerMultiDet, self).__init__(
        self.event_index = 0

    def cluster_template_events_single_ifo(
            self, tcolumn, column, window_size, ifo):
        """
        self.cluster_template_events(tcolumn, column, window_size)

    def finalize_template_events(self, perform_coincidence=True,
                                 coinc_window=0.0):
        # Set ids
            num_events = len(self.template_event_dict[ifo])
            new_event_ids = numpy.arange(self.event_index,
                                         self.event_index+num_events)
            self.template_event_dict[ifo]['event_id'] = new_event_ids
            self.event_index = self.event_index+num_events

        if perform_coincidence:
            if not len(self.ifos) == 2:
                err_msg = "Coincidence currently only supported for 2 ifos."
                raise ValueError(err_msg)
            ifo1 = self.ifos[0]
            ifo2 = self.ifos[1]
            end_times1 = self.template_event_dict[ifo1]['time_index'] /\
              float(self.opt.sample_rate[ifo1]) + self.opt.gps_start_time[ifo1]
            end_times2 = self.template_event_dict[ifo2]['time_index'] /\
              float(self.opt.sample_rate[ifo2]) + self.opt.gps_start_time[ifo2]
            light_travel_time = Detector(ifo1).light_travel_time_to_detector(
                                                                Detector(ifo2))
            coinc_window = coinc_window + light_travel_time
            # FIXME: Remove!!!
            coinc_window = 2.0
            if len(end_times1) and len(end_times2):
                idx_list1, idx_list2, _ = \
                                 coinc.time_coincidence(end_times1, end_times2,
                                                        coinc_window)
                if len(idx_list1):
                    for idx1, idx2 in zip(idx_list1, idx_list2):
                        event1 = self.template_event_dict[ifo1][idx1]
                        event2 = self.template_event_dict[ifo2][idx2]
                        self.coinc_list.append((event1, event2))
            self.events = numpy.append(self.events,
                                       self.template_event_dict[ifo])
            self.template_event_dict[ifo] = numpy.array([],
                                                        dtype=self.event_dtype)

        """

        else:

                                                         self.template_params])
            ifo_events = numpy.array([e for e in self.events if
                                      e['ifo'] == self.ifo_dict[ifo]],
                                     dtype=self.event_dtype)
                try:
                          self.opt.gps_start_time[ifo_str]
                try:
                    template_sigmasq_plus = numpy.array([t['sigmasq_plus'] for
                               t in self.template_params], dtype=numpy.float32)
                    template_sigmasq_cross = numpy.array([t['sigmasq_cross']
                           for t in self.template_params], dtype=numpy.float32)
                    template_sigmasq = numpy.array([t['sigmasq'][ifo] for t in
                                                    self.template_params],

                f['template_duration'] = \
                      numpy.array(template_durations, dtype=numpy.float32)[tid]

                cont_dof = self.opt.autochi_number_points
                if self.opt.autochi_onesided is None:
                    cont_dof = cont_dof * 2
                f['cont_chisq_dof'] = numpy.repeat(cont_dof, len(ifo_events))

                else:

                if self.opt.psdvar_segment is not None:
                    f['psd_var_val'] = ifo_events['psd_var_val']

                f['search/start_time'] = numpy.array(
                            [self.opt.trig_start_time[ifo]], dtype=numpy.int32)
            else:
                          [self.opt.gps_start_time[ifo] +
                           self.opt.segment_start_pad[ifo]], dtype=numpy.int32)
                              [self.opt.trig_end_time[ifo]], dtype=numpy.int32)
            else:
                            [self.opt.gps_end_time[ifo] -
                             self.opt.segment_end_pad[ifo]], dtype=numpy.int32)

                time_ratio = numpy.array(
                            [float(self.analysis_time) / float(self.run_time)])

__all__ = ['threshold_and_cluster', 'findchirp_cluster_over_window',
           'threshold', 'cluster_reduce', 'ThresholdCluster',
           'threshold_real_numpy', 'threshold_only',
           'EventManager', 'EventManagerMultiDet', 'EventManagerCoherent']

""" This module contains functions for calculating single-ifo ranking
statistic values
"""
from six import raise_from

def effsnr(snr, reduced_x2, fac=250.):
    """Calculate the effective SNR statistic. See (S5y1 paper) for definition.
    """
    snr = numpy.array(snr, ndmin=1, dtype=numpy.float64)
    rchisq = numpy.array(reduced_x2, ndmin=1, dtype=numpy.float64)
    esnr = snr / (1 + snr ** 2 / fac) ** 0.25 / rchisq ** 0.25

    # If snr input is float, return a float. Otherwise return numpy array.
    if hasattr(snr, '__len__'):
        return esnr
    else:
        return esnr[0]

def newsnr(snr, reduced_x2, q=6., n=2.):
    """Calculate the re-weighted SNR statistic ('newSNR') from given SNR and
    reduced chi-squared values. See http://arxiv.org/abs/1208.3491 for
    definition. Previous implementation in glue/ligolw/lsctables.py
    """
    nsnr = numpy.array(snr, ndmin=1, dtype=numpy.float64)
    reduced_x2 = numpy.array(reduced_x2, ndmin=1, dtype=numpy.float64)

    # newsnr is only different from snr if reduced chisq > 1
    ind = numpy.where(reduced_x2 > 1.)[0]
    nsnr[ind] *= (0.5 * (1. + reduced_x2[ind] ** (q/n))) ** (-1./q)

        return nsnr
    else:
        return nsnr[0]

def newsnr_sgveto(snr, brchisq, sgchisq):
    """ Combined SNR derived from NewSNR and Sine-Gaussian Chisq"""
    nsnr = numpy.array(newsnr(snr, brchisq), ndmin=1)
    sgchisq = numpy.array(sgchisq, ndmin=1)
    t = numpy.array(sgchisq > 4, ndmin=1)
    if len(t):
        nsnr[t] = nsnr[t] / (sgchisq[t] / 4.0) ** 0.5

    else:

def newsnr_sgveto_psdvar(snr, brchisq, sgchisq, psd_var_val,
                         min_expected_psdvar=0.65):
    """ Combined SNR derived from SNR, reduced Allen chisq, sine-Gaussian chisq and
    PSD variation statistic"""
    # If PSD var is lower than the 'minimum usually expected value' stop this
    # being used in the statistic. This low value might arise because a
    # significant fraction of the "short" PSD period was gated (for instance).
    psd_var_val = numpy.array(psd_var_val, copy=True)
    psd_var_val[psd_var_val < min_expected_psdvar] = 1.
    scaled_snr = snr * (psd_var_val ** -0.5)
    scaled_brchisq = brchisq * (psd_var_val ** -1.)
    nsnr = newsnr_sgveto(scaled_snr, scaled_brchisq, sgchisq)

    else:

def newsnr_sgveto_psdvar_threshold(snr, brchisq, sgchisq, psd_var_val,
                                   min_expected_psdvar=0.65,
                                   brchisq_threshold=10.0,
                                   psd_var_val_threshold=10.0):
    """ newsnr_sgveto_psdvar with thresholds applied.

    This is the newsnr_sgveto_psdvar statistic with additional options
    to threshold on chi-squared or PSD variation.
    """
    nsnr = newsnr_sgveto_psdvar(snr, brchisq, sgchisq, psd_var_val,
                                min_expected_psdvar=min_expected_psdvar)
    nsnr = numpy.array(nsnr, ndmin=1)
    nsnr[brchisq > brchisq_threshold] = 1.
    nsnr[psd_var_val > psd_var_val_threshold] = 1.

    else:

def newsnr_sgveto_psdvar_scaled(snr, brchisq, sgchisq, psd_var_val,
                                scaling=0.33, min_expected_psdvar=0.65):
    """ Combined SNR derived from NewSNR, Sine-Gaussian Chisq and scaled PSD
    variation statistic. """
    nsnr = numpy.array(newsnr_sgveto(snr, brchisq, sgchisq), ndmin=1)
    psd_var_val = numpy.array(psd_var_val, ndmin=1, copy=True)

    # Default scale is 0.33 as tuned from analysis of data from O2 chunks
    nsnr = nsnr / psd_var_val ** scaling

    else:

def newsnr_sgveto_psdvar_scaled_threshold(snr, bchisq, sgchisq, psd_var_val,
                                          threshold=2.0):
    """ Combined SNR derived from NewSNR and Sine-Gaussian Chisq, and
    scaled psd variation.
    """
    nsnr = newsnr_sgveto_psdvar_scaled(snr, bchisq, sgchisq, psd_var_val)
    nsnr[bchisq > threshold] = 1.

    else:

def get_snr(trigs):
    """
    Return SNR from a trigs/dictionary object

    trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
        Dictionary-like object holding single detector trigger information.
        'snr' is a required key

    Returns
    -------
    numpy.ndarray
        Array of snr values
    """
    return numpy.array(trigs['snr'][:], ndmin=1, dtype=numpy.float32)

def get_newsnr(trigs):
    """
    Calculate newsnr ('reweighted SNR') for a trigs/dictionary object

        'chisq_dof', 'snr', and 'chisq' are required keys

    Returns
    -------
        Array of newsnr values
    """
    dof = 2. * trigs['chisq_dof'][:] - 2.
    nsnr = newsnr(trigs['snr'][:], trigs['chisq'][:] / dof)
    return numpy.array(nsnr, ndmin=1, dtype=numpy.float32)

def get_newsnr_sgveto(trigs):
    """
    Calculate newsnr re-weigthed by the sine-gaussian veto

        'chisq_dof', 'snr', 'sg_chisq' and 'chisq' are required keys

    Returns
    -------
    """
    nsnr_sg = newsnr_sgveto(trigs['snr'][:],
                            trigs['chisq'][:] / dof,
                            trigs['sg_chisq'][:])
    return numpy.array(nsnr_sg, ndmin=1, dtype=numpy.float32)

def get_newsnr_sgveto_psdvar(trigs):
    """
    Calculate snr re-weighted by Allen chisq, sine-gaussian veto and
    psd variation statistic

    trigs: dict of numpy.ndarrays
        Dictionary holding single detector trigger information.
    'chisq_dof', 'snr', 'chisq' and 'psd_var_val' are required keys

    Returns
    -------
     numpy.ndarray
    """
    nsnr_sg_psd = \
                 newsnr_sgveto_psdvar(trigs['snr'][:], trigs['chisq'][:] / dof,
                                      trigs['sg_chisq'][:],
                                      trigs['psd_var_val'][:])
    return numpy.array(nsnr_sg_psd, ndmin=1, dtype=numpy.float32)

def get_newsnr_sgveto_psdvar_threshold(trigs):
    """
    Calculate newsnr re-weighted by the sine-gaussian veto and scaled

    Returns
    -------
    """
    nsnr_sg_psdt = newsnr_sgveto_psdvar_threshold(
        trigs['snr'][:], trigs['chisq'][:] / dof,
        trigs['sg_chisq'][:],
        trigs['psd_var_val'][:]
    )
    return numpy.array(nsnr_sg_psdt, ndmin=1, dtype=numpy.float32)

def get_newsnr_sgveto_psdvar_scaled(trigs):
    """

    Returns
    -------
    """
    nsnr_sg_psdscale = \
                 newsnr_sgveto_psdvar_scaled(
                     trigs['snr'][:], trigs['chisq'][:] / dof,
                     trigs['sg_chisq'][:],
                     trigs['psd_var_val'][:])
    return numpy.array(nsnr_sg_psdscale, ndmin=1, dtype=numpy.float32)

def get_newsnr_sgveto_psdvar_scaled_threshold(trigs):
    """
    psd variation statistic. A further threshold is applied to the
    reduced chisq.

    Returns
    -------
    """
    nsnr_sg_psdt = \
                 newsnr_sgveto_psdvar_scaled_threshold(

sngls_ranking_function_dict = {
    'snr': get_snr,
    'newsnr': get_newsnr,
    'new_snr': get_newsnr,
    'newsnr_sgveto': get_newsnr_sgveto,
    'newsnr_sgveto_psdvar': get_newsnr_sgveto_psdvar,
    'newsnr_sgveto_psdvar_threshold': get_newsnr_sgveto_psdvar_threshold,
    'newsnr_sgveto_psdvar_scaled': get_newsnr_sgveto_psdvar_scaled,
    'newsnr_sgveto_psdvar_scaled_threshold': get_newsnr_sgveto_psdvar_scaled_threshold,
}

def get_sngls_ranking_from_trigs(trigs, statname, **kwargs):
    """
    Return ranking for all trigs given a statname.

    Compute the single-detector ranking for a list of input triggers for a
    specific statname.

    statname:
        The statistic to use.
    """
    # Identify correct function
    try:
        sngl_func = sngls_ranking_function_dict[statname]
    except KeyError as exc:
        err_msg = 'Single-detector ranking {} not recognized'.format(statname)
        raise_from(ValueError(err_msg), exc)

    # NOTE: In the sngl_funcs all the kwargs are explicitly stated, so any
    #       kwargs sent here must be known to the function.
    return sngl_func(trigs, **kwargs)

""" utilities for assigning FAR to single detector triggers
"""
from pycbc.events import ranking, trigger_fits as fits
from pycbc.types import MultiDetOptionAction
from pycbc import conversions as conv
from pycbc import bin_utils

class LiveSingle(object):
    def __init__(self, ifo,
                 newsnr_threshold=10.0,
                 reduced_chisq_threshold=5,
                 duration_threshold=0,
                 fit_file=None,
                 sngl_ifar_est_dist=None,
                 fixed_ifar=None):
        self.ifo = ifo
        if sngl_ifar_est_dist in ["fixed", None]:
            fit_bins = None
            fit_rates = None
            fit_thresh = None
            fit_coeffs = None
        else:
            with h5py.File(fit_file, 'r') as fit_file:
                bin_edges = fit_file['bins_edges'][:]
                fit_bins = bin_utils.IrregularBins(bin_edges)
                dist_grp = fit_file[ifo + '/' + sngl_ifar_est_dist]
                live_time = fit_file[ifo].attrs['live_time']
                fit_rates = dist_grp['counts'][:] / live_time
                fit_coeffs = dist_grp['fit_coeff'][:]
                fit_thresh = fit_file.attrs['fit_threshold']
        self.thresholds = {
            "newsnr": newsnr_threshold,
            "reduced_chisq": reduced_chisq_threshold,
            "duration": duration_threshold}
        self.fit_info = {
            "fixed_ifar": fixed_ifar,
            "bins": fit_bins,
            "rates": fit_rates,
            "coeffs": fit_coeffs,
            "thresh": fit_thresh}

        parser.add_argument('--single-newsnr-threshold', nargs='+',
                            type=float, action=MultiDetOptionAction,
                            help='Newsnr min threshold for single triggers. '
                                 'Can be given as a single value or as '
                                 'detector-value pairs, e.g. H1:6 L1:7 V1:6.5')
        parser.add_argument('--single-reduced-chisq-threshold', nargs='+',
                            help='Maximum reduced chi-squared threshold for '
                                 'single triggers. Can be given as a single '
                                 'value or as detector-value pairs, e.g. '
                                 'H1:2 L1:2 V1:3')
        parser.add_argument('--single-duration-threshold', nargs='+',
                            help='Minimum duration threshold for single '
                                 'triggers. Can be given as a single value '
                                 'or as detector-value pairs, e.g. H1:6 L1:6 '
                                 'V1:8')
        parser.add_argument('--single-fixed-ifar', nargs='+',
                            help='A fixed value for IFAR, still uses cuts '
                                 'defined by command line. Can be given as '
                                 'a single value or as detector-value pairs, '
                                 'e.g. H1:0.001 L1:0.001 V1:0.0005')
        parser.add_argument('--single-fit-file',
                            help='File which contains definitons of fit '
                                 'coefficients and counts for specific '
                                 'single trigger IFAR fitting.')
        parser.add_argument('--sngl-ifar-est-dist', nargs='+',
                            default='conservative',
                            choices=['conservative', 'mean', 'fixed'],
                            action=MultiDetOptionAction,
                            help='Which trigger distribution to use when '
                                 'calculating IFAR of single triggers. '
                                 'Default conservative. Can be given as '
                                 'e.g. H1:mean L1:mean V1:conservative')

    def from_cli(cls, args, ifo):
        sngl_opts_required = all([args.single_fit_file,
                                  args.single_reduced_chisq_threshold,
                                  args.single_duration_threshold,
                                  args.single_newsnr_threshold])
        if args.enable_single_detector_background and not sngl_opts_required:
            raise RuntimeError("Single detector trigger options "
                               "(--single-fit-file, "
                               "--single-reduced-chisq-threshold, "
                               "--single-duration-threshold, "
                               "--single-newsnr-threshold) "
                               "must ALL be given if single detector "
                               "background is enabled")
        return cls(
           ifo, newsnr_threshold=args.single_newsnr_threshold[ifo],
           reduced_chisq_threshold=args.single_reduced_chisq_threshold[ifo],
           duration_threshold=args.single_duration_threshold[ifo],
           fixed_ifar=args.single_fixed_ifar,
           fit_file=args.single_fit_file,
           sngl_ifar_est_dist=args.sngl_ifar_est_dist[ifo]
           )

    def check(self, trigs, data_reader):
        """ Look for a single detector trigger that passes the thresholds in
        the current data.
        """

        if len(trigs['snr']) == 0:
            return None

        # Apply cuts to trigs before clustering
        # Cut on snr so that triggers which could not reach newsnr
        # threshold do not have newsnr calculated
        valid_idx = (trigs['template_duration'] >
                     self.thresholds['duration']) & \
                    (trigs['chisq'] <
                     self.thresholds['reduced_chisq']) & \
                    (trigs['snr'] >
                     self.thresholds['newsnr'])
        if not np.any(valid_idx):

        cutdurchi_trigs = {k: trigs[k][valid_idx] for k in trigs}

        # This uses the pycbc live convention of chisq always meaning the
        # reduced chisq.
        nsnr_all = ranking.newsnr(cutdurchi_trigs['snr'],
                                  cutdurchi_trigs['chisq'])
        nsnr_idx = nsnr_all > self.thresholds['newsnr']
        if not np.any(nsnr_idx):

        cutall_trigs = {k: cutdurchi_trigs[k][nsnr_idx]
                        for k in trigs}

        # 'cluster' by taking the maximal newsnr value over the trigger set
        i = nsnr_all[nsnr_idx].argmax()

        nsnr = nsnr_all[nsnr_idx][i]
        dur = cutall_trigs['template_duration'][i]

        # create the coincidence
        fake_coinc = {'foreground/%s/%s' % (self.ifo, k):
                      cutall_trigs[k][i] for k in trigs}
        fake_coinc['foreground/stat'] = nsnr
        fake_coinc['foreground/ifar'] = self.calculate_ifar(nsnr, dur)
        fake_coinc['HWINJ'] = data_reader.near_hwinj()

        return fake_coinc

    def calculate_ifar(self, newsnr, duration):
        if self.fit_info['fixed_ifar']:
            return self.fit_info['fixed_ifar'][self.ifo]
        dur_bin = self.fit_info['bins'][duration]
        rate = self.fit_info['rates'][dur_bin]
        coeff = self.fit_info['coeffs'][dur_bin]
        rate_louder = rate * fits.cum_fit('exponential', [newsnr],
                                          coeff, self.fit_info['thresh'])[0]
        # apply a trials factor of the number of duration bins
        rate_louder *= len(self.fit_info['rates'])
        return conv.sec_to_year(1. / rate_louder)

# Copyright (C) 2016 Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
"""
This module contains functions for calculating coincident ranking statistic
values.
"""
from . import ranking
from . import coinc_rate

class Stat(object):
    """Base class which should be extended to provide a coincident statistic"""

    def __init__(self, sngl_ranking, files=None, ifos=None, **kwargs):
        """
        Create a statistic class instance

            The name of the ranking to use for the single-detector triggers.
        files: list of strs, needed for some statistics
            A list containing the filenames of hdf format files used to help
            construct the coincident statistics. The files must have a 'stat'
            attribute which is used to associate them with the appropriate
            statistic class.
        ifos: list of strs, needed for some statistics
            The list of detector names
        """
        import h5py

        self.files = {}
        files = files or []
        for filename in files:
            f = h5py.File(filename, 'r')
            stat = f.attrs['stat']
            if hasattr(stat, 'decode'):
                stat = stat.decode()
            if stat in self.files:
                raise RuntimeError("We already have one file with stat attr ="
                                   " %s. Can't provide more than one!" % stat)
            logging.info("Found file %s for stat %s", filename, stat)
            self.files[stat] = f

        # Provide the dtype of the single detector method's output
        # This is used by background estimation codes that need to maintain
        # a buffer of such values.
        self.single_dtype = numpy.float32
        # True if a larger single detector statistic will produce a larger
        # coincident statistic
        self.single_increasing = True

        self.ifos = ifos or []

        self.sngl_ranking = sngl_ranking
        self.sngl_ranking_kwargs = {}
        for key, value in kwargs.items():
            if key.startswith('sngl_ranking_'):
                self.sngl_ranking_kwargs[key[13:]] = value

    def get_sngl_ranking(self, trigs):
        """
        Returns the ranking for the single detector triggers.

        trigs: dict of numpy.ndarrays, h5py group (or similar dict-like object)
            Dictionary-like object holding single detector trigger information.

        Returns
        -------
            The array of single detector values
        """
        return ranking.get_sngls_ranking_from_trigs(
            trigs,
            self.sngl_ranking,
            **self.sngl_ranking_kwargs
        )

    def single(self, trigs): # pylint:disable=unused-argument
        """
        Calculate the necessary single detector information

        Returns
        -------
        """
        err_msg = "This function is a stub that should be overridden by the "
        err_msg += "sub-classes. You shouldn't be seeing this error!"
        raise NotImplementedError(err_msg)

    def rank_stat_single(self, single_info):
        """
        Calculate the statistic for a single detector candidate

        single_info: tuple
            Tuple containing two values. The first is the ifo (str) and the
            second is the single detector triggers.

        Returns
        -------
            The array of single detector statistics
        """

    def rank_stat_coinc(self, s, slide, step, to_shift,
                        **kwargs): # pylint:disable=unused-argument
        """
        Calculate the coincident detection statistic.
        """

    def _check_coinc_lim_subclass(self, allowed_names):
        """
        Check that we are not using coinc_lim_for_thresh when not valid.

        coinc_lim_for_thresh is only defined for the statistic it is present
        in. If we subclass, we must check explicitly that it is still valid and
        indicate this in the code. If the code does not have this explicit
        check you will see the failure message here.

        allowed_names : list
            list of allowed classes for the specific sub-classed method.
        """
        if type(self).__name__ not in allowed_names:
            err_msg = "This is being called from a subclass which has not "
            err_msg += "been checked for validity with this method. If it is "
            err_msg += "valid for the subclass to come here, include in the "
            err_msg += "list of allowed_names above."
            raise NotImplementedError(err_msg)

    def coinc_lim_for_thresh(self, s, thresh, limifo,
                             **kwargs): # pylint:disable=unused-argument
        """
        Optimization function to identify coincs too quiet to be of interest

        Calculate the required single detector statistic to exceed
        the threshold for each of the input triggers.
        """

class QuadratureSumStatistic(Stat):
    """Calculate the quadrature sum coincident detection statistic"""

    def single(self, trigs):
        """

        Here just the ranking is computed and returned.

        Returns
        -------
        """
        return self.get_sngl_ranking(trigs)

        """

        Returns
        -------
        """
        return self.single(single_info[1])

    def rank_stat_coinc(self, sngls_list, slide, step, to_shift,
        """

        sngls_list: list
            List of (ifo, single detector statistic) tuples
        slide: (unused in this statistic)
        step: (unused in this statistic)
        to_shift: list
            List of integers indicating what multiples of the time shift will
            be applied (unused in this statistic)

        Returns
        -------
            Array of coincident ranking statistic values
        """
        cstat = sum(sngl[1] ** 2. for sngl in sngls_list) ** 0.5
        # For single-detector "cuts" the single ranking is set to -1
        for sngls in sngls_list:
            cstat[sngls == -1] = 0
        return cstat

        """

        s: list
            List of (ifo, single detector statistic) tuples for all detectors
            except limifo.
        thresh: float
            The threshold on the coincident statistic.
        limifo: string
            The ifo for which the limit is to be found.

        Returns
        -------
            Array of limits on the limifo single statistic to
            exceed thresh.
        """
        # Safety against subclassing and not rethinking this
        allowed_names = ['QuadratureSumStatistic']
        self._check_coinc_lim_subclass(allowed_names)

        s0 = thresh ** 2. - sum(sngl[1] ** 2. for sngl in s)
        s0[s0 < 0] = 0
        return s0 ** 0.5

class PhaseTDStatistic(QuadratureSumStatistic):
    """
    Statistic that re-weights combined newsnr using coinc parameters.

    The weighting is based on the PDF of time delays, phase differences and
    amplitude ratios between triggers in different ifos.
    """

        """

        files: list of strs, unused here

        ifos: list of strs, needed here
        """

        QuadratureSumStatistic.__init__(self, sngl_ranking, files=files,
                                        ifos=ifos, **kwargs)

        self.single_dtype = [
            ('snglstat', numpy.float32),
            ('coa_phase', numpy.float32),
            ('end_time', numpy.float64),
            ('sigmasq', numpy.float32),
            ('snr', numpy.float32)
        ]

        # Assign attribute so that it can be replaced with other functions
        self.has_hist = False
        self.hist_ifos = None
        self.ref_snr = 5.0
        self.relsense = {}
        self.swidth = self.pwidth = self.twidth = None
        self.srbmin = self.srbmax = None
        self.max_penalty = None
        self.pdtype = []
        self.weights = {}
        self.param_bin = {}
        self.two_det_flag = (len(ifos) == 2)
        self.two_det_weights = {}

    def get_hist(self, ifos=None):
        """
        Read in a signal density file for the ifo combination

        ifos: list
            The list of ifos. Needed if not given when initializing the class
            instance.
        """

        ifos = ifos or self.ifos

        selected = None
        for name in self.files:
            # Pick out the statistic files that provide phase / time/ amp
            # relationships and match to the ifos in use
            if 'phasetd_newsnr' in name:
                ifokey = name.split('_')[2]
                num = len(ifokey) / 2
                if num != len(ifos):
                    continue

                match = [ifo in ifokey for ifo in ifos]
                if False in match:
                    continue
                selected = name
                break

        if selected is None:
            raise RuntimeError("Couldn't figure out which stat file to use")

        logging.info("Using signal histogram %s for ifos %s", selected, ifos)
        histfile = self.files[selected]
        self.hist_ifos = histfile.attrs['ifos']
        n_ifos = len(self.hist_ifos)

        # Histogram bin attributes
        self.twidth = histfile.attrs['twidth']
        self.pwidth = histfile.attrs['pwidth']
        self.swidth = histfile.attrs['swidth']
        self.srbmin = histfile.attrs['srbmin']
        self.srbmax = histfile.attrs['srbmax']

        bin_volume = (self.twidth * self.pwidth * self.swidth) ** (n_ifos - 1)
        self.hist_max = - 1. * numpy.inf

        # Read histogram for each ifo, to use if that ifo has smallest SNR in
        # the coinc
        for ifo in self.hist_ifos:

            weights = histfile[ifo]['weights'][:]
            # renormalise to PDF
            self.weights[ifo] = weights / (weights.sum() * bin_volume)

            param = histfile[ifo]['param_bin'][:]

            if param.dtype == numpy.int8:
                # Older style, incorrectly sorted histogram file
                ncol = param.shape[1]
                self.pdtype = [('c%s' % i, param.dtype) for i in range(ncol)]
                self.param_bin[ifo] = numpy.zeros(len(self.weights[ifo]),
                                                  dtype=self.pdtype)
                for i in range(ncol):
                    self.param_bin[ifo]['c%s' % i] = param[:, i]

                lsort = self.param_bin[ifo].argsort()
                self.param_bin[ifo] = self.param_bin[ifo][lsort]
                self.weights[ifo] = self.weights[ifo][lsort]
            else:
                # New style, efficient histogram file
                # param bin and weights have already been sorted
                self.param_bin[ifo] = param
                self.pdtype = self.param_bin[ifo].dtype

            # Max_penalty is a small number to assigned to any bins without
            # histogram entries. All histograms in a given file have the same
            # min entry by design, so use the min of the last one read in.
            self.max_penalty = self.weights[ifo].min()
            self.hist_max = max(self.hist_max, self.weights[ifo].max())

            if self.two_det_flag:
                # The density of signals is computed as a function of 3 binned
                # parameters: time difference (t), phase difference (p) and
                # SNR ratio (s). These are computed for each combination of
                # detectors, so for detectors 6 differences are needed. However
                # many combinations of these parameters are highly unlikely and
                # no instances of these combinations occurred when generating
                # the statistic files. Rather than storing a bunch of 0s, these
                # values are just not stored at all. This reduces the size of
                # the statistic file, but means we have to identify the correct
                # value to read for every trigger. For 2 detectors we can
                # expand the weights lookup table here, basically adding in all
                # the "0" values. This makes looking up a value in the
                # "weights" table a O(N) rather than O(NlogN) operation. It
                # sacrifices RAM to do this, so is a good tradeoff for 2
                # detectors, but not for 3!
                if not hasattr(self, 'c0_size'):
                    self.c0_size = {}
                    self.c1_size = {}
                    self.c2_size = {}

                self.c0_size[ifo] = 2 * (abs(self.param_bin[ifo]['c0']).max()
                                         + 1)
                self.c1_size[ifo] = 2 * (abs(self.param_bin[ifo]['c1']).max()
                                         + 1)
                self.c2_size[ifo] = 2 * (abs(self.param_bin[ifo]['c2']).max()
                                         + 1)

                array_size = [self.c0_size[ifo], self.c1_size[ifo],
                              self.c2_size[ifo]]
                dtypec = self.weights[ifo].dtype
                self.two_det_weights[ifo] = \
                    numpy.zeros(array_size, dtype=dtypec) + self.max_penalty
                id0 = self.param_bin[ifo]['c0'].astype(numpy.int32) \
                    + self.c0_size[ifo] // 2
                id1 = self.param_bin[ifo]['c1'].astype(numpy.int32) \
                    + self.c1_size[ifo] // 2
                id2 = self.param_bin[ifo]['c2'].astype(numpy.int32) \
                    + self.c2_size[ifo] // 2
                self.two_det_weights[ifo][id0, id1, id2] = self.weights[ifo]

        relfac = histfile.attrs['sensitivity_ratios']
        for ifo, sense in zip(self.hist_ifos, relfac):
            self.relsense[ifo] = sense

        self.has_hist = True

    def logsignalrate(self, stats, shift, to_shift):
        """
        Calculate the normalized log rate density of signals via lookup

        stats: dict of dicts
            Single-detector quantities for each detector
        shift: numpy array of float
            Time shift vector for each coinc to be ranked
        to_shift: list of ints
            Multiple of the time shift to apply, ordered as self.ifos

        Returns
        -------
        value: log of coinc signal rate density for the given single-ifo
            triggers and time shifts
        """
        # Convert time shift vector to dict, as hist ifos and self.ifos may
        # not be in same order
        to_shift = {ifo: s for ifo, s in zip(self.ifos, to_shift)}

        if not self.has_hist:
            self.get_hist()

        # Figure out which ifo of the contributing ifos has the smallest SNR,
        # to use as reference for choosing the signal histogram.
        snrs = numpy.array([numpy.array(stats[ifo]['snr'], ndmin=1)
                           for ifo in self.ifos])
        smin = numpy.argmin(snrs, axis=0)
        # Store a list of the triggers using each ifo as reference
        rtypes = {ifo: numpy.where(smin == j)[0]
                  for j, ifo in enumerate(self.ifos)}

        # Get reference ifo information
        rate = numpy.zeros(len(shift), dtype=numpy.float32)
        for ref_ifo in self.ifos:
            rtype = rtypes[ref_ifo]
            ref = stats[ref_ifo]
            pref = numpy.array(ref['coa_phase'], ndmin=1)[rtype]
            tref = numpy.array(ref['end_time'], ndmin=1)[rtype]
            sref = numpy.array(ref['snr'], ndmin=1)[rtype]
            sigref = numpy.array(ref['sigmasq'], ndmin=1) ** 0.5
            sigref = sigref[rtype]
            senseref = self.relsense[self.hist_ifos[0]]

            binned = []
            other_ifos = [ifo for ifo in self.ifos if ifo != ref_ifo]
            for ifo in other_ifos:
                sc = stats[ifo]
                p = numpy.array(sc['coa_phase'], ndmin=1)[rtype]
                t = numpy.array(sc['end_time'], ndmin=1)[rtype]
                s = numpy.array(sc['snr'], ndmin=1)[rtype]

                sense = self.relsense[ifo]
                sig = numpy.array(sc['sigmasq'], ndmin=1) ** 0.5
                sig = sig[rtype]

                # Calculate differences
                pdif = (pref - p) % (numpy.pi * 2.0)
                tdif = shift[rtype] * to_shift[ref_ifo] + \
                    tref - shift[rtype] * to_shift[ifo] - t
                sdif = s / sref * sense / senseref * sigref / sig

                # Put into bins
                tbin = (tdif / self.twidth).astype(numpy.int)
                pbin = (pdif / self.pwidth).astype(numpy.int)
                sbin = (sdif / self.swidth).astype(numpy.int)
                binned += [tbin, pbin, sbin]

            # Convert binned to same dtype as stored in hist
            nbinned = numpy.zeros(len(pbin), dtype=self.pdtype)
            for i, b in enumerate(binned):
                nbinned['c%s' % i] = b

            # Read signal weight from precalculated histogram
                # High-RAM, low-CPU option for two-det
                rate[rtype] = numpy.zeros(len(nbinned)) + self.max_penalty

                id0 = nbinned['c0'].astype(numpy.int32) \
                    + self.c0_size[ref_ifo] // 2
                id1 = nbinned['c1'].astype(numpy.int32) \
                    + self.c1_size[ref_ifo] // 2
                id2 = nbinned['c2'].astype(numpy.int32) \
                    + self.c2_size[ref_ifo] // 2

                # look up keys which are within boundaries
                within = (id0 > 0) & (id0 < self.c0_size[ref_ifo])
                within = within & (id1 > 0) & (id1 < self.c1_size[ref_ifo])
                within = within & (id2 > 0) & (id2 < self.c2_size[ref_ifo])
                within = numpy.where(within)[0]
                rate[rtype[within]] = \
                    self.two_det_weights[ref_ifo][id0[within], id1[within],
                                                  id2[within]]
            else:
                # Low[er]-RAM, high[er]-CPU option for >two det
                loc = numpy.searchsorted(self.param_bin[ref_ifo], nbinned)
                loc[loc == len(self.weights[ref_ifo])] = 0
                rate[rtype] = self.weights[ref_ifo][loc]

                # These weren't in our histogram so give them max penalty
                # instead of random value
                missed = numpy.where(
                    self.param_bin[ref_ifo][loc] != nbinned
                )[0]
                rate[rtype[missed]] = self.max_penalty

            # Scale by signal population SNR
            rate[rtype] *= (sref / self.ref_snr) ** -4.0

        return numpy.log(rate)

        """

        Here the ranking as well as phase, endtime and sigma-squared values.

        trigs: dict of numpy.ndarrays, h5py group or similar dict-like object
            Object holding single detector trigger information. 'snr', 'chisq',
            'chisq_dof', 'coa_phase', 'end_time', and 'sigmasq' are required
            keys.

        Returns
        -------
            Array of single detector parameter values
        """
        sngl_stat = self.get_sngl_ranking(trigs)
        singles = numpy.zeros(len(sngl_stat), dtype=self.single_dtype)
        singles['snglstat'] = sngl_stat
        singles['coa_phase'] = trigs['coa_phase'][:]
        singles['end_time'] = trigs['end_time'][:]
        singles['sigmasq'] = trigs['sigmasq'][:]
        singles['snr'] = trigs['snr'][:]
        return numpy.array(singles, ndmin=1)

        """

        Returns
        -------
        """

                        **kwargs):  # pylint:disable=unused-argument
        """
        Calculate the coincident detection statistic, defined in Eq 2 of
        [Nitz et al, 2017](https://doi.org/10.3847/1538-4357/aa8f50).
        """
        rstat = sum(s[1]['snglstat'] ** 2 for s in sngls_list)
        cstat = rstat + 2. * self.logsignalrate(dict(sngls_list),
                                                slide * step,
                                                to_shift)
        cstat[cstat < 0] = 0
        return cstat ** 0.5

    def coinc_lim_for_thresh(self, sngls_list, thresh, limifo,
                             **kwargs):  # pylint:disable=unused-argument
        """
        Optimization function to identify coincs too quiet to be of interest.
        Calculate the required single detector statistic to exceed the
        threshold for each of the input triggers.
        """
        allowed_names = ['PhaseTDStatistic']

        lim_stat = [b['snglstat'] for a, b in sngls_list if a == limifo][0]
        s1 = thresh ** 2. - lim_stat ** 2.
        # Assume best case scenario and use maximum signal rate
        s1 -= 2. * self.hist_max
        s1[s1 < 0] = 0
        return s1 ** 0.5

class ExpFitStatistic(QuadratureSumStatistic):
    """
    Detection statistic using an exponential falloff noise model.

    Statistic approximates the negative log noise coinc rate density per
    template over single-ifo newsnr values.
    """

        """

        files: list of strs, needed here

        ifos: list of strs, not used here
        """

        if not files:
            raise RuntimeError("Statistic files not specified")

        # the stat file attributes are hard-coded as '%{ifo}-fit_coeffs'
        parsed_attrs = [f.split('-') for f in self.files.keys()]
        self.bg_ifos = [at[0] for at in parsed_attrs if
                       (len(at) == 2 and at[1] == 'fit_coeffs')]
        if not len(self.bg_ifos):
            raise RuntimeError("None of the statistic files has the required "
                               "attribute called {ifo}-fit_coeffs !")
        self.fits_by_tid = {}
        self.alphamax = {}
        for i in self.bg_ifos:
            self.fits_by_tid[i] = self.assign_fits(i)
            self.get_ref_vals(i)

        self.single_increasing = False

    def assign_fits(self, ifo):
        """
        Extract fits from fit files

        ifo: str
            The detector to get fits for.

        Returns
        -------
        rate_dict: dict
            A dictionary containing the fit information in the `alpha`, `rate`
            and `thresh` keys/.
        """
        coeff_file = self.files[ifo+'-fit_coeffs']
        template_id = coeff_file['template_id'][:]
        alphas = coeff_file['fit_coeff'][:]
        rates = coeff_file['count_above_thresh'][:]
        # the template_ids and fit coeffs are stored in an arbitrary order
        # create new arrays in template_id order for easier recall
        tid_sort = numpy.argsort(template_id)
        return {'alpha': alphas[tid_sort],
                'rate': rates[tid_sort],
                'thresh': coeff_file.attrs['stat_threshold']
                }

    def get_ref_vals(self, ifo):
        """
        Get the largest `alpha` value over all templates for given ifo.

        This is stored in `self.alphamax[ifo]` in the class instance.

        ifo: str
        """
        self.alphamax[ifo] = self.fits_by_tid[ifo]['alpha'].max()

    def find_fits(self, trigs):
        """
        Get fit coeffs for a specific ifo and template id(s)

            The coincidence executable will always call this using a bunch of
            trigs from a single template, there template_num is stored as an
            attribute and we just return the single value for all templates.
            If multiple templates are in play we must return arrays.

        Returns
        --------
        alphai: float or numpy array
            The alpha fit value(s)
        ratei: float or numpy array
            The rate fit value(s)
        thresh: float or numpy array
            The thresh fit value(s)
        """

        try:
            tnum = trigs.template_num  # exists if accessed via coinc_findtrigs
            ifo = trigs.ifo
        except AttributeError:
            tnum = trigs['template_id']  # exists for SingleDetTriggers
            assert len(self.ifos) == 1
            # Should be exactly one ifo provided
            ifo = self.ifos[0]
        # fits_by_tid is a dictionary of dictionaries of arrays
        # indexed by ifo / coefficient name / template_id
        alphai = self.fits_by_tid[ifo]['alpha'][tnum]
        ratei = self.fits_by_tid[ifo]['rate'][tnum]
        thresh = self.fits_by_tid[ifo]['thresh']
        return alphai, ratei, thresh

    def lognoiserate(self, trigs):
        """
        Calculate the log noise rate density over single-ifo ranking

        Read in single trigger information, compute the ranking
        and rescale by the fitted coefficients alpha and rate

        Returns
        ---------
        lognoisel: numpy.array
            Array of log noise rate density for each input trigger.

        """
        alphai, ratei, thresh = self.find_fits(trigs)
        # alphai is constant of proportionality between single-ifo newsnr and
        #   negative log noise likelihood in given template
        # ratei is rate of trigs in given template compared to average
        # thresh is stat threshold used in given ifo
        lognoisel = - alphai * (sngl_stat - thresh) + numpy.log(alphai) + \
                      numpy.log(ratei)
        return numpy.array(lognoisel, ndmin=1, dtype=numpy.float32)

        """

        In this case the ranking rescaled (see the lognoiserate method here).

        Returns
        -------
        """

        return self.lognoiserate(trigs)

        """

        Returns
        -------
        """
        err_msg = "Sorry! No-one has implemented this method yet! "

        """
        """

        """
        """

    # Keeping this here to help write the new coinc method.
    def coinc_OLD(self, s0, s1, slide, step): # pylint:disable=unused-argument
        """Calculate the final coinc ranking statistic"""

        # Approximate log likelihood ratio by summing single-ifo negative
        # log noise likelihoods
        loglr = - s0 - s1
        # add squares of threshold stat values via idealized Gaussian formula
        threshes = [self.fits_by_tid[i]['thresh'] for i in self.bg_ifos]
        loglr += sum([t**2. / 2. for t in threshes])
        # convert back to a coinc-SNR-like statistic
        # via log likelihood ratio \propto rho_c^2 / 2
        return (2. * loglr) ** 0.5

    # Keeping this here to help write the new coinc_lim method
    def coinc_lim_for_thresh_OLD(self, s0, thresh):
        """Calculate the required single detector statistic to exceed

        s0: numpy.ndarray
            Single detector ranking statistic for the first detector.

        Returns
        -------
            Array of limits on the second detector single statistic to
        exceed thresh.
        """
        s1 = - (thresh ** 2.) / 2. - s0
        s1 += sum([t**2. / 2. for t in threshes])
        return s1

class ExpFitCombinedSNR(ExpFitStatistic):
    """
    Reworking of ExpFitStatistic designed to resemble network SNR

    Use a monotonic function of the negative log noise rate density which
    approximates combined (new)snr for coincs with similar newsnr in each ifo
    """

        """

        """

        ExpFitStatistic.__init__(self, sngl_ranking, files=files, ifos=ifos,
                                 **kwargs)
        # for low-mass templates the exponential slope alpha \approx 6
        self.alpharef = 6.

    def use_alphamax(self):
        """
        Compute the reference alpha from the fit files.

        Use the harmonic mean of the maximum individual ifo slopes as the
        reference value of alpha.
        """

        inv_alphas = [1. / self.alphamax[i] for i in self.bg_ifos]
        self.alpharef = 1. / (sum(inv_alphas) / len(inv_alphas))

        """

        Returns
        -------
        """

        logr_n = self.lognoiserate(trigs)
        _, _, thresh = self.find_fits(trigs)
        # shift by log of reference slope alpha
        logr_n += -1. * numpy.log(self.alpharef)
        # add threshold and rescale by reference slope
        stat = thresh - (logr_n / self.alpharef)
        return numpy.array(stat, ndmin=1, dtype=numpy.float32)

        """
        Calculate the statistic for single detector candidates

        Returns
        -------
        """
        sngl_rnk = self.single(single_info[1])
        if self.single_increasing:
            sngl_multiifo = sngl_rnk['snglstat']
        else:
            sngl_multiifo = -1.0 * sngl_rnk['snglstat']
        return sngl_multiifo

        """

        Returns
        -------
        """

        # scale by 1/sqrt(number of ifos) to resemble network SNR
        return sum(sngl[1] for sngl in s) / len(s)**0.5

        """

        s: list

        Returns
        -------
        """
        allowed_names = ['ExpFitCombinedSNR']

        return thresh * ((len(s) + 1) ** 0.5) - sum(sngl[1] for sngl in s)

class PhaseTDExpFitStatistic(PhaseTDStatistic, ExpFitCombinedSNR):
    """
    Statistic combining exponential noise model with signal histogram PDF
    """

    # default is 2-ifo operation with exactly 1 'phasetd' file
        """

        """

        # read in both foreground PDF and background fit info
        ExpFitCombinedSNR.__init__(self, sngl_ranking, files=files, ifos=ifos,
                                   **kwargs)
        # need the self.single_dtype value from PhaseTDStatistic
        PhaseTDStatistic.__init__(self, sngl_ranking, files=files,
                                  ifos=ifos, **kwargs)

        """

        In this case the ranking rescaled (see the lognoiserate method here)
        with the phase, end time, sigma and SNR values added in.

        Returns
        -------
        """

        # same single-ifo stat as ExpFitCombinedSNR
        sngl_stat = ExpFitCombinedSNR.single(self, trigs)

        """

        Returns
        -------
        """

        """
        """

        """
        """

    # Keeping the old statistic code here for now to help with reimplementing
    def coinc_OLD(self, s0, s1, slide, step):
        # logsignalrate function inherited from PhaseTDStatistic
        logr_s = self.logsignalrate(s0, s1, slide * step)
        # rescale by ExpFitCombinedSNR reference slope as for sngl stat
        cstat = s0['snglstat'] + s1['snglstat'] + logr_s / self.alpharef
        # cut off underflowing and very small values
        cstat[cstat < 8.] = 8.
        # scale to resemble network SNR
        return cstat / (2.**0.5)

        # if the threshold is below this value all triggers will
        # pass because of rounding in the coinc method
        if thresh <= (8. / (2.**0.5)):
            return -1. * numpy.ones(len(s0['snglstat'])) * numpy.inf
        logr_s = self.hist_max
        s1 = (2 ** 0.5) * thresh - s0['snglstat'] - logr_s / self.alpharef
        return s1

class ExpFitSGBgRateStatistic(ExpFitStatistic):
    """

    Statistic calculates the log noise coinc rate for each
    """

    def __init__(self, sngl_ranking, files=None, ifos=None,
                 benchmark_lograte=-14.6, **kwargs):
        """

        benchmark_lograte: float, default=-14.6
            benchmark_lograte is log of a representative noise trigger rate.
            The default comes from H1L1 (O2) and is 4.5e-7 Hz.
        """

        super(ExpFitSGBgRateStatistic, self).__init__(sngl_ranking,
                                                      files=files, ifos=ifos,
                                                      **kwargs)
        self.benchmark_lograte = benchmark_lograte
        self.get_newsnr = ranking.get_newsnr_sgveto

        # Reassign the rate to be number per time rather than an arbitrarily
        # normalised number
        for ifo in self.bg_ifos:
            self.reassign_rate(ifo)

    def reassign_rate(self, ifo):
        """
        Reassign the rate to be number per time rather

        Reassign the rate to be number per time rather than an arbitrarily
        normalised number.

        ifo: str
            The ifo to consider.
        """
        # create arrays in template_id order for easier recall
        self.fits_by_tid[ifo]['rate'] = \
            coeff_file['count_above_thresh'][:][tid_sort] / \
            float(coeff_file.attrs['analysis_time'])

        """

        Returns
        -------
        """

        # ranking statistic is -ln(expected rate density of noise triggers)
        # plus normalization constant
        sngl_dict = {sngl[0]: sngl[1] for sngl in s}
        ln_noise_rate = coinc_rate.combination_noise_lograte(
                                  sngl_dict, kwargs['time_addition'])
        loglr = - ln_noise_rate + self.benchmark_lograte
        return loglr

    def coinc_lim_for_thresh(self, s, thresh, limifo, **kwargs):
        """

        s: list

        Returns
        -------
        """

        allowed_names = ['ExpFitSGBgRateStatistic']

        sngl_dict[limifo] = numpy.zeros(len(s[0][1]))
        loglr = - thresh - ln_noise_rate + self.benchmark_lograte

class ExpFitSGFgBgNormStatistic(PhaseTDStatistic,
                                ExpFitSGBgRateStatistic):
    """
    Statistic combining PhaseTD, ExpFitSGBg and additional foreground info.
    """

                 reference_ifos='H1,L1', **kwargs):
        """

        reference_ifos: string of comma separated ifo prefixes
            Detectors to be used as the reference network for network
            sensitivity comparisons. Each must be in fits_by_tid
        """

        # read in background fit info and store it
        ExpFitSGBgRateStatistic.__init__(self, sngl_ranking, files=files,
                                         ifos=ifos, **kwargs)
        # if ifos not already set, determine via background fit info
        self.ifos = self.ifos or self.bg_ifos
        # PhaseTD statistic single_dtype plus network sensitivity benchmark
                                  ifos=self.ifos, **kwargs)
        self.single_dtype.append(('benchmark_logvol', numpy.float32))

            self.assign_median_sigma(ifo)

        ref_ifos = reference_ifos.split(',')

        # benchmark_logvol is a benchmark sensitivity array over template id
        hl_net_med_sigma = numpy.amin([self.fits_by_tid[ifo]['median_sigma']
                                       for ifo in ref_ifos], axis=0)
        self.benchmark_logvol = 3.0 * numpy.log(hl_net_med_sigma)

    def assign_median_sigma(self, ifo):
        """
        Read and sort the median_sigma values from input files.

        ifo: str
        """

        coeff_file = self.files[ifo + '-fit_coeffs']
        self.fits_by_tid[ifo]['median_sigma'] = \
            coeff_file['median_sigma'][:][tid_sort]

    def lognoiserate(self, trigs, alphabelow=6):
        """

        Read in single trigger information, make the newsnr statistic

        alphabelow: float, default=6
            Use this slope to fit the noise triggers below the point at which
            fits are present in the input files.

        Returns
        ---------
        """
        newsnr = self.get_newsnr(trigs)
        # Above the threshold we use the usual fit coefficient (alpha)
        # below threshold use specified alphabelow
        bt = newsnr < thresh
        lognoisel = - alphai * (newsnr - thresh) + numpy.log(alphai) + \
                        numpy.log(ratei)
        lognoiselbt = - alphabelow * (newsnr - thresh) + \
                           numpy.log(alphabelow) + numpy.log(ratei)
        lognoisel[bt] = lognoiselbt[bt]

        """

        with the phase, end time, sigma, SNR, template_id and the
        benchmark_logvol values added in.

        Returns
        -------
        """

        # single-ifo stat = log of noise rate
        sngl_stat = self.lognoiserate(trigs)
        # populate other fields to calculate phase/time/amp consistency
        # and sigma comparison
        try:
            # Should only be one ifo fit file provided
        # Store benchmark log volume as single-ifo information since the coinc
        # method does not have access to template id
        singles['benchmark_logvol'] = self.benchmark_logvol[tnum]

        """

        Returns
        -------
        """
        sngls = self.single(single_info[1])

        ln_noise_rate = sngls['snglstat']
        ln_noise_rate -= self.benchmark_lograte
        network_sigmasq = sngls['sigmasq']
        network_logvol = 1.5 * numpy.log(network_sigmasq)
        benchmark_logvol = sngls['benchmark_logvol']
        network_logvol -= benchmark_logvol
        ln_s = -4 * numpy.log(sngls['snr'] / self.ref_snr)
        loglr = network_logvol - ln_noise_rate + ln_s
        loglr[loglr < -30.] = -30.

        """

        Returns
        -------
        """

        sngl_rates = {sngl[0]: sngl[1]['snglstat'] for sngl in s}
                                  sngl_rates, kwargs['time_addition'])

        # Network sensitivity for a given coinc type is approximately
        # determined by the least sensitive ifo
        network_sigmasq = numpy.amin([sngl[1]['sigmasq'] for sngl in s],
                                     axis=0)
        # Volume \propto sigma^3 or sigmasq^1.5
        # Get benchmark log volume as single-ifo information :
        # benchmark_logvol for a given template is not ifo-dependent, so
        # choose the first ifo for convenience
        benchmark_logvol = s[0][1]['benchmark_logvol']

        # Use prior histogram to get Bayes factor for signal vs noise
        # given the time, phase and SNR differences between IFOs

        # First get signal PDF logr_s
        stat = {ifo: st for ifo, st in s}
        logr_s = self.logsignalrate(stat, slide * step, to_shift)

        # Find total volume of phase-time-amplitude space occupied by noise
        # coincs
        # Extent of time-difference space occupied
        noise_twindow = coinc_rate.multiifo_noise_coincident_area(
                            self.hist_ifos, kwargs['time_addition'])
        # Volume is the allowed time difference window, multiplied by 2pi for
        # each phase difference dimension and by allowed range of SNR ratio
        # for each SNR ratio dimension : there are (n_ifos - 1) dimensions
        # for both phase and SNR
        hist_vol = noise_twindow * \
            (2 * numpy.pi * (self.srbmax - self.srbmin) * self.swidth) ** \
            (n_ifos - 1)
        # Noise PDF is 1/volume, assuming a uniform distribution of noise
        # coincs
        logr_n = - numpy.log(hist_vol)

        # Combine to get final statistic: log of
        # ((rate of signals / rate of noise) * PTA Bayes factor)
        loglr = network_logvol - ln_noise_rate + logr_s - logr_n

        """

        s: list

        Returns
        -------
        """

        allowed_names = ['ExpFitSGFgBgNormStatistic']

        if thresh <= -30:
            return numpy.ones(len(s[0][1]['snglstat'])) * numpy.inf
        # Add limifo to singles dict so that overlap time is calculated correctly
        sngl_rates[limifo] = numpy.zeros(len(s[0][1]))

        # Assume best case and use the maximum sigma squared from all triggers
        network_sigmasq = numpy.ones(len(s[0][1])) * kwargs['max_sigmasq']

        logr_s = numpy.log(self.hist_max
                           * (kwargs['min_snr'] / self.ref_snr) ** -4.0)

        # coincs
        # coincs

        loglr = - thresh + network_logvol - ln_noise_rate + logr_s - logr_n

class ExpFitSGPSDFgBgNormBBHStatistic(ExpFitSGFgBgNormStatistic):
    """
    The ExpFitSGFgBgNormStatistic with a mass weighting factor.

    This is the same as the ExpFitSGFgBgNormStatistic except the likelihood
    is multiplied by a signal rate prior modelled as uniform over chirp mass.
    As templates are distributed roughly according to mchirp^(-11/3) we
    weight by the inverse of this. This ensures that loud events at high mass
    where template density is sparse are not swamped by events at lower masses
    where template density is high.
    """

                 max_chirp_mass=None, **kwargs):
        """

        max_chirp_mass: float, default=None
            If given, if a template's chirp mass is above this value it will
            be reweighted as if it had this chirp mass. This is to avoid the
            problem where the distribution fails to be accurate at high mass
            and we can have a case where a single highest-mass template might
            produce *all* the loudest background (and foreground) events.
        """
        ExpFitSGFgBgNormStatistic.__init__(self, sngl_ranking, files=files,
                                           ifos=ifos, **kwargs)
        self.get_newsnr = ranking.get_newsnr_sgveto_psdvar
        self.mcm = max_chirp_mass
        self.curr_mchirp = None

        """

        This calls back to the Parent class and then applies the chirp mass
        weighting factor.

        stats: list of dicts giving single-ifo quantities, ordered as
            self.ifos
        shift: numpy array of float, size of the time shift vector for each
            coinc to be ranked
        to_shift: list of int, multiple of the time shift to apply ordered
            as self.ifos

        Returns
        -------
        """

        # model signal rate as uniform over chirp mass, background rate is
        # proportional to mchirp^(-11/3) due to density of templates
        logr_s = ExpFitSGFgBgNormStatistic.logsignalrate(
                    self,
                    stats,
                    shift,
                    to_shift
                    )
        logr_s += numpy.log((self.curr_mchirp / 20.0) ** (11./3.0))
        return logr_s

        """

        benchmark_logvol values added in. This also stored the current chirp
        mass for use when computing the coinc statistic values.

        Returns
        -------
        """
        from pycbc.conversions import mchirp_from_mass1_mass2
        self.curr_mchirp = mchirp_from_mass1_mass2(trigs.param['mass1'],
                                                   trigs.param['mass2'])
        if self.mcm is not None:
            # Careful - input might be a str, so cast to float
            self.curr_mchirp = min(self.curr_mchirp, float(self.mcm))
        return ExpFitSGFgBgNormStatistic.single(self, trigs)

        """

        s: list

        Returns
        -------
        """

        loglr = ExpFitSGFgBgNormStatistic.coinc_lim_for_thresh(
                    self, s, thresh, limifo, **kwargs)
        loglr += numpy.log((self.curr_mchirp / 20.0) ** (11./3.0))

statistic_dict = {
    'quadsum': QuadratureSumStatistic,
    'single_ranking_only': QuadratureSumStatistic,
    'phasetd': PhaseTDStatistic,
    'exp_fit_stat': ExpFitStatistic,
    'exp_fit_csnr': ExpFitCombinedSNR,
    'phasetd_exp_fit_stat': PhaseTDExpFitStatistic,
    'exp_fit_sg_bg_rate': ExpFitSGBgRateStatistic,
    'phasetd_exp_fit_sg_fgbg_norm': ExpFitSGFgBgNormStatistic,
    'phasetd_exp_fit_sg_fgbg_bbh_norm': ExpFitSGPSDFgBgNormBBHStatistic,
}

def get_statistic(stat):
    """
    Error-handling sugar around dict lookup for coincident statistics

    stat : string
        Name of the coincident statistic

    Returns
    -------
    class
        Subclass of Stat base class

    Raises
    ------
    RuntimeError
        If the string is not recognized as corresponding to a Stat subclass
    """
    try:
        return statistic_dict[stat]
        raise RuntimeError('%s is not an available detection statistic' % stat)

def insert_statistic_option_group(parser):
    """
    Add ranking statistic options to the optparser object.

    Adds the options used to initialize a PyCBC Stat class.

    Returns
    --------
    strain_opt_group : optparser.argument_group
        The argument group that is added to the parser.
    """

    statistic_opt_group = parser.add_argument_group(
        "Options needed to initialize a PyCBC Stat class for computing the "
        "ranking of events from a PyCBC search."
    )

    statistic_opt_group.add_argument(
        "--ranking-statistic",
        choices=statistic_dict.keys(),
        required=True,
        help="The coinc ranking statistic to calculate"
    )

        "--sngl-ranking",
        choices=ranking.sngls_ranking_function_dict.keys(),
        help="The single-detector trigger ranking to use."
    )

        "--statistic-files",
        nargs='*',
        action='append',
        default=[],
        help="Files containing ranking statistic info"
    )

        "--statistic-keywords",
        help="Provide additional key-word arguments to be sent to "
             "the statistic class when it is initialized. Should "
             "be given in format --statistic-keywords "
             "KWARG1:VALUE1 KWARG2:VALUE2 KWARG3:VALUE3 ..."
    )

    return statistic_opt_group

def parse_statistic_keywords_opt(stat_kwarg_list):
    """
    Parse the list of statistic keywords into an appropriate dictionary.

    Take input from the input argument ["KWARG1:VALUE1", "KWARG2:VALUE2",
    "KWARG3:VALUE3"] and convert into a dictionary.

    stat_kwarg_list : list
        Statistic keywords in list format

    Returns
    -------
    stat_kwarg_dict : dict
        Statistic keywords in dict format
    """
    stat_kwarg_dict = {}
    for inputstr in stat_kwarg_list:
        try:
            key, value = inputstr.split(':')
            stat_kwarg_dict[key] = value
            err_txt = "--statistic-keywords must take input in the " \
                      "form KWARG1:VALUE1 KWARG2:VALUE2 KWARG3:VALUE3 ... " \
                      "Received {}".format(' '.join(stat_kwarg_list))
            raise ValueError(err_txt)

    return stat_kwarg_dict

def get_statistic_from_opts(opts, ifos):
    """
    Return a Stat class from an optparser object.

    This will assume that the options in the statistic_opt_group are present
    and will use these options to call stat.get_statistic and initialize the
    appropriate Stat subclass with appropriate kwargs.

    opts : optparse.OptParser instance
        The command line options
    ifos : list
        The list of detector names

    Returns
    -------
    class
    """
    # Allow None inputs
    if opts.statistic_files is None:
        opts.statistic_files = []
    if opts.statistic_keywords is None:
        opts.statistic_keywords = []

    # flatten the list of lists of filenames to a single list (may be empty)
    opts.statistic_files = sum(opts.statistic_files, [])

    extra_kwargs = parse_statistic_keywords_opt(opts.statistic_keywords)

    stat_class = get_statistic(opts.ranking_statistic)(
        opts.sngl_ranking,
        opts.statistic_files,
        ifos=ifos,
        **extra_kwargs
    )

    return stat_class

#
#

#
#
#                                   Preamble
#
#
from .simd_threshold_cython import parallel_thresh_cluster, parallel_threshold
from .eventmgr import _BaseThresholdCluster
from .. import opt

if opt.HAVE_GETCONF:
    default_segsize = opt.LEVEL2_CACHE_SIZE / numpy.dtype('complex64').itemsize
else:
    # Seems to work for Sandy Bridge/Ivy Bridge/Haswell, for now?
    default_segsize = 32768

def threshold_numpy(series, value):
    locs = numpy.where(arr.real**2 + arr.imag**2 > value**2)[0]

outl = None
outv = None
count = None
def threshold_inline(series, value):
    arr = numpy.array(series.data, copy=False, dtype=numpy.complex64)
    global outl, outv, count
    if outl is None or len(outl) < len(series):
        outl = numpy.zeros(len(series), dtype=numpy.uint32)
        outv = numpy.zeros(len(series), dtype=numpy.complex64)
        count = numpy.zeros(1, dtype=numpy.uint32)

    N = len(series)
    threshold = value**2.0
    parallel_threshold(N, arr, outv, outl, count, threshold)
    num = count[0]
    if num > 0:
        return outl[0:num], outv[0:num]
    else:
        return numpy.array([], numpy.uint32), numpy.array([], numpy.float32)

# threshold_numpy can also be used here, but for now we use the inline code
# in all instances. Not sure why we're defining threshold *and* threshold_only
# but we are, and I'm not going to change this at this point.
threshold = threshold_inline
threshold_only = threshold_inline

class CPUThresholdCluster(_BaseThresholdCluster):
    def __init__(self, series):
        self.series = numpy.array(series.data, copy=False,
                                  dtype=numpy.complex64)
        self.slen = numpy.uint32(len(series))
        self.outv = numpy.zeros(self.slen, numpy.complex64)
        self.outl = numpy.zeros(self.slen, numpy.uint32)
        self.segsize = numpy.uint32(default_segsize)

        self.count = parallel_thresh_cluster(self.series, self.slen,
                                             self.outv, self.outl,
                                             numpy.float32(threshold),
                                             numpy.uint32(window),
                                             self.segsize)
        if self.count > 0:
            return self.outv[0:self.count], self.outl[0:self.count]
        else:
            return numpy.array([], dtype = numpy.complex64), numpy.array([], dtype = numpy.uint32)

    return CPUThresholdCluster

"""
Tools for maximum likelihood fits to single trigger statistic values

For some set of values above a threshold, e.g. trigger SNRs, the functions
in this module perform maximum likelihood fits with 1-sigma uncertainties
to various simple functional forms of PDF, all normalized to 1.
You can also obtain the fitted function and its (inverse) CDF and perform
a Kolmogorov-Smirnov test.

Usage:
# call the fit function directly if the threshold is known
alpha, sigma_alpha = fit_exponential(snrs, 5.5)

# apply a threshold explicitly
alpha, sigma_alpha = fit_above_thresh('exponential', snrs, thresh=6.25)

# let the code work out the threshold from the smallest value via the default thresh=None
alpha, sigma_alpha = fit_above_thresh('exponential', snrs)

# or only fit the largest N values, i.e. tail fitting
thresh = tail_threshold(snrs, N=500)
alpha, sigma_alpha = fit_above_thresh('exponential', snrs, thresh)

# obtain the fitted function directly
xvals = numpy.xrange(5.5, 10.5, 20)
exponential_fit = expfit(xvals, alpha, thresh)

# or access function by name
exponential_fit_1 = fit_fn('exponential', xvals, alpha, thresh)

# get the KS test statistic and p-value - see scipy.stats.kstest
ks_stat, ks_pval = KS_test('exponential', snrs, alpha, thresh)

"""

# Copyright T. Dent 2015 (thomas.dent@aei.mpg.de)
#
#

from scipy.stats import kstest

fitalpha_dict = {
    'exponential' : lambda vals, thresh : 1. / (numpy.mean(vals) - thresh),
    'rayleigh'    : lambda vals, thresh : 2. / (numpy.mean(vals**2.) - thresh**2.),
    'power'       : lambda vals, thresh : numpy.mean(numpy.log(vals/thresh))**-1. + 1.
}

# measurement standard deviation = (-d^2 log L/d alpha^2)^(-1/2)
fitstd_dict = {
    'exponential' : lambda vals, alpha : alpha / len(vals)**0.5,
    'rayleigh'    : lambda vals, alpha : alpha / len(vals)**0.5,
    'power'       : lambda vals, alpha : (alpha - 1.) / len(vals)**0.5
}

def fit_above_thresh(distr, vals, thresh=None):
    """
    Maximum likelihood fit for the coefficient alpha

    Fitting a distribution of discrete values above a given threshold.
    Exponential  p(x) = alpha exp(-alpha (x-x_t))
    Rayleigh     p(x) = alpha x exp(-alpha (x**2-x_t**2)/2)
    Power        p(x) = ((alpha-1)/x_t) (x/x_t)**-alpha
    Values below threshold will be discarded.
    If no threshold is specified the minimum sample value will be used.

    distr : {'exponential', 'rayleigh', 'power'}
        Name of distribution
    vals : sequence of floats
        Values to fit
    thresh : float
        Threshold to apply before fitting; if None, use min(vals)

    Returns
    -------
    alpha : float
        Fitted value
    sigma_alpha : float
        Standard error in fitted value
    """
    if thresh is None:
        thresh = min(vals)
    else:
        vals = vals[vals >= thresh]
    alpha = fitalpha_dict[distr](vals, thresh)
    return alpha, fitstd_dict[distr](vals, alpha)

fitfn_dict = {
    'exponential' : lambda x, alpha, t : alpha * numpy.exp(-alpha * (x - t)),
    'rayleigh' : lambda x, alpha, t : (alpha * x * \
                                       numpy.exp(-alpha * (x**2 - t**2) / 2.)),
    'power' : lambda x, alpha, t : (alpha - 1.) * x**(-alpha) * t**(alpha - 1.)
}

def fit_fn(distr, xvals, alpha, thresh):
    """
    The fitted function normalized to 1 above threshold

    To normalize to a given total count multiply by the count.

    xvals : sequence of floats
        Values where the function is to be evaluated
        The fitted parameter
        Threshold value applied to fitted values

    Returns
    -------
    fit : array of floats
        Fitted function at the requested xvals
    """
    xvals = numpy.array(xvals)
    fit = fitfn_dict[distr](xvals, alpha, thresh)
    # set fitted values below threshold to 0
    numpy.putmask(fit, xvals < thresh, 0.)
    return fit

cum_fndict = {
    'exponential' : lambda x, alpha, t : numpy.exp(-alpha * (x - t)),
    'rayleigh' : lambda x, alpha, t : numpy.exp(-alpha * (x**2. - t**2.) / 2.),
    'power' : lambda x, alpha, t : x**(1. - alpha) * t**(alpha - 1.)
}

def cum_fit(distr, xvals, alpha, thresh):
    """
    Integral of the fitted function above a given value (reverse CDF)

    The fitted function is normalized to 1 above threshold

    Returns
    -------
    cum_fit : array of floats
        Reverse CDF of fitted function at the requested xvals
    """
    cum_fit = cum_fndict[distr](xvals, alpha, thresh)
    numpy.putmask(cum_fit, xvals < thresh, 0.)
    return cum_fit

def tail_threshold(vals, N=1000):
    """Determine a threshold above which there are N louder values"""
    if len(vals) < N:
        raise RuntimeError('Not enough input values to determine threshold')
    vals.sort()
    return min(vals[-N:])

def KS_test(distr, vals, alpha, thresh=None):
    """
    Perform Kolmogorov-Smirnov test for fitted distribution

    Compare the given set of discrete values above a given threshold to the
    fitted distribution function.
    If no threshold is specified, the minimum sample value will be used.
    Returns the KS test statistic and its p-value: lower p means less
    probable under the hypothesis of a perfect fit

        Values to compare to fit
        Fitted distribution parameter

    Returns
    -------
    D : float
        KS test statistic
    p-value : float
        p-value, assumed to be two-tailed
    """
    else:
    def cdf_fn(x):
        return 1 - cum_fndict[distr](x, alpha, thresh)
    return kstest(vals, cdf_fn)

def which_bin(par, minpar, maxpar, nbins, log=False):
    """
    Helper function

    Returns bin index where a parameter value belongs (from 0 through nbins-1)
    when dividing the range between minpar and maxpar equally into bins.

    par : float
        Parameter value being binned
    minpar : float
        Minimum parameter value
    maxpar : float
        Maximum parameter value
    nbins : int
        Number of bins to use
    log : boolean
        If True, use log spaced bins

    Returns
    -------
    binind : int
        Bin index
    """
    assert (par >= minpar and par <= maxpar)
    if log:
        par, minpar, maxpar = numpy.log(par), numpy.log(minpar), numpy.log(maxpar)
    # par lies some fraction of the way between min and max
    if minpar != maxpar:
        frac = float(par - minpar) / float(maxpar - minpar)
    else:
        # if they are equal there is only one size 0 bin
        # must be in that bin
        frac = 0
    # binind then lies between 0 and nbins - 1
    binind = int(frac * nbins)
    # corner case
    if par == maxpar:
        binind = nbins - 1
    return binind

#
#
#
""" This modules contains functions for reading single and coincident triggers
from the command line.
"""

from pycbc import conversions, pnutils
from pycbc.events import coinc

def insert_bank_bins_option_group(parser):
    """ Add options to the optparser object for selecting templates in bins.

    """
    bins_group = parser.add_argument_group(
                                 "Options for selecting templates in bins.")
    bins_group.add_argument("--bank-bins", nargs="+", default=None,
                            help="Ordered list of mass bin upper boundaries. "
                                 "An ordered list of type-boundary pairs, "
                                 "applied sequentially. Must provide a name "
                                 "(can be any unique string for tagging "
                                 "purposes), the parameter to bin "
                                 "on, and the membership condition via "
                                 "'lt' / 'gt' operators. "
                                 "Ex. name1:component:lt2 name2:total:lt15")
    bins_group.add_argument("--bank-file", default=None,
                            help="HDF format template bank file.")
    bins_group.add_argument("--f-lower", default=None,
                            help="Low frequency cutoff in Hz.")
    return bins_group

def bank_bins_from_cli(opts):
    """ Parses the CLI options related to binning templates in the bank.

    opts : object
        Result of parsing the CLI with OptionParser.

    Results
    -------
    bins_idx : dict
        A dict with bin names as key and an array of their indices as value.
    bank : dict
        A dict of the datasets from the bank file.
    """
    bank = {}
    fp = h5py.File(opts.bank_file)
    for key in fp.keys():
        bank[key] = fp[key][:]
    bank["f_lower"] = float(opts.f_lower) if opts.f_lower else None
    if opts.bank_bins:
        bins_idx = coinc.background_bin_from_string(opts.bank_bins, bank)
    else:
        bins_idx = {"all" : numpy.arange(0, len(bank[tuple(fp.keys())[0]]))}
    fp.close()
    return bins_idx, bank

def get_mass_spin(bank, tid):
    """

    bank : h5py File object
        Bank parameter file
    tid : integer or array of int
        Indices of the entries to be returned

    Returns
    -------
    m1, m2, s1z, s2z : tuple of floats or arrays of floats
        Parameter values of the bank entries
    """
    m1 = bank['mass1'][:][tid]
    m2 = bank['mass2'][:][tid]
    s1z = bank['spin1z'][:][tid]
    s2z = bank['spin2z'][:][tid]
    return m1, m2, s1z, s2z

def get_param(par, args, m1, m2, s1z, s2z):
    """

    par : string
        Name of parameter to calculate
    args : Namespace object returned from ArgumentParser instance
        Calling code command line options, used for f_lower value
    m1 : float or array of floats
        First binary component mass (etc.)

    Returns
    -------
    parvals : float or array of floats
        Calculated parameter values
    """
    if par == 'mchirp':
        parvals = conversions.mchirp_from_mass1_mass2(m1, m2)
    elif par == 'mtotal':
        parvals = m1 + m2
    elif par == 'eta':
        parvals = conversions.eta_from_mass1_mass2(m1, m2)
    elif par in ['chi_eff', 'effective_spin']:
        parvals = conversions.chi_eff(m1, m2, s1z, s2z)
    elif par == 'template_duration':
        # default to SEOBNRv4 duration function
        if not hasattr(args, 'approximant') or args.approximant is None:
            args.approximant = "SEOBNRv4"
        parvals = pnutils.get_imr_duration(m1, m2, s1z, s2z, args.f_lower,
                                           args.approximant)
        if args.min_duration:
            parvals += args.min_duration
    elif par == 'tau0':
        parvals = conversions.tau0_from_mass1_mass2(m1, m2, args.f_lower)
    elif par == 'tau3':
        parvals = conversions.tau3_from_mass1_mass2(m1, m2, args.f_lower)
    elif par in pnutils.named_frequency_cutoffs.keys():
        parvals = pnutils.frequency_cutoff_from_name(par, m1, m2, s1z, s2z)
    else:
        # try asking for a LALSimulation frequency function
        parvals = pnutils.get_freq(par, m1, m2, s1z, s2z)
    return parvals

def get_found_param(injfile, bankfile, trigfile, param, ifo, args=None):
    """
    Translates some popular trigger parameters into functions that calculate
    them from an hdf found injection file

    injfile: hdf5 File object
        Injection file of format known to ANitz (DOCUMENTME)
    bankfile: hdf5 File object or None
        Template bank file
    trigfile: hdf5 File object or None
        Single-detector trigger file
    param: string
        Parameter to be calculated for the recovered triggers
    ifo: string or None
        Standard ifo name, ex. 'L1'

    Returns
    -------
    [return value]: NumPy array of floats, array of boolean
        The calculated parameter values and a Boolean mask indicating which
        injections were found in the given ifo (if supplied)
    """
    foundtmp = injfile["found_after_vetoes/template_id"][:]
    # will record whether inj was found in the given ifo
    found_in_ifo = numpy.ones_like(foundtmp, dtype=bool)
    if trigfile is not None:
        try:  # old 2-ifo behaviour
            # get the name of the ifo in the injection file, eg "detector_1"
            # and the integer from that name
            ifolabel = [name for name, val in injfile.attrs.items() if \
                        "detector" in name and val == ifo][0]
            foundtrg = injfile["found_after_vetoes/trigger_id" + ifolabel[-1]]
        except IndexError:  # multi-ifo
            foundtrg = injfile["found_after_vetoes/%s/trigger_id" % ifo]
            # multi-ifo pipeline assigns -1 for inj not found in specific ifo
            found_in_ifo = foundtrg[:] != -1
    if bankfile is not None and param in bankfile.keys():
        return bankfile[param][:][foundtmp], found_in_ifo
    elif trigfile is not None and param in trigfile[ifo].keys():
        return trigfile[ifo][param][:][foundtrg], found_in_ifo
    else:
        assert bankfile
        b = bankfile
        return get_param(param, args, b['mass1'][:], b['mass2'][:],
                         b['spin1z'][:], b['spin2z'][:])[foundtmp],\
               found_in_ifo

def get_inj_param(injfile, param, ifo, args=None):
    """
    Translates some popular injection parameters into functions that calculate

        Parameter to be calculated for the injected signals
        Standard detector name, ex. 'L1'
    args: Namespace object returned from ArgumentParser instance

    Returns
    -------
    [return value]: NumPy array of floats
        The calculated parameter values
    """
    det = pycbc.detector.Detector(ifo)

    inj = injfile["injections"]
    if param in inj.keys():
        return inj["injections/"+param]

    if param == "end_time_"+ifo[0].lower():
        return inj['end_time'][:] + det.time_delay_from_earth_center(
                                        inj['longitude'][:],
                                        inj['latitude'][:],
                                        inj['end_time'][:])
    else:
        return get_param(param, args, inj['mass1'][:], inj['mass2'][:],
                                     inj['spin1z'][:], inj['spin2z'][:])

""" This module contains utilities to manipulate trigger lists based on
segment.
"""
from glue.ligolw import table, lsctables, utils as ligolw_utils
from ligo.segments import segment, segmentlist

def start_end_to_segments(start, end):
    return segmentlist([segment(s, e) for s, e in zip(start, end)])

def segments_to_start_end(segs):
    segs.coalesce()
    return (numpy.array([s[0] for s in segs]),
            numpy.array([s[1] for s in segs]))

def start_end_from_segments(segment_file):
    """
    Return the start and end time arrays from a segment file.

    segment_file: xml segment file

    Returns
    -------
    start: numpy.ndarray
    end: numpy.ndarray
    """
    from glue.ligolw.ligolw import LIGOLWContentHandler as h; lsctables.use_in(h)
    indoc = ligolw_utils.load_filename(segment_file, False, contenthandler=h)
    segment_table  = table.get_table(indoc, lsctables.SegmentTable.tableName)
    start = numpy.array(segment_table.getColumnByName('start_time'))
    start_ns = numpy.array(segment_table.getColumnByName('start_time_ns'))
    end = numpy.array(segment_table.getColumnByName('end_time'))
    end_ns = numpy.array(segment_table.getColumnByName('end_time_ns'))
    return start + start_ns * 1e-9, end + end_ns * 1e-9

def indices_within_times(times, start, end):
    """
    Return an index array into times that lie within the durations defined by start end arrays

    times: numpy.ndarray
        Array of times
        Array of duration start times
        Array of duration end times

    Returns
    -------
    indices: numpy.ndarray
        Array of indices into times
    """
    # coalesce the start/end segments
    start, end = segments_to_start_end(start_end_to_segments(start, end).coalesce())

    tsort = times.argsort()
    times_sorted = times[tsort]
    left = numpy.searchsorted(times_sorted, start)
    right = numpy.searchsorted(times_sorted, end)

    if len(left) == 0:
        return numpy.array([], dtype=numpy.uint32)

    return tsort[numpy.hstack([numpy.r_[s:e] for s, e in zip(left, right)])]

def indices_outside_times(times, start, end):
    """
    Return an index array into times that like outside the durations defined by start end arrays

    Returns
    -------
    """
    exclude = indices_within_times(times, start, end)
    indices = numpy.arange(0, len(times))
    return numpy.delete(indices, exclude)

def select_segments_by_definer(segment_file, segment_name=None, ifo=None):
    """ Return the list of segments that match the segment name

    segment_file: str
        path to segment xml file

    segment_name: str
        Name of segment
    ifo: str, optional

    Returns
    -------
    seg: list of segments
    """
    segment_table  = table.get_table(indoc, 'segment')

    seg_def_table = table.get_table(indoc, 'segment_definer')
    def_ifos = seg_def_table.getColumnByName('ifos')
    def_names = seg_def_table.getColumnByName('name')
    def_ids = seg_def_table.getColumnByName('segment_def_id')

    valid_id = []
    for def_ifo, def_name, def_id in zip(def_ifos, def_names, def_ids):
        if ifo and ifo != def_ifo:
            continue
        if segment_name and segment_name != def_name:
            continue
        valid_id += [def_id]

    start, end = start + 1e-9 * start_ns, end + 1e-9 * end_ns
    did = segment_table.getColumnByName('segment_def_id')

    keep = numpy.array([d in valid_id for d in did])
    if sum(keep) > 0:
        return start_end_to_segments(start[keep], end[keep])
    else:
        return segmentlist([])

def indices_within_segments(times, segment_files, ifo=None, segment_name=None):
    """ Return the list of indices that should be vetoed by the segments in the
    list of veto_files.

    times: numpy.ndarray of integer type
        Array of gps start times
    segment_files: string or list of strings
        A string or list of strings that contain the path to xml files that
        contain a segment table
    ifo: string, optional
        The ifo to retrieve segments for from the segment files
    segment_name: str, optional
        name of segment
    Returns
    -------
        The array of index values within the segments
    segmentlist:
        The segment list corresponding to the selected time.
    """
    veto_segs = segmentlist([])
    indices = numpy.array([], dtype=numpy.uint32)
    for veto_file in segment_files:
        veto_segs += select_segments_by_definer(veto_file, segment_name, ifo)
    veto_segs.coalesce()

    start, end = segments_to_start_end(veto_segs)
    if len(start) > 0:
        idx = indices_within_times(times, start, end)
        indices = numpy.union1d(indices, idx)

    return indices, veto_segs.coalesce()

def indices_outside_segments(times, segment_files, ifo=None, segment_name=None):
    """ Return the list of indices that are outside the segments in the
    list of segment files.

    Returns
    --------
        The array of index values outside the segments
    """
    exclude, segs = indices_within_segments(times, segment_files,
                                         ifo=ifo, segment_name=segment_name)
    return numpy.delete(indices, exclude), segs

def get_segment_definer_comments(xml_file, include_version=True):
    """Returns a dict with the comment column as the value for each segment"""

    # read segment definer table
    xmldoc, _ = ligolw_utils.load_fileobj(xml_file,
                                        gz=xml_file.name.endswith(".gz"),
                                        contenthandler=h)
    seg_def_table = table.get_table(xmldoc,
                                    lsctables.SegmentDefTable.tableName)

    # put comment column into a dict
    comment_dict = {}
    for seg_def in seg_def_table:
        if include_version:
            full_channel_name = ':'.join([str(seg_def.ifos),
                                          str(seg_def.name),
                                          str(seg_def.version)])
        else:
                                          str(seg_def.name)])

        comment_dict[full_channel_name] = seg_def.comment

    return comment_dict

#  Copyright (C) 2014 Josh Willis
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with with program; see the file COPYING. If not, write to the
#  Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#  MA  02111-1307  USA

from .core import _list_available

_backend_dict = {'fftw' : 'fftw',
                 'mkl' : 'mkl',
                 'numpy' : 'npfft'}
_backend_list = ['fftw','mkl','numpy']

_alist, _adict = _list_available(_backend_list, _backend_dict)

cpu_backend = None

def set_backend(backend_list):
    global cpu_backend
    for backend in backend_list:
        if backend in _alist:
            cpu_backend = backend
            break

def get_backend():
    return _adict[cpu_backend]

set_backend(_backend_list)

#
#
#

_backend_dict = {'mkl' : 'mkl'}
_backend_list = ['mkl']

mkl_backend = None

    global mkl_backend
            mkl_backend = backend
            break

    return _adict[mkl_backend]

# Copyright (C) 2012  Josh Willis, Andrew Miller
#
#
#

#
#
#                                   Preamble
#
#
"""
This package provides a front-end to various fast Fourier transform
implementations within PyCBC.
"""

import pycbc.scheme

# These are global variables, that are modified by the various scheme-
# dependent submodules, to maintain a list of all possible backends
# for all possible schemes that are available at runtime.  This list
# and dict are then used when parsing command-line options.

_all_backends_list = []
_all_backends_dict = {}

# The following is the function called by each scheme's setup to add whatever new
# backends may have been found to the global list.  Since some backends may be
# shared, we must first check to make sure that the item in the list is not already
# in the global list, and we assume that the keys to the dict are in one-to-one
# correspondence with the items in the list.

def _update_global_available(new_list, new_dict, global_list, global_dict):
    for item in new_list:
        if item not in global_list:
            global_list.append(item)
            global_dict.update({item:new_dict[item]})

def get_backend_modules():
    return _all_backends_dict.values()

def get_backend_names():
    return list(_all_backends_dict.keys())

BACKEND_PREFIX="pycbc.fft.backend_"

@pycbc.scheme.schemed(BACKEND_PREFIX)
    err_msg = "This function is a stub that should be overridden using "
    err_msg += "the scheme. You shouldn't be seeing this error!"

# Import all scheme-dependent backends, to get _all_backends accurate:

for scheme_name in ["cpu", "mkl", "cuda"]:
    try:
        mod = __import__('pycbc.fft.backend_' + scheme_name, fromlist = ['_alist', '_adict'])
        _alist = getattr(mod, "_alist")
        _adict = getattr(mod, "_adict")
        _update_global_available(_alist, _adict, _all_backends_list,
                                 _all_backends_dict)
    except ImportError:
        pass

#
#
#

#
#
#                                   Preamble
#
#
"""
"""

from .backend_support import get_backend

def _fft_factory(invec, outvec, nbatch=1, size=None):
    backend = get_backend()
    cls = getattr(backend, 'FFT')
    return cls

def _ifft_factory(invec, outvec, nbatch=1, size=None):
    cls = getattr(backend, 'IFFT')
    return cls

class FFT(object):
    """ Create a forward FFT  engine

    invec : complex64 or float32
      Input pycbc.types.Array (or subclass); its FFT will be computed
    outvec : complex64
      Output pycbc.types.Array (or subclass); it will hold the FFT of invec
    nbatch : int (default 1)
      When not one, specifies that invec and outvec should each be interpreted
      as nbatch distinct vectors. The total length of invec and outvec should
      then be that appropriate to a single vector, multiplied by nbatch
    size : int (default None)
      When nbatch is not 1, this parameter gives the logical size of each
      transform.  If nbatch is 1 (the default) this can be None, and the
      logical size is the length of invec.

    The addresses in memory of both vectors should be divisible by
    pycbc.PYCBC_ALIGNMENT.
    """
        real_cls = _fft_factory(*args, **kwargs)
        return real_cls(*args, **kwargs)

class IFFT(object):
    """ Create a reverse FFT  engine

    invec : complex64
      Input pycbc.types.Array (or subclass); its IFFT will be computed
    outvec : complex64 or float32
      Output pycbc.types.Array (or subclass); it will hold the IFFT of invec
      logical size is the length of outvec.

    """
        real_cls = _ifft_factory(*args, **kwargs)

#!/usr/bin/python
import os, subprocess, ctypes
from mako.template import Template

full_corr = """
    __device__ cufftComplex in_call(void* input, size_t offset,
                                void* caller_info, void* shared) {
        cufftComplex r;

        cufftComplex* hp =  ((cufftComplex*) callback_params.htilde);
        cufftComplex h = hp[offset];
        cufftComplex s = ((cufftComplex*) input)[offset];

        r.x = h.x * s.x + h.y * s.y;
        r.y = h.x * s.y - h.y * s.x;

        return r;
    }
"""

zero_corr = """
        if (offset >= callback_params.in_kmax)
            return (cufftComplex){0, 0};
        else{
            cufftComplex r;

            cufftComplex s = ((cufftComplex*) input)[offset];
            cufftComplex* hp =  ((cufftComplex*) callback_params.htilde);
            cufftComplex h = hp[offset];

            r.x = h.x * s.x + h.y * s.y;
            r.y = h.x * s.y - h.y * s.x;

            return r;
        }

    }
"""

zero_out = """
    __device__ void out_call(void *out, size_t offset, cufftComplex element,
                             void *caller_info, void *shared){
           if (offset > callback_params.out_kmin && offset < callback_params.out_kmax)
                ((cufftComplex*) out)[offset] = element;
    }

"""

copy_callback = """
    __device__ cufftComplex correlate(void* input, size_t offset,
        return ((cufftComplex*)input)[offset];
    }
"""

copy_out = """
           ((cufftComplex*) out)[offset] = element;
    }

"""

real_out = """
           ((cufftReal*) out)[offset] = element.x * element.x + element.y * element.y;
    }

"""

copy_out_fp16 = """
           ((short*) out)[offset*2] = __float2half_rn(element.x);
           ((short*) out)[offset*2+1] = __float2half_rn(element.y);
    }
"""

no_out = """
    }

"""

fftsrc = Template("""
    #include <stdlib.h>
    #include <stdio.h>
    #include <string.h>
    #include <math.h>
    #include <cuda_runtime.h>
    #include <cufft.h>
    #include <cufftXt.h>

    typedef struct {
        %for t, n in parameters:
            ${t} ${n};
        %endfor
    } param_t;

    __constant__ param_t callback_params;

    #define checkCudaErrors(val)  __checkCudaErrors__ ( (val), #val, __FILE__, __LINE__ )

    template <typename T>
    inline void __checkCudaErrors__(T code, const char *func, const char *file, int line)
    {
        if (code) {
            fprintf(stderr, "CUDA error at %s:%d code=%d \\"%s\\" \\n",
                    file, line, (unsigned int)code, func);
            cudaDeviceReset();
            exit(EXIT_FAILURE);
        }
    }

    % if input_callback:
        ${input_callback}
        __device__ cufftCallbackLoadC input_callback = in_call;
    % endif

    % if output_callback:
        ${output_callback}
        __device__ cufftCallbackStoreC output_callback = out_call;
    % endif

    extern "C"  cufftHandle* create_plan(unsigned int size){
        cufftHandle* plan = new cufftHandle;
        size_t work_size;
        cufftCreate(plan);
        checkCudaErrors(cufftMakePlan1d(*plan, size, CUFFT_C2C, 1, &work_size));

        % if input_callback:
            cufftCallbackLoadC h_input_callback;
            checkCudaErrors(cudaMemcpyFromSymbol(&h_input_callback, input_callback,
                                             sizeof(h_input_callback)));
            checkCudaErrors(cufftXtSetCallback(*plan, (void **) &h_input_callback,
                                          CUFFT_CB_LD_COMPLEX, 0));
        % endif

        % if output_callback:
            cufftCallbackStoreC h_output_callback;
            checkCudaErrors(cudaMemcpyFromSymbol(&h_output_callback, output_callback,
                                                 sizeof(h_output_callback)));
            checkCudaErrors(cufftXtSetCallback(*plan, (void **) &h_output_callback,
                                              CUFFT_CB_ST_COMPLEX, 0));
        % endif

        return plan;
    }

    extern "C" void execute(cufftHandle* plan, cufftComplex* in, cufftComplex* out, param_t* p){
         if (p != NULL)
            checkCudaErrors(cudaMemcpyToSymbolAsync(callback_params, p, sizeof(param_t), 0,  cudaMemcpyHostToDevice, 0));
         checkCudaErrors(cufftExecC2C(*plan, in, out, CUFFT_INVERSE));
    }
""")

def compile(source, name):
    """ Compile the string source code into a shared object linked against
    the static version of cufft for callback support.
    """
    # If we start using this again, we should find a better place for the cache
    cache = os.path.join('/tmp', name)
    hash_file = cache + ".hash"
    lib_file = cache + ".so"
    obj_file = cache + ".o"

    try:
        if int(open(hash_file, "r").read()) == hash(source):
            return lib_file
        raise ValueError
    except:
        pass

    src_file = cache + ".cu"
    fsrc = open(src_file, "w")
    fsrc.write(source)
    fsrc.close()

    cmd = ["nvcc", "-ccbin", "g++", "-dc", "-m64",
           "--compiler-options", "'-fPIC'",
           "-o", obj_file,
           "-c", src_file]
    print(" ".join(cmd))
    subprocess.check_call(cmd)

    cmd = ["nvcc", "-shared", "-ccbin", "g++", "-m64",
       "-o", lib_file, obj_file, "-lcufft_static", "-lculibos"]

    fhash = open(hash_file, "w")
    fhash.write(str(hash(source)))
    return lib_file

def get_fn_plan(callback=None, out_callback=None, name='pycbc_cufft', parameters=None):
    """ Get the IFFT execute and plan functions
    """
    if parameters is None:
        parameters = []
    source = fftsrc.render(input_callback=callback, output_callback=out_callback, parameters=parameters)
    path = compile(source, name)
    lib = ctypes.cdll.LoadLibrary(path)
    fn = lib.execute
    fn.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
    plan = lib.create_plan
    plan.restype = ctypes.c_void_p
    plan.argyptes = [ctypes.c_uint]
    return fn, plan

_plans = {}

class param(ctypes.Structure):
    _fields_ = [("htilde", ctypes.c_void_p)]
hparam = param()

def c2c_correlate_ifft(htilde, stilde, outvec):
    key = 'cnf'
    if key not in _plans:
        fn, pfn = get_fn_plan(callback=full_corr, parameters = [("void*", "htilde")])
        plan = pfn(len(outvec), int(htilde.data.gpudata))
        _plans[key] = (fn, plan, int(htilde.data.gpudata))
    fn, plan, _ = _plans[key]
    hparam.htilde = htilde.data.gpudata
    fn(plan, int(stilde.data.gpudata), int(outvec.data.gpudata), ctypes.pointer(hparam))

class param2(ctypes.Structure):
    _fields_ = [("htilde", ctypes.c_void_p),
                ("in_kmax", ctypes.c_uint),
                ("out_kmin", ctypes.c_uint),
                ("out_kmax", ctypes.c_uint)]
hparam_zeros = param2()

def c2c_half_correlate_ifft(htilde, stilde, outvec):
    key = 'cn'
        fn, pfn = get_fn_plan(callback=zero_corr,
                              parameters = [("void*", "htilde"),
                                            ("unsigned int", "in_kmax"),
                                            ("unsigned int", "out_kmin"),
                                            ("unsigned int", "out_kmax")],
                              out_callback=zero_out)
    hparam_zeros.htilde = htilde.data.gpudata
    hparam_zeros.in_kmax = htilde.end_idx
    hparam_zeros.out_kmin = stilde.analyze.start
    hparam_zeros.out_kmax = stilde.analyze.stop
    fn(plan, int(stilde.data.gpudata), int(outvec.data.gpudata), ctypes.pointer(hparam_zeros))

from pycbc.types import zeros, complex64, complex128
import numpy as _np
import ctypes
import pycbc.scheme as _scheme
from pycbc.libutils import get_ctypes_library
from .core import _BaseFFT, _BaseIFFT
from ..types import check_aligned

# IMPORTANT NOTE TO PYCBC DEVELOPERS:
# Because this module is loaded automatically when present, and because
# no FFTW function should be called until the user has had the chance
# to set the threading backend, it is ESSENTIAL that simply loading this
# module should not actually *call* ANY functions.

#FFTW constants, these are pulled from fftw3.h
FFTW_FORWARD = -1
FFTW_BACKWARD = 1

FFTW_MEASURE = 0
FFTW_DESTROY_INPUT = 1 << 0
FFTW_UNALIGNED = 1 << 1
FFTW_CONSERVE_MEMORY = 1 << 2
FFTW_EXHAUSTIVE = 1 << 3
FFTW_PRESERVE_INPUT = 1 << 4
FFTW_PATIENT = 1 << 5
FFTW_ESTIMATE = 1 << 6
FFTW_WISDOM_ONLY = 1 << 21

# Load the single and double precision libraries
# We need to construct them directly with CDLL so
# we can give the RTLD_GLOBAL mode, which we must do
# in order to use the threaded libraries as well.
double_lib = get_ctypes_library('fftw3',['fftw3'],mode=ctypes.RTLD_GLOBAL)
float_lib = get_ctypes_library('fftw3f',['fftw3f'],mode=ctypes.RTLD_GLOBAL)
if (double_lib is None) or (float_lib is None):
    raise ImportError("Unable to find FFTW libraries")

# Support for FFTW's two different threading backends
_fftw_threaded_lib = None
_fftw_threaded_set = False
_double_threaded_lib = None
_float_threaded_lib = None

HAVE_FFTW_THREADED = False

# Although we set the number of threads based on the scheme,
# we need a private variable that records the last value used so
# we know whether we need to call plan_with_nthreads() again.
_fftw_current_nthreads = 0

# This function sets the number of threads used internally by FFTW
# in planning. It just takes a number of threads, rather than itself
# looking at scheme.mgr.num_threads, because it should not be called
# directly, but only by functions that get the value they use from
# scheme.mgr.num_threads

def _fftw_plan_with_nthreads(nthreads):
    global _fftw_current_nthreads
    if not HAVE_FFTW_THREADED:
        if (nthreads > 1):
            raise ValueError("Threading is NOT enabled, but {0} > 1 threads specified".format(nthreads))
        else:
            _pycbc_current_threads = nthreads
    else:
        dplanwthr = _double_threaded_lib.fftw_plan_with_nthreads
        fplanwthr = _float_threaded_lib.fftwf_plan_with_nthreads
        dplanwthr.restype = None
        fplanwthr.restype = None
        dplanwthr(nthreads)
        fplanwthr(nthreads)
        _fftw_current_nthreads = nthreads

# This is a global dict-of-dicts used when initializing threads and
# setting the threading library

_fftw_threading_libnames = { 'unthreaded' : {'double' : None, 'float' : None},
                             'openmp' : {'double' : 'fftw3_omp', 'float' : 'fftw3f_omp'},
                             'pthreads' : {'double' : 'fftw3_threads', 'float' : 'fftw3f_threads'}}

def _init_threads(backend):
    # This function actually sets the backend and initializes. It returns zero on
    # success and 1 if given a valid backend but that cannot be loaded.  It raises
    # an exception if called after the threading backend has already been set, or
    # if given an invalid backend.
    global _fftw_threaded_set
    global _fftw_threaded_lib
    global HAVE_FFTW_THREADED
    global _double_threaded_lib
    global _float_threaded_lib
    if _fftw_threaded_set:
        raise RuntimeError(
            "Threading backend for FFTW already set to {0}; cannot be changed".format(_fftw_threaded_lib))
    try:
        double_threaded_libname = _fftw_threading_libnames[backend]['double']
        float_threaded_libname =  _fftw_threading_libnames[backend]['float']
        raise ValueError("Backend {0} for FFTW threading does not exist!".format(backend))
    if double_threaded_libname is not None:
        try:
            # Note that the threaded libraries don't have their own pkg-config files;
            # we must look for them wherever we look for double or single FFTW itself
            _double_threaded_lib = get_ctypes_library(double_threaded_libname,['fftw3'],mode=ctypes.RTLD_GLOBAL)
            _float_threaded_lib =  get_ctypes_library(float_threaded_libname,['fftw3f'],mode=ctypes.RTLD_GLOBAL)
            if (_double_threaded_lib is None) or (_float_threaded_lib is None):
                raise RuntimeError("Unable to load threaded libraries {0} or {1}".format(double_threaded_libname,
                                                                                         float_threaded_libname))
            dret = _double_threaded_lib.fftw_init_threads()
            fret = _float_threaded_lib.fftwf_init_threads()
            # FFTW for some reason uses *0* to indicate failure.  In C.
            if (dret == 0) or (fret == 0):
                return 1
            HAVE_FFTW_THREADED = True
            _fftw_threaded_set = True
            _fftw_threaded_lib = backend
            return 0
        except:
            return 1
    else:
        # We get here when we were given the 'unthreaded' backend
        HAVE_FFTW_THREADED = False
        _fftw_threaded_set = True
        _fftw_threaded_lib = backend
        return 0

def set_threads_backend(backend=None):
    # This is the user facing function.  If given a backend it just
    # calls _init_threads and lets it do the work.  If not (the default)
    # then it cycles in order through threaded backends,
    if backend is not None:
        retval = _init_threads(backend)
        # Since the user specified this backend raise an exception if the above failed
        if retval != 0:
            raise RuntimeError("Could not initialize FFTW threading backend {0}".format(backend))
    else:
        # Note that we pop() from the end, so 'openmp' is the first thing tried
        _backend_list = ['unthreaded','pthreads','openmp']
        while not _fftw_threaded_set:
            _next_backend = _backend_list.pop()
            retval = _init_threads(_next_backend)

# Function to import system-wide wisdom files.

def import_sys_wisdom():
    if not _fftw_threaded_set:
        set_threads_backend()
    double_lib.fftw_import_system_wisdom()
    float_lib.fftwf_import_system_wisdom()

# We provide an interface for changing the "measure level"
# By default this is 0, which does no planning,
# but we provide functions to read and set it

_default_measurelvl = 0
def get_measure_level():
    """
    Get the current 'measure level' used in deciding how much effort to put into
    creating FFTW plans.  From least effort (and shortest planning time) to most
    they are 0 to 3.  No arguments.
    """
    return _default_measurelvl

def set_measure_level(mlvl):
    """
    Set the current 'measure level' used in deciding how much effort to expend
    creating FFTW plans.  Must be an integer from 0 (least effort, shortest time)
    to 3 (most effort and time).
    """
    global _default_measurelvl
    if mlvl not in (0,1,2,3):
        raise ValueError("Measure level can only be one of 0, 1, 2, or 3")
    _default_measurelvl = mlvl

_flag_dict = {0: FFTW_ESTIMATE,
              1: FFTW_MEASURE,
              2: FFTW_MEASURE|FFTW_PATIENT,
              3: FFTW_MEASURE|FFTW_PATIENT|FFTW_EXHAUSTIVE}
def get_flag(mlvl,aligned):
    if aligned:
        return _flag_dict[mlvl]
    else:
        return (_flag_dict[mlvl]|FFTW_UNALIGNED)

# Add the ability to read/store wisdom to filenames

def wisdom_io(filename, precision, action):
    """Import or export an FFTW plan for single or double precision.
    """
    fmap = {('float', 'import'): float_lib.fftwf_import_wisdom_from_filename,
            ('float', 'export'): float_lib.fftwf_export_wisdom_to_filename,
            ('double', 'import'): double_lib.fftw_import_wisdom_from_filename,
            ('double', 'export'): double_lib.fftw_export_wisdom_to_filename}
    f = fmap[(precision, action)]
    f.argtypes = [ctypes.c_char_p]
    retval = f(filename.encode())
    if retval == 0:
        raise RuntimeError(('Could not {0} wisdom '
                            'from file {1}').format(action, filename))

def import_single_wisdom_from_filename(filename):
    wisdom_io(filename, 'float', 'import')

def import_double_wisdom_from_filename(filename):
    wisdom_io(filename, 'double', 'import')

def export_single_wisdom_to_filename(filename):
    wisdom_io(filename, 'float', 'export')

def export_double_wisdom_to_filename(filename):
    wisdom_io(filename, 'double', 'export')

def set_planning_limit(time):

    f = double_lib.fftw_set_timelimit
    f.argtypes = [ctypes.c_double]
    f(time)

    f = float_lib.fftwf_set_timelimit
    f(time)

# Create function maps for the dtypes
plan_function = {'float32': {'complex64': float_lib.fftwf_plan_dft_r2c_1d},
                 'float64': {'complex128': double_lib.fftw_plan_dft_r2c_1d},
                 'complex64': {'float32': float_lib.fftwf_plan_dft_c2r_1d,
                               'complex64': float_lib.fftwf_plan_dft_1d},
                 'complex128': {'float64': double_lib.fftw_plan_dft_c2r_1d,
                                'complex128': double_lib.fftw_plan_dft_1d}
                }

execute_function = {'float32': {'complex64': float_lib.fftwf_execute_dft_r2c},
                    'float64': {'complex128': double_lib.fftw_execute_dft_r2c},
                    'complex64': {'float32': float_lib.fftwf_execute_dft_c2r,
                                  'complex64': float_lib.fftwf_execute_dft},
                    'complex128': {'float64': double_lib.fftw_execute_dft_c2r,
                                   'complex128': double_lib.fftw_execute_dft}
                   }

def plan(size, idtype, odtype, direction, mlvl, aligned, nthreads, inplace):
    if nthreads != _fftw_current_nthreads:
        _fftw_plan_with_nthreads(nthreads)
    # Convert a measure-level to flags
    flags = get_flag(mlvl,aligned)

    # We make our arrays of the necessary type and size.  Things can be
    # tricky, especially for in-place transforms with one of input or
    # output real.
    if (idtype == odtype):
        # We're in the complex-to-complex case, so lengths are the same
        ip = zeros(size, dtype=idtype)
        if inplace:
            op = ip
        else:
            op = zeros(size, dtype=odtype)
    elif (idtype.kind == 'c') and (odtype.kind == 'f'):
        # Complex-to-real (reverse), so size is length of real array.
        # However the complex array may be larger (in bytes) and
        # should therefore be allocated first and reused for an in-place
        # transform
        ip = zeros(size/2+1, dtype=idtype)
            op = ip.view(dtype=odtype)[0:size]
        else:
    else:
        # Real-to-complex (forward), and size is still that of real.
        # However it is still true that the complex array may be larger
        # (in bytes) and should therefore be allocated first and reused
        # for an in-place transform
        op = zeros(size/2+1, dtype=odtype)
            ip = op.view(dtype=idtype)[0:size]
        else:
            ip = zeros(size, dtype=idtype)

    # Get the plan function
    idtype = _np.dtype(idtype)
    odtype = _np.dtype(odtype)
    f = plan_function[str(idtype)][str(odtype)]
    f.restype = ctypes.c_void_p

    # handle the C2C cases (forward and reverse)
    if idtype.kind == odtype.kind:
        f.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p,
                      ctypes.c_int, ctypes.c_int]
        theplan = f(size, ip.ptr, op.ptr, direction, flags)
    # handle the R2C and C2R case
    else:
                      ctypes.c_int]
        theplan = f(size, ip.ptr, op.ptr, flags)

    # We don't need ip or op anymore
    del ip, op

    # Make the destructors
    if idtype.char in ['f', 'F']:
        destroy = float_lib.fftwf_destroy_plan
    else:
        destroy = double_lib.fftw_destroy_plan 

    destroy.argtypes = [ctypes.c_void_p]

    return theplan, destroy

# Note that we don't need to check whether we've set the threading backend
# in the following functions, since execute is not called directly and
# the fft and ifft will call plan first.
def execute(plan, invec, outvec):
    f = execute_function[str(invec.dtype)][str(outvec.dtype)]
    f.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
    f(plan, invec.ptr, outvec.ptr)

def fft(invec, outvec, prec, itype, otype):
    theplan, destroy = plan(len(invec), invec.dtype, outvec.dtype, FFTW_FORWARD,
                            get_measure_level(),(check_aligned(invec.data) and check_aligned(outvec.data)),
                   _scheme.mgr.state.num_threads, (invec.ptr == outvec.ptr))
    execute(theplan, invec, outvec)
    destroy(theplan)

def ifft(invec, outvec, prec, itype, otype):
    theplan, destroy = plan(len(outvec), invec.dtype, outvec.dtype, FFTW_BACKWARD,

# Class based API

# First, set up a lot of different ctypes functions:
plan_many_c2c_f = float_lib.fftwf_plan_many_dft
plan_many_c2c_f.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
                            ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_int,
                            ctypes.c_int, ctypes.c_uint]
plan_many_c2c_f.restype = ctypes.c_void_p

plan_many_c2c_d = double_lib.fftw_plan_many_dft
plan_many_c2c_d.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
plan_many_c2c_d.restype = ctypes.c_void_p

plan_many_c2r_f = float_lib.fftwf_plan_many_dft_c2r
plan_many_c2r_f.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
                            ctypes.c_uint]
plan_many_c2r_f.restype = ctypes.c_void_p

plan_many_c2r_d = double_lib.fftw_plan_many_dft_c2r
plan_many_c2r_d.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
plan_many_c2r_d.restype = ctypes.c_void_p

plan_many_r2c_f = float_lib.fftwf_plan_many_dft_r2c
plan_many_r2c_f.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
plan_many_r2c_f.restype = ctypes.c_void_p

plan_many_r2c_d = double_lib.fftw_plan_many_dft_r2c
plan_many_r2c_d.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
plan_many_r2c_d.restype = ctypes.c_void_p

# Now set up a dictionary indexed by (str(input_dtype), str(output_dtype)) to
# translate input and output dtypes into the correct planning function.

_plan_funcs_dict = { ('complex64', 'complex64') : plan_many_c2c_f,
                     ('complex64', 'float32') : plan_many_r2c_f,
                     ('float32', 'complex64') : plan_many_c2r_f,
                     ('complex128', 'complex128') : plan_many_c2c_d,
                     ('complex128', 'float64') : plan_many_r2c_d,
                     ('float64', 'complex128') : plan_many_c2r_d }

# To avoid multiple-inheritance, we set up a function that returns much
# of the initialization that will need to be handled in __init__ of both
# classes.

def _fftw_setup(fftobj):
    n = _np.asarray([fftobj.size], dtype=_np.int32)
    inembed = _np.asarray([len(fftobj.invec)], dtype=_np.int32)
    onembed = _np.asarray([len(fftobj.outvec)], dtype=_np.int32)
    nthreads = _scheme.mgr.state.num_threads
    mlvl = get_measure_level()
    aligned = check_aligned(fftobj.invec.data) and check_aligned(fftobj.outvec.data)
    flags = get_flag(mlvl, aligned)
    plan_func = _plan_funcs_dict[ (str(fftobj.invec.dtype), str(fftobj.outvec.dtype)) ]
    tmpin = zeros(len(fftobj.invec), dtype = fftobj.invec.dtype)
    tmpout = zeros(len(fftobj.outvec), dtype = fftobj.outvec.dtype)
    # C2C, forward
    if fftobj.forward and (fftobj.outvec.dtype in [complex64, complex128]):
        plan = plan_func(1, n.ctypes.data, fftobj.nbatch,
                         tmpin.ptr, inembed.ctypes.data, 1, fftobj.idist,
                         tmpout.ptr, onembed.ctypes.data, 1, fftobj.odist,
                         FFTW_FORWARD, flags)
    # C2C, backward
    elif not fftobj.forward and (fftobj.invec.dtype in [complex64, complex128]):
                         FFTW_BACKWARD, flags)
    # R2C or C2R (hence no direction argument for plan creation)
    else:
                         flags)
    del tmpin
    del tmpout
    return plan

class FFT(_BaseFFT):
    def __init__(self, invec, outvec, nbatch=1, size=None):
        super(FFT, self).__init__(invec, outvec, nbatch, size)
        self.iptr = self.invec.ptr
        self.optr = self.outvec.ptr
        self._efunc = execute_function[str(self.invec.dtype)][str(self.outvec.dtype)]
        self._efunc.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
        self.plan = _fftw_setup(self)

    def execute(self):
        self._efunc(self.plan, self.iptr, self.optr)

class IFFT(_BaseIFFT):
        super(IFFT, self).__init__(invec, outvec, nbatch, size)

def insert_fft_options(optgroup):
    """
    Inserts the options that affect the behavior of this backend

    optgroup: fft_option
       OptionParser argument group whose options are extended
    """
    optgroup.add_argument("--fftw-measure-level",
                      help="Determines the measure level used in planning "
                           "FFTW FFTs; allowed values are: " + str([0,1,2,3]),
                      type=int, default=_default_measurelvl)
    optgroup.add_argument("--fftw-threads-backend",
                      help="Give 'openmp', 'pthreads' or 'unthreaded' to specify which threaded FFTW to use",
                      default=None)
    optgroup.add_argument("--fftw-input-float-wisdom-file",
                      help="Filename from which to read single-precision wisdom",
    optgroup.add_argument("--fftw-input-double-wisdom-file",
                      help="Filename from which to read double-precision wisdom",
    optgroup.add_argument("--fftw-output-float-wisdom-file",
                      help="Filename to which to write single-precision wisdom",
    optgroup.add_argument("--fftw-output-double-wisdom-file",
                      help="Filename to which to write double-precision wisdom",
    optgroup.add_argument("--fftw-import-system-wisdom",
                          help = "If given, call fftw[f]_import_system_wisdom()",
                          action = "store_true")

def verify_fft_options(opt,parser):
    """Parses the FFT options and verifies that they are

    """
    if opt.fftw_measure_level not in [0,1,2,3]:
        parser.error("{0} is not a valid FFTW measure level.".format(opt.fftw_measure_level))

    if opt.fftw_import_system_wisdom and ((opt.fftw_input_float_wisdom_file is not None)
                                          or (opt.fftw_input_double_wisdom_file is not None)):
        parser.error("If --fftw-import-system-wisdom is given, then you cannot give"
                     " either of --fftw-input-float-wisdom-file or --fftw-input-double-wisdom-file")

    if opt.fftw_threads_backend is not None:
        if opt.fftw_threads_backend not in ['openmp','pthreads','unthreaded']:
            parser.error("Invalid threads backend; must be 'openmp', 'pthreads' or 'unthreaded'")

    # Since opt.fftw_threads_backend defaults to None, the following is always
    # appropriate:
    set_threads_backend(opt.fftw_threads_backend)

    # Set the user-provided measure level
    set_measure_level(opt.fftw_measure_level)

"""This module provides a functions to perform a pruned FFT based on FFTW

This should be considered a test and example module, as the functionality
can and should be generalized to other FFT backends, and precisions.

These functions largely implemented the generic FFT decomposition as
described rather nicely by wikipedia.

http://en.wikipedia.org/wiki/Cooley%E2%80%93Tukey_FFT_algorithm

I use a similar naming convention here, with minor simplifications to the
twiddle factors.
"""
import numpy, ctypes, pycbc.types
from .fftw_pruned_cython import second_phase_cython

warn_msg = ("The FFTW_pruned module can be used to speed up computing SNR "
            "timeseries by computing first at a low sample rate and then "
            "computing at full sample rate only at certain samples. This code "
            "has not yet been used in production, and has no test case. "
            "This was also ported to Cython in this state. "
            "This code would need verification before trusting results. "
            "Please do contribute test cases.")

logging.warning(warn_msg)

# FFTW constants
float_lib = get_ctypes_library('fftw3f', ['fftw3f'],mode=ctypes.RTLD_GLOBAL)
fexecute = float_lib.fftwf_execute_dft
fexecute.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

ftexecute = float_lib.fftwf_execute_dft
ftexecute.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

def plan_transpose(N1, N2):
    """
    Create a plan for transposing internally to the pruned_FFT calculation.
    (Alex to provide a write up with more details.)

    N1 : int
        Number of rows.
    N2 : int
        Number of columns.

    Returns
    --------
    plan : FFTWF plan
        The plan for performing the FFTW transpose.
    """

    rows = N1
    cols = N2

    iodim = numpy.zeros(6, dtype=numpy.int32)
    iodim[0] = rows
    iodim[1] = 1
    iodim[2] = cols
    iodim[3] = cols
    iodim[4] = rows
    iodim[5] = 1

    N = N1*N2
    vin = pycbc.types.zeros(N, dtype=numpy.complex64)
    vout = pycbc.types.zeros(N, dtype=numpy.complex64)

    f = float_lib.fftwf_plan_guru_dft
    f.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int,
                  ctypes.c_void_p, ctypes.c_void_p,
                  ctypes.c_int]
    return f(0, None, 2, iodim.ctypes.data, vin.ptr, vout.ptr, None, FFTW_MEASURE)

def plan_first_phase(N1, N2):
    """
    Create a plan for the first stage of the pruned FFT operation.

    N1 : int
    N2 : int

    Returns
    --------
        The plan for performing the first phase FFT.
    """
    N = N1*N2
    f = float_lib.fftwf_plan_many_dft
                  ctypes.c_int, ctypes.c_int,
                  ctypes.c_int, ctypes.c_int]
    return f(1, ctypes.byref(ctypes.c_int(N2)), N1,
             vin.ptr, None, 1, N2,
             vout.ptr, None, 1, N2, FFTW_BACKWARD, FFTW_MEASURE)

_theplan = None
def first_phase(invec, outvec, N1, N2):
    """
    This implements the first phase of the FFT decomposition, using
    the standard FFT many plans.

    invec : array
        The input array.
    outvec : array
        The output array.
    N1 : int
    N2 : int
    """
    global _theplan
    if _theplan is None:
        _theplan = plan_first_phase(N1, N2)
    fexecute(_theplan, invec.ptr, outvec.ptr)

def second_phase(invec, indices, N1, N2):
    """
    This is the second phase of the FFT decomposition that actually performs
    the pruning. It is an explicit calculation for the subset of points. Note
    that there seem to be some numerical accumulation issues at various values
    of N1 and N2.

    invec :
        The result of the first phase FFT
    indices : array of ints
        The index locations to calculate the FFT
    N1 : int
        The length of the second phase "FFT"
    N2 : int
        The length of the first phase FFT

    Returns
    -------
    out : array of floats
    """
    invec = numpy.array(invec.data, copy=False)
    NI = len(indices) # pylint:disable=unused-variable
    N1 = int(N1)
    N2 = int(N2)
    out = numpy.zeros(len(indices), dtype=numpy.complex64)
    indices = numpy.array(indices, dtype=numpy.uint32)

    # Note, the next step if this needs to be faster is to invert the loops
    second_phase_cython(N1, N2, NI, indices, out, invec)

    return out

_thetransposeplan = None
def fft_transpose_fftw(vec):
    """
    Perform an FFT transpose from vec into outvec.
    (Alex to provide more details in a write-up.)

    vec : array
        Input array.

    Returns
    --------
        Transposed output array.
    """
    global _thetransposeplan
    outvec = pycbc.types.zeros(len(vec), dtype=vec.dtype)
        N1, N2 = splay(vec)
        _thetransposeplan = plan_transpose(N1, N2)
    ftexecute(_thetransposeplan, vec.ptr, outvec.ptr)
    return  outvec

fft_transpose = fft_transpose_fftw

def splay(vec):
    """ Determine two lengths to split stride the input vector by
    """
    N2 = 2 ** int(numpy.log2( len(vec) ) / 2)
    N1 = len(vec) / N2
    return N1, N2

def pruned_c2cifft(invec, outvec, indices, pretransposed=False):
    """
    Perform a pruned iFFT, only valid for power of 2 iffts as the
    decomposition is easier to choose. This is not a strict requirement of the
    functions, but it is unlikely to the optimal to use anything but power
    of 2. (Alex to provide more details in write up.

        The input vector. This should be the correlation between the data and
        the template at full sample rate. Ideally this is pre-transposed, but
        if not this will be transposed in this function.
        The output of the first phase of the pruned FFT.
        The indexes at which to calculate the full sample-rate SNR.
    pretransposed : boolean, default=False
        Used to indicate whether or not invec is pretransposed.

    Returns
    --------
    SNRs : array
        The complex SNRs at the indexes given by indices.
    """
    N1, N2 = splay(invec)

    if not pretransposed:
        invec = fft_transpose(invec)
    first_phase(invec, outvec, N1=N1, N2=N2)
    out = second_phase(outvec, indices, N1=N1, N2=N2)
    return out

#
#
#

#
#
#                                   Preamble
#
#
"""
"""

from pycbc.types import TimeSeries as _TimeSeries
from pycbc.types import FrequencySeries as _FrequencySeries
from .core import _check_fft_args, _check_fwd_args, _check_inv_args

def fft(invec, outvec):
    """ Fourier transform from invec to outvec.

    Perform a fourier transform. The type of transform is determined
    by the dtype of invec and outvec.

    invec : TimeSeries or FrequencySeries
        The input vector.
    outvec : TimeSeries or FrequencySeries
        The output.
    """
    prec, itype, otype = _check_fft_args(invec, outvec)
    _check_fwd_args(invec, itype, outvec, otype, 1, None)

    # The following line is where all the work is done:
    backend.fft(invec, outvec, prec, itype, otype)
    # For a forward FFT, the length of the *input* vector is the length
    # we should divide by, whether C2C or R2HC transform
    if isinstance(invec, _TimeSeries):
        outvec._epoch = invec._epoch
        outvec._delta_f = 1.0/(invec._delta_t * len(invec))
        outvec *= invec._delta_t
    elif isinstance(invec, _FrequencySeries):
        outvec._delta_t = 1.0/(invec._delta_f * len(invec))
        outvec *= invec._delta_f

def ifft(invec, outvec):
    """ Inverse fourier transform from invec to outvec.

    Perform an inverse fourier transform. The type of transform is determined

    """
    _check_inv_args(invec, itype, outvec, otype, 1, None)

    backend.ifft(invec, outvec, prec, itype, otype)
    # For an inverse FFT, the length of the *output* vector is the length
    # we should divide by, whether C2C or HC2R transform
        outvec._delta_f = 1.0/(invec._delta_t * len(outvec))
    elif isinstance(invec,_FrequencySeries):
        outvec._delta_t = 1.0/(invec._delta_f * len(outvec))

import ctypes, pycbc.libutils
from pycbc.types import zeros

lib = pycbc.libutils.get_ctypes_library('mkl_rt', [])
if lib is None:
    raise ImportError

#MKL constants  taken from mkl_df_defines.h
DFTI_FORWARD_DOMAIN = 0
DFTI_DIMENSION = 1
DFTI_LENGTHS = 2
DFTI_PRECISION = 3
DFTI_FORWARD_SCALE  = 4
DFTI_BACKWARD_SCALE = 5
DFTI_NUMBER_OF_TRANSFORMS = 7
DFTI_COMPLEX_STORAGE = 8
DFTI_REAL_STORAGE = 9
DFTI_CONJUGATE_EVEN_STORAGE = 10
DFTI_PLACEMENT = 11
DFTI_INPUT_STRIDES = 12
DFTI_OUTPUT_STRIDES = 13
DFTI_INPUT_DISTANCE = 14
DFTI_OUTPUT_DISTANCE = 15
DFTI_WORKSPACE = 17
DFTI_ORDERING = 18
DFTI_TRANSPOSE = 19
DFTI_DESCRIPTOR_NAME = 20
DFTI_PACKED_FORMAT = 21
DFTI_COMMIT_STATUS = 22
DFTI_VERSION = 23
DFTI_NUMBER_OF_USER_THREADS = 26
DFTI_THREAD_LIMIT = 27
DFTI_COMMITTED = 30
DFTI_UNCOMMITTED = 31
DFTI_COMPLEX = 32
DFTI_REAL = 33
DFTI_SINGLE = 35
DFTI_DOUBLE = 36
DFTI_COMPLEX_COMPLEX = 39
DFTI_COMPLEX_REAL = 40
DFTI_REAL_COMPLEX = 41
DFTI_REAL_REAL = 42
DFTI_INPLACE = 43
DFTI_NOT_INPLACE = 44
DFTI_ORDERED = 48
DFTI_BACKWARD_SCRAMBLED = 49
DFTI_ALLOW = 51
DFTI_AVOID = 52
DFTI_NONE = 53
DFTI_CCS_FORMAT = 54
DFTI_PACK_FORMAT = 55
DFTI_PERM_FORMAT = 56
DFTI_CCE_FORMAT = 57

mkl_prec = {'single': DFTI_SINGLE,
            'double': DFTI_DOUBLE,
           }

mkl_domain = {'real': {'complex': DFTI_REAL},
              'complex': {'real': DFTI_REAL,
                          'complex':DFTI_COMPLEX,
                         }
             }

def check_status(status):
    """ Check the status of a mkl functions and raise a python exeption if
    there is an error.
    """
    if status:
        msg = lib.DftiErrorMessage(status)
        msg = ctypes.c_char_p(msg).value
        raise RuntimeError(msg)

def create_descriptor(size, idtype, odtype, inplace):
    invec = zeros(1, dtype=idtype)
    outvec = zeros(1, dtype=odtype)

    desc = ctypes.c_void_p(1)
    f = lib.DftiCreateDescriptor
    f.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int]

    prec = mkl_prec[invec.precision]
    domain = mkl_domain[str(invec.kind)][str(outvec.kind)]

    status = f(ctypes.byref(desc), prec, domain, 1, size)
    if inplace:
        lib.DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
    else:
        lib.DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)
    lib.DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE, DFTI_CCS_FORMAT)
    lib.DftiCommitDescriptor(desc)
    check_status(status)
    return desc

    descr = create_descriptor(max(len(invec), len(outvec)), invec.dtype,
                              outvec.dtype, (invec.ptr == outvec.ptr))
    f = lib.DftiComputeForward
    status = f(descr, invec.ptr, outvec.ptr)
    lib.DftiFreeDescriptor(ctypes.byref(descr))

    f = lib.DftiComputeBackward

_create_descr = lib.DftiCreateDescriptor
_create_descr.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, ctypes.c_int]

def _get_desc(fftobj):
    prec = mkl_prec[fftobj.invec.precision]
    domain = mkl_domain[str(fftobj.invec.kind)][str(fftobj.outvec.kind)]
    status = _create_descr(ctypes.byref(desc), prec, domain,
                           1, int(fftobj.size))
    # Now we set various things depending on exactly what kind of transform we're
    # performing.

    lib.DftiSetValue.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int]

    # The following only matters if the transform is C2R or R2C
    status = lib.DftiSetValue(desc, DFTI_CONJUGATE_EVEN_STORAGE,
                              DFTI_COMPLEX_COMPLEX)

    # In-place or out-of-place:
    if fftobj.inplace:
        status = lib.DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
    else:
        status = lib.DftiSetValue(desc, DFTI_PLACEMENT, DFTI_NOT_INPLACE)

    # If we are performing a batched transform:
    if fftobj.nbatch > 1:
        status = lib.DftiSetValue(desc, DFTI_NUMBER_OF_TRANSFORMS, fftobj.nbatch)
        check_status(status)
        status = lib.DftiSetValue(desc, DFTI_INPUT_DISTANCE, fftobj.idist)
        status = lib.DftiSetValue(desc, DFTI_OUTPUT_DISTANCE, fftobj.odist)

    # Knowing how many threads will be allowed may help select a better transform
    status = lib.DftiSetValue(desc, DFTI_THREAD_LIMIT, nthreads)

    # Now everything's ready, so commit
    status = lib.DftiCommitDescriptor(desc)

        self._efunc = lib.DftiComputeForward
        self.desc = _get_desc(self)

        self._efunc(self.desc, self.iptr, self.optr)

        self._efunc = lib.DftiComputeBackward

# Copyright (C) 2012  Josh Willis
#
#
#

#
#
#                                   Preamble
#
#
"""
This module provides the numpy backend of the fast Fourier transform
for the PyCBC package.
"""

import numpy.fft

def fft(invec,outvec,prec,itype,otype):
    if invec.ptr == outvec.ptr:
        raise NotImplementedError("numpy backend of pycbc.fft does not support in-place transforms")
    if itype == 'complex' and otype == 'complex':
        outvec.data = numpy.asarray(numpy.fft.fft(invec.data),dtype=outvec.dtype)
    elif itype == 'real' and otype == 'complex':
        outvec.data = numpy.asarray(numpy.fft.rfft(invec.data),dtype=outvec.dtype)

def ifft(invec,outvec,prec,itype,otype):
        outvec.data = numpy.asarray(numpy.fft.ifft(invec.data),dtype=outvec.dtype)
        outvec *= len(outvec)
    elif itype == 'complex' and otype == 'real':
        outvec.data = numpy.asarray(numpy.fft.irfft(invec.data,len(outvec)),
                                    dtype=outvec.dtype)

#
#
#

#
#
#                                   Preamble
#
#
"""
"""

from .backend_support import get_backend_modules, get_backend_names
from .backend_support import set_backend, get_backend

# Next we add all of the machinery to set backends and their options
# from the command line.

def insert_fft_option_group(parser):
    """
    Adds the options used to choose an FFT backend. This should be used
    if your program supports the ability to select the FFT backend; otherwise
    you may simply call the fft and ifft functions and rely on default
    choices.  This function will also attempt to add any options exported
    by available backends through a function called insert_fft_options.
    These submodule functions should take the fft_group object as argument.

    """
    fft_group = parser.add_argument_group("Options for selecting the"
                                          " FFT backend and controlling its performance"
                                          " in this program.")
    # We have one argument to specify the backends.  This becomes the default list used
    # if none is specified for a particular call of fft() of ifft().  Note that this
    # argument expects a *list* of inputs, as indicated by the nargs='*'.
    fft_group.add_argument("--fft-backends",
                      help="Preference list of the FFT backends. "
                           "Choices are: \n" + str(get_backend_names()),
                      nargs='*', default=[])

    for backend in get_backend_modules():
        try:
            backend.insert_fft_options(fft_group)
            pass

def verify_fft_options(opt, parser):

    """

    if len(opt.fft_backends) > 0:
        _all_backends = get_backend_names()
        for backend in opt.fft_backends:
            if backend not in _all_backends:
                parser.error("Backend {0} is not available".format(backend))

        try:
            backend.verify_fft_options(opt, parser)
            pass

# The following function is the only one that is designed
# only to work with the active scheme.  We'd like to fix that,
# eventually, but it's non-trivial because of how poorly MKL
# and FFTW cooperate.

    """Parses the command line options and sets the FFT backend
    for each (available) scheme. Aside from setting the default
    backed for this context, this function will also call (if
    it exists) the from_cli function of the specified backends in
    the *current* scheme; typically one would only call this function
    once inside of a scheme context manager, but if it is desired
    to perform FFTs both inside and outside of a context, then
    this function would need to be called again.

    Returns
    """

    set_backend(opt.fft_backends)

    # Eventually, we need to be able to parse command lines
    # from more than just the current scheme's preference. But
    # the big problem is that calling from_cli for more than one
    # backend could cause interference; apparently, FFTW and MKL
    # don't play nice unless FFTW has been compiled and linked
    # with icc (and possibly numpy, scipy, and/or Python as well?)

    try:
        backend.from_cli(opt)
    except AttributeError:
        pass

# Copyright (C) 2016  Christopher M. Biwer
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides functions for calculating the autocorrelation function
and length of a data series.
"""

from math import isnan
from pycbc.filter.matchedfilter import correlate
from pycbc.types import FrequencySeries, TimeSeries, zeros

def calculate_acf(data, delta_t=1.0, unbiased=False):
    r"""Calculates the one-sided autocorrelation function.

    Calculates the autocorrelation function (ACF) and returns the one-sided
    ACF. The ACF is defined as the autocovariance divided by the variance. The
    ACF can be estimated using

    .. math::

        \hat{R}(k) = \frac{1}{n \sigma^{2}} \sum_{t=1}^{n-k} \left( X_{t} - \mu \right) \left( X_{t+k} - \mu \right)

    Where :math:`\hat{R}(k)` is the ACF, :math:`X_{t}` is the data series at
    time t, :math:`\mu` is the mean of :math:`X_{t}`, and :math:`\sigma^{2}` is
    the variance of :math:`X_{t}`.

    data : TimeSeries or numpy.array
        A TimeSeries or numpy.array of data.
    delta_t : float
        The time step of the data series if it is not a TimeSeries instance.
    unbiased : bool
        If True the normalization of the autocovariance function is n-k
        instead of n. This is called the unbiased estimation of the
        autocovariance. Note that this does not mean the ACF is unbiased.

    Returns
    -------
    acf : numpy.array
        If data is a TimeSeries then acf will be a TimeSeries of the
        one-sided ACF. Else acf is a numpy.array.
    """

    # if given a TimeSeries instance then get numpy.array
    if isinstance(data, TimeSeries):
        y = data.numpy()
        delta_t = data.delta_t
    else:
        y = data

    # Zero mean
    y = y - y.mean()
    ny_orig = len(y)

    npad = 1
    while npad < 2*ny_orig:
        npad = npad << 1
    ypad = numpy.zeros(npad)
    ypad[:ny_orig] = y

    # FFT data minus the mean
    fdata = TimeSeries(ypad, delta_t=delta_t).to_frequencyseries()

    # correlate
    # do not need to give the congjugate since correlate function does it
    cdata = FrequencySeries(zeros(len(fdata), dtype=fdata.dtype),
                           delta_f=fdata.delta_f, copy=False)
    correlate(fdata, fdata, cdata)

    # IFFT correlated data to get unnormalized autocovariance time series
    acf = cdata.to_timeseries()
    acf = acf[:ny_orig]

    # normalize the autocovariance
    # note that dividing by acf[0] is the same as ( y.var() * len(acf) )
    if unbiased:
        acf /= ( y.var() * numpy.arange(len(acf), 0, -1) )
    else:
        acf /= acf[0]

    # return input datatype
        return TimeSeries(acf, delta_t=delta_t)
    else:
        return acf

def calculate_acl(data, m=5, dtype=int):
    r"""Calculates the autocorrelation length (ACL).

    Given a normalized autocorrelation function :math:`\rho[i]` (by normalized,
    we mean that :math:`\rho[0] = 1`), the ACL :math:`\tau` is:

    .. math::

        \tau = 1 + 2 \sum_{i=1}^{K} \rho[i].

    The number of samples used :math:`K` is found by using the first point
    such that:

    .. math::

        m \tau[K] \leq K,

    where :math:`m` is a tuneable parameter (default = 5). If no such point
    exists, then the given data set it too short to estimate the ACL; in this
    case ``inf`` is returned.

    This algorithm for computing the ACL is taken from:

    N. Madras and A.D. Sokal, J. Stat. Phys. 50, 109 (1988).

    data : TimeSeries or array
        A TimeSeries of data.
    m : int
        The number of autocorrelation lengths to use for determining the window
        size :math:`K` (see above).
    dtype : int or float
        The datatype of the output. If the dtype was set to int, then the
        ceiling is returned.

    Returns
    -------
    acl : int or float
        The autocorrelation length. If the ACL cannot be estimated, returns
        ``numpy.inf``.
    """

    # sanity check output data type
    if dtype not in [int, float]:
        raise ValueError("The dtype must be either int or float.")

    # if we have only a single point, just return 1
    if len(data) < 2:
        return 1

    # calculate ACF that is normalized by the zero-lag value
    acf = calculate_acf(data)

    cacf = 2 * acf.numpy().cumsum() - 1
    win = m * cacf <= numpy.arange(len(cacf))
    if win.any():
        acl = cacf[numpy.where(win)[0][0]]
        if dtype == int:
            acl = int(numpy.ceil(acl))
    else:
        acl = numpy.inf
    return acl

#
#

#
#
#                                   Preamble
#
#
"""
This modules provides functions for matched filtering along with associated
utilities.
"""

from math import sqrt
from pycbc.types import TimeSeries, FrequencySeries, zeros, Array
from pycbc.types import complex_same_precision_as, real_same_precision_as
from pycbc.fft import fft, ifft, IFFT
from pycbc import events
from pycbc.events import ranking

BACKEND_PREFIX="pycbc.filter.matchedfilter_"

def correlate(x, y, z):

class BatchCorrelator(object):
    """ Create a batch correlation engine
    """
    def __init__(self, xs, zs, size):
        """ Correlate x and y, store in z. Arrays need not be equal length, but
        must be at least size long and of the same dtype. No error checking
        will be performed, so be careful. All dtypes must be complex64.
        Note, must be created within the processing context that it will be used in.
        """
        self.size = int(size)
        self.dtype = xs[0].dtype
        self.num_vectors = len(xs)

        # keep reference to arrays
        self.xs = xs
        self.zs = zs

        # Store each pointer as in integer array
        self.x = Array([v.ptr for v in xs], dtype=numpy.int)
        self.z = Array([v.ptr for v in zs], dtype=numpy.int)

    @pycbc.scheme.schemed(BACKEND_PREFIX)
    def batch_correlate_execute(self, y):
        pass

    execute = batch_correlate_execute

def _correlate_factory(x, y, z):

class Correlator(object):
    """ Create a correlator engine

    ---------
    x : complex64
      Input pycbc.types.Array (or subclass); it will be conjugated
    y : complex64
      Input pycbc.types.Array (or subclass); it will not be conjugated
    z : complex64
      Output pycbc.types.Array (or subclass).
      It will contain conj(x) * y, element by element

    The addresses in memory of the data of all three parameter vectors
    must be the same modulo pycbc.PYCBC_ALIGNMENT
    """
        real_cls = _correlate_factory(*args, **kwargs)

#
#
class _BaseCorrelator(object):
    def correlate(self):
        """
        Compute the correlation of the vectors specified at object
        instantiation, writing into the output vector given when the
        object was instantiated. The intention is that this method
        should be called many times, with the contents of those vectors
        changing between invocations, but not their locations in memory
        or length.
        """
        pass

class MatchedFilterControl(object):
    def __init__(self, low_frequency_cutoff, high_frequency_cutoff, snr_threshold, tlen,
                 delta_f, dtype, segment_list, template_output, use_cluster,
                 downsample_factor=1, upsample_threshold=1, upsample_method='pruned_fft',
                 gpu_callback_method='none', cluster_function='symmetric'):
        """ Create a matched filter engine.

        low_frequency_cutoff : {None, float}, optional
            The frequency to begin the filter calculation. If None, begin at the
            first frequency after DC.
        high_frequency_cutoff : {None, float}, optional
            The frequency to stop the filter calculation. If None, continue to the
            the nyquist frequency.
        snr_threshold : float
            The minimum snr to return when filtering
        segment_list : list
            List of FrequencySeries that are the Fourier-transformed data segments
        template_output : complex64
            Array of memory given as the 'out' parameter to waveform.FilterBank
        use_cluster : boolean
            If true, cluster triggers above threshold using a window; otherwise,
            only apply a threshold.
        downsample_factor : {1, int}, optional
            The factor by which to reduce the sample rate when doing a heirarchical
            matched filter
        upsample_threshold : {1, float}, optional
            The fraction of the snr_threshold to trigger on the subsampled filter.
        upsample_method : {pruned_fft, str}
            The method to upsample or interpolate the reduced rate filter.
        cluster_function : {symmetric, str}, optional
            Which method is used to cluster triggers over time. If 'findchirp', a
            sliding forward window; if 'symmetric', each window's peak is compared
            to the windows before and after it, and only kept as a trigger if larger
            than both.
        """
        # Assuming analysis time is constant across templates and segments, also
        # delta_f is constant across segments.
        self.tlen = tlen
        self.flen = self.tlen / 2 + 1
        self.delta_f = delta_f
        self.delta_t = 1.0/(self.delta_f * self.tlen)
        self.dtype = dtype
        self.snr_threshold = snr_threshold
        self.flow = low_frequency_cutoff
        self.fhigh = high_frequency_cutoff
        self.gpu_callback_method = gpu_callback_method
        if cluster_function not in ['symmetric', 'findchirp']:
            raise ValueError("MatchedFilter: 'cluster_function' must be either 'symmetric' or 'findchirp'")
        self.cluster_function = cluster_function
        self.segments = segment_list
        self.htilde = template_output

        if downsample_factor == 1:
            self.snr_mem = zeros(self.tlen, dtype=self.dtype)
            self.corr_mem = zeros(self.tlen, dtype=self.dtype)

            if use_cluster and (cluster_function == 'symmetric'):
                self.matched_filter_and_cluster = self.full_matched_filter_and_cluster_symm
                # setup the threasholding/clustering operations for each segment
                self.threshold_and_clusterers = []
                for seg in self.segments:
                    thresh = events.ThresholdCluster(self.snr_mem[seg.analyze])
                    self.threshold_and_clusterers.append(thresh)
            elif use_cluster and (cluster_function == 'findchirp'):
                self.matched_filter_and_cluster = self.full_matched_filter_and_cluster_fc
            else:
                self.matched_filter_and_cluster = self.full_matched_filter_thresh_only

            # Assuming analysis time is constant across templates and segments, also
            # delta_f is constant across segments.
            self.kmin, self.kmax = get_cutoff_indices(self.flow, self.fhigh,
                                                      self.delta_f, self.tlen)

            # Set up the correlation operations for each analysis segment
            corr_slice = slice(self.kmin, self.kmax)
            self.correlators = []
            for seg in self.segments:
                corr = Correlator(self.htilde[corr_slice],
                                  seg[corr_slice],
                                  self.corr_mem[corr_slice])
                self.correlators.append(corr)

            # setup up the ifft we will do
            self.ifft = IFFT(self.corr_mem, self.snr_mem)

        elif downsample_factor >= 1:
            self.matched_filter_and_cluster = self.heirarchical_matched_filter_and_cluster
            self.downsample_factor = downsample_factor
            self.upsample_method = upsample_method
            self.upsample_threshold = upsample_threshold

            N_full = self.tlen
            N_red = N_full / downsample_factor
            self.kmin_full, self.kmax_full = get_cutoff_indices(self.flow,
                                              self.fhigh, self.delta_f, N_full)

            self.kmin_red, _ = get_cutoff_indices(self.flow,
                                                  self.fhigh, self.delta_f, N_red)

            if self.kmax_full < N_red:
                self.kmax_red = self.kmax_full
            else:
                self.kmax_red = N_red - 1

            self.snr_mem = zeros(N_red, dtype=self.dtype)
            self.corr_mem_full = FrequencySeries(zeros(N_full, dtype=self.dtype), delta_f=self.delta_f)
            self.corr_mem = Array(self.corr_mem_full[0:N_red], copy=False)
            self.inter_vec = zeros(N_full, dtype=self.dtype)

        else:
            raise ValueError("Invalid downsample factor")

    def full_matched_filter_and_cluster_symm(self, segnum, template_norm, window, epoch=None):
        """ Returns the complex snr timeseries, normalization of the complex snr,
        the correlation vector frequency series, the list of indices of the
        triggers, and the snr values at the trigger locations. Returns empty
        lists for these for points that are not above the threshold.

        Calculated the matched filter, threshold, and cluster.

        segnum : int
            Index into the list of segments at MatchedFilterControl construction
            against which to filter.
        template_norm : float
            The htilde, template normalization factor.
        window : int
            Size of the window over which to cluster triggers, in samples

        Returns
        -------
        snr : TimeSeries
            A time series containing the complex snr.
        norm : float
            The normalization of the complex snr.
        corrrelation: FrequencySeries
            A frequency series containing the correlation vector.
        idx : Array
            List of indices of the triggers.
        snrv : Array
            The snr values at the trigger locations.
        """
        norm = (4.0 * self.delta_f) / sqrt(template_norm)
        self.correlators[segnum].correlate()
        self.ifft.execute()
        snrv, idx = self.threshold_and_clusterers[segnum].threshold_and_cluster(self.snr_threshold / norm, window)

        if len(idx) == 0:
            return [], [], [], [], []

        logging.info("%s points above threshold" % str(len(idx)))

        snr = TimeSeries(self.snr_mem, epoch=epoch, delta_t=self.delta_t, copy=False)
        corr = FrequencySeries(self.corr_mem, delta_f=self.delta_f, copy=False)
        return snr, norm, corr, idx, snrv

    def full_matched_filter_and_cluster_fc(self, segnum, template_norm, window, epoch=None):

        Returns
        -------
        idx : Array
        """
        idx, snrv = events.threshold(self.snr_mem[self.segments[segnum].analyze],
                                     self.snr_threshold / norm)
        idx, snrv = events.cluster_reduce(idx, snrv, window)

    def full_matched_filter_thresh_only(self, segnum, template_norm, window=None, epoch=None):

            Size of the window over which to cluster triggers, in samples.
            This is IGNORED by this function, and provided only for API compatibility.

        Returns
        -------
        idx : Array
        """
        idx, snrv = events.threshold_only(self.snr_mem[self.segments[segnum].analyze],
                                          self.snr_threshold / norm)

    def heirarchical_matched_filter_and_cluster(self, segnum, template_norm, window):

        Returns
        -------
            A time series containing the complex snr at the reduced sample rate.
        idx : Array
        """
        from pycbc.fft.fftw_pruned import pruned_c2cifft, fft_transpose
        htilde = self.htilde
        stilde = self.segments[segnum]

        norm = (4.0 * stilde.delta_f) / sqrt(template_norm)

        correlate(htilde[self.kmin_red:self.kmax_red],
                  stilde[self.kmin_red:self.kmax_red],
                  self.corr_mem[self.kmin_red:self.kmax_red])

        ifft(self.corr_mem, self.snr_mem)

        if not hasattr(stilde, 'red_analyze'):
            stilde.red_analyze = \
                             slice(stilde.analyze.start/self.downsample_factor,
                                   stilde.analyze.stop/self.downsample_factor)

        idx_red, snrv_red = events.threshold(self.snr_mem[stilde.red_analyze],
                                self.snr_threshold / norm * self.upsample_threshold)
        if len(idx_red) == 0:
            return [], None, [], [], []

        idx_red, _ = events.cluster_reduce(idx_red, snrv_red, window / self.downsample_factor)
        logging.info("%s points above threshold at reduced resolution"\
                      %(str(len(idx_red)),))

        # The fancy upsampling is here
        if self.upsample_method=='pruned_fft':
            idx = (idx_red + stilde.analyze.start/self.downsample_factor)\
                   * self.downsample_factor

            idx = smear(idx, self.downsample_factor)

            # cache transposed  versions of htilde and stilde
            if not hasattr(self.corr_mem_full, 'transposed'):
                self.corr_mem_full.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype)

            if not hasattr(htilde, 'transposed'):
                htilde.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype)
                htilde.transposed[self.kmin_full:self.kmax_full] = htilde[self.kmin_full:self.kmax_full]
                htilde.transposed = fft_transpose(htilde.transposed)

            if not hasattr(stilde, 'transposed'):
                stilde.transposed = zeros(len(self.corr_mem_full), dtype=self.dtype)
                stilde.transposed[self.kmin_full:self.kmax_full] = stilde[self.kmin_full:self.kmax_full]
                stilde.transposed = fft_transpose(stilde.transposed)

            correlate(htilde.transposed, stilde.transposed, self.corr_mem_full.transposed)
            snrv = pruned_c2cifft(self.corr_mem_full.transposed, self.inter_vec, idx, pretransposed=True)
            idx = idx - stilde.analyze.start
            idx2, snrv = events.threshold(Array(snrv, copy=False), self.snr_threshold / norm)

            if len(idx2) > 0:
                correlate(htilde[self.kmax_red:self.kmax_full],
                          stilde[self.kmax_red:self.kmax_full],
                          self.corr_mem_full[self.kmax_red:self.kmax_full])
                idx, snrv = events.cluster_reduce(idx[idx2], snrv, window)
            else:
                idx, snrv = [], []

            logging.info("%s points at full rate and clustering" % len(idx))
            return self.snr_mem, norm, self.corr_mem_full, idx, snrv
        else:
            raise ValueError("Invalid upsample method")

def compute_max_snr_over_sky_loc_stat(hplus, hcross, hphccorr,
                                                      hpnorm=None, hcnorm=None,
                                                      out=None, thresh=0,
                                                      analyse_slice=None):
    """
    Compute the maximized over sky location statistic.

    hplus : TimeSeries
        This is the IFFTed complex SNR time series of (h+, data). If not
        normalized, supply the normalization factor so this can be done!
        It is recommended to normalize this before sending through this
        function
    hcross : TimeSeries
        This is the IFFTed complex SNR time series of (hx, data). If not
    hphccorr : float
        The real component of the overlap between the two polarizations
        Re[(h+, hx)]. Note that the imaginary component does not enter the
        detection statistic. This must be normalized and is sign-sensitive.
        Used for optimization. If we do not care about the value of SNR
        values below thresh we can calculate a quick statistic that will
        always overestimate SNR and then only calculate the proper, more
        expensive, statistic at points where the quick SNR is above thresh.
    hpsigmasq : float
        The normalization factor (h+, h+). Default = None (=1, already
        normalized)
    hcsigmasq : float
        The normalization factor (hx, hx). Default = None (=1, already
    out : TimeSeries (optional, default=None)
        If given, use this array to store the output.

    Returns
    --------
    det_stat : TimeSeries
        The SNR maximized over sky location
    """
    # NOTE: Not much optimization has been done here! This may need to be
    # Cythonized.

    if out is None:
        out = zeros(len(hplus))
        out.non_zero_locs = numpy.array([], dtype=out.dtype)
    else:
        if not hasattr(out, 'non_zero_locs'):
            # Doing this every time is not a zero-cost operation
            out.data[:] = 0
            out.non_zero_locs = numpy.array([], dtype=out.dtype)
        else:
            # Only set non zero locations to zero
            out.data[out.non_zero_locs] = 0

    # If threshold is given we can limit the points at which to compute the
    # full statistic
    if thresh:
        # This is the statistic that always overestimates the SNR...
        # It allows some unphysical freedom that the full statistic does not
        idx_p, _ = events.threshold_only(hplus[analyse_slice],
                                                    thresh / (2**0.5 * hpnorm))
        idx_c, _ = events.threshold_only(hcross[analyse_slice],
                                                    thresh / (2**0.5 * hcnorm))
        idx_p = idx_p + analyse_slice.start
        idx_c = idx_c + analyse_slice.start
        hp_red = hplus[idx_p] * hpnorm
        hc_red = hcross[idx_p] * hcnorm
        stat_p = hp_red.real**2 + hp_red.imag**2 + \
                     hc_red.real**2 + hc_red.imag**2
        locs_p = idx_p[stat_p > (thresh*thresh)]
        hp_red = hplus[idx_c] * hpnorm
        hc_red = hcross[idx_c] * hcnorm
        stat_c = hp_red.real**2 + hp_red.imag**2 + \
        locs_c = idx_c[stat_c > (thresh*thresh)]
        locs = numpy.unique(numpy.concatenate((locs_p, locs_c)))

        hplus = hplus[locs]
        hcross = hcross[locs]

    hplus = hplus * hpnorm
    hcross = hcross * hcnorm

    # Calculate and sanity check the denominator
    denom = 1 - hphccorr*hphccorr
    if denom < 0:
        if hphccorr > 1:
            err_msg = "Overlap between hp and hc is given as %f. " %(hphccorr)
            err_msg += "How can an overlap be bigger than 1?"
        else:
            err_msg = "There really is no way to raise this error!?! "
            err_msg += "If you're seeing this, it is bad."
    if denom == 0:
        # This case, of hphccorr==1, makes the statistic degenerate
        # This case should not physically be possible luckily.
        err_msg = "You have supplied a real overlap between hp and hc of 1. "
        err_msg += "Ian is reasonably certain this is physically impossible "
        err_msg += "so why are you seeing this?"
        raise ValueError(err_msg)

    assert(len(hplus) == len(hcross))

    # Now the stuff where comp. cost may be a problem
    hplus_magsq = numpy.real(hplus) * numpy.real(hplus) + \
                       numpy.imag(hplus) * numpy.imag(hplus)
    hcross_magsq = numpy.real(hcross) * numpy.real(hcross) + \
                       numpy.imag(hcross) * numpy.imag(hcross)
    rho_pluscross = numpy.real(hplus) * numpy.real(hcross) + numpy.imag(hplus)*numpy.imag(hcross)

    sqroot = (hplus_magsq - hcross_magsq)**2
    sqroot += 4 * (hphccorr * hplus_magsq - rho_pluscross) * \
                  (hphccorr * hcross_magsq - rho_pluscross)
    # Sometimes this can be less than 0 due to numeric imprecision, catch this.
    if (sqroot < 0).any():
        indices = numpy.arange(len(sqroot))[sqroot < 0]
        # This should not be *much* smaller than 0 due to numeric imprecision
        if (sqroot[indices] < -0.0001).any():
            err_msg = "Square root has become negative. Something wrong here!"
        sqroot[indices] = 0
    sqroot = numpy.sqrt(sqroot)
    det_stat_sq = 0.5 * (hplus_magsq + hcross_magsq - \
                         2 * rho_pluscross*hphccorr + sqroot) / denom

    det_stat = numpy.sqrt(det_stat_sq)

    if thresh:
        out.data[locs] = det_stat
        out.non_zero_locs = locs
        return out
    else:
        return Array(det_stat, copy=False)

def compute_u_val_for_sky_loc_stat(hplus, hcross, hphccorr,
                                 hpnorm=None, hcnorm=None, indices=None):
    """The max-over-sky location detection statistic maximizes over a phase,
    an amplitude and the ratio of F+ and Fx, encoded in a variable called u.
    Here we return the value of u for the given indices.
    """
    if indices is not None:
        hplus = hplus[indices]
        hcross = hcross[indices]

    if hpnorm is not None:
        hplus = hplus * hpnorm
    if hcnorm is not None:
        hcross = hcross * hcnorm

    # Sanity checking in func. above should already have identified any points
    # which are bad, and should be used to construct indices for input here
    rho_pluscross = numpy.real(hplus) * numpy.real(hcross) + \
                       numpy.imag(hplus)*numpy.imag(hcross)

    a = hphccorr * hplus_magsq - rho_pluscross
    b = hplus_magsq - hcross_magsq
    c = rho_pluscross - hphccorr * hcross_magsq

    sq_root = b*b - 4*a*c
    sq_root = sq_root**0.5
    sq_root = -sq_root
    # Catch the a->0 case
    bad_lgc = (a == 0)
    dbl_bad_lgc = numpy.logical_and(c == 0, b == 0)
    dbl_bad_lgc = numpy.logical_and(bad_lgc, dbl_bad_lgc)
    # Initialize u
    u = sq_root * 0.
    # In this case u is completely degenerate, so set it to 1
    u[dbl_bad_lgc] = 1.
    # If a->0 avoid overflow by just setting to a large value
    u[bad_lgc & ~dbl_bad_lgc] = 1E17
    # Otherwise normal statistic
    u[~bad_lgc] = (-b[~bad_lgc] + sq_root[~bad_lgc]) / (2*a[~bad_lgc])

    snr_cplx = hplus * u + hcross
    coa_phase = numpy.angle(snr_cplx)

    return u, coa_phase

def compute_max_snr_over_sky_loc_stat_no_phase(hplus, hcross, hphccorr,
                                               hpnorm=None, hcnorm=None,
                                               out=None, thresh=0,
                                               analyse_slice=None):
    """
    Compute the match maximized over polarization phase.

    In contrast to compute_max_snr_over_sky_loc_stat_no_phase this function
    performs no maximization over orbital phase, treating that as an intrinsic
    parameter. In the case of aligned-spin 2,2-mode only waveforms, this
    collapses to the normal statistic (at twice the computational cost!)

        function

    Returns
    --------
    """

    else:
        else:

    if thresh:
        #
        # For now this is copied from the max-over-phase statistic. One could
        # probably make this faster by removing the imaginary components of
        # the matched filter, as these are not used here.

        else:

    hplus_magsq = numpy.real(hplus) * numpy.real(hplus)
    hcross_magsq = numpy.real(hcross) * numpy.real(hcross)
    rho_pluscross = numpy.real(hplus) * numpy.real(hcross)

    det_stat_sq = (hplus_magsq + hcross_magsq - 2 * rho_pluscross*hphccorr)

    det_stat = numpy.sqrt(det_stat_sq / denom)

    if thresh:
        return out
    else:

def compute_u_val_for_sky_loc_stat_no_phase(hplus, hcross, hphccorr,
                                 hpnorm=None , hcnorm=None, indices=None):
    """The max-over-sky location (no phase) detection statistic maximizes over

    """

    rhoplusre=numpy.real(hplus)
    rhocrossre=numpy.real(hcross)
    overlap=numpy.real(hphccorr)

    denom = (-rhocrossre+overlap*rhoplusre)
    # Initialize tan_kappa array
    u_val = denom * 0.
    # Catch the denominator -> 0 case
    numpy.putmask(u_val, denom == 0, 1E17)
    # Otherwise do normal statistic
    numpy.putmask(u_val, denom != 0, (-rhoplusre+overlap*rhocrossre)/(-rhocrossre+overlap*rhoplusre))
    coa_phase = numpy.zeros(len(indices), dtype=numpy.float32)

    return u_val, coa_phase

class MatchedFilterSkyMaxControl(object):
    # FIXME: This seems much more simplistic than the aligned-spin class.
    #        E.g. no correlators. Is this worth updating?
    def __init__(self, low_frequency_cutoff, high_frequency_cutoff,
                snr_threshold, tlen, delta_f, dtype):
        """
        Create a matched filter engine.

            The frequency to begin the filter calculation. If None, begin
            at the first frequency after DC.
            The frequency to stop the filter calculation. If None, continue
            to the nyquist frequency.
        """

        self.matched_filter_and_cluster = \
                                    self.full_matched_filter_and_cluster
        self.snr_plus_mem = zeros(self.tlen, dtype=self.dtype)
        self.corr_plus_mem = zeros(self.tlen, dtype=self.dtype)
        self.snr_cross_mem = zeros(self.tlen, dtype=self.dtype)
        self.corr_cross_mem = zeros(self.tlen, dtype=self.dtype)
        self.snr_mem = zeros(self.tlen, dtype=self.dtype)
        self.cached_hplus_hcross_correlation = None
        self.cached_hplus_hcross_hplus = None
        self.cached_hplus_hcross_hcross = None
        self.cached_hplus_hcross_psd = None

    def full_matched_filter_and_cluster(self, hplus, hcross, hplus_norm,
                                        hcross_norm, psd, stilde, window):
        """
        Return the complex snr and normalization.

        h_quantities : Various
            FILL ME IN
        stilde : FrequencySeries
            The strain data to be filtered.
            The size of the cluster window in samples.

        Returns
        -------
        correlation: FrequencySeries
        idx : Array
        """

        I_plus, Iplus_corr, Iplus_norm = matched_filter_core(hplus, stilde,
                                          h_norm=hplus_norm,
                                          low_frequency_cutoff=self.flow,
                                          high_frequency_cutoff=self.fhigh,
                                          out=self.snr_plus_mem,
                                          corr_out=self.corr_plus_mem)

        I_cross, Icross_corr, Icross_norm = matched_filter_core(hcross,
                                          stilde, h_norm=hcross_norm,
                                          out=self.snr_cross_mem,
                                          corr_out=self.corr_cross_mem)

        # The information on the complex side of this overlap is important
        # we may want to use this in the future.
        if not id(hplus) == self.cached_hplus_hcross_hplus:
            self.cached_hplus_hcross_correlation = None
        if not id(hcross) == self.cached_hplus_hcross_hcross:
        if not id(psd) == self.cached_hplus_hcross_psd:
        if self.cached_hplus_hcross_correlation is None:
            hplus_cross_corr = overlap_cplx(hplus, hcross, psd=psd,
                                           low_frequency_cutoff=self.flow,
                                           high_frequency_cutoff=self.fhigh,
                                           normalized=False)
            hplus_cross_corr = numpy.real(hplus_cross_corr)
            hplus_cross_corr = hplus_cross_corr / (hcross_norm*hplus_norm)**0.5
            self.cached_hplus_hcross_correlation = hplus_cross_corr
            self.cached_hplus_hcross_hplus = id(hplus)
            self.cached_hplus_hcross_hcross = id(hcross)
            self.cached_hplus_hcross_psd = id(psd)
        else:
            hplus_cross_corr = self.cached_hplus_hcross_correlation

        snr = self._maximized_snr(I_plus,I_cross,
                                  hplus_cross_corr,
                                  hpnorm=Iplus_norm,
                                  hcnorm=Icross_norm,
                                  out=self.snr_mem,
                                  thresh=self.snr_threshold,
                                  analyse_slice=stilde.analyze)
        # FIXME: This should live further down
        # Convert output to pycbc TimeSeries
        delta_t = 1.0 / (self.tlen * stilde.delta_f)

        snr = TimeSeries(snr, epoch=stilde.start_time, delta_t=delta_t,
                         copy=False)

        idx, snrv = events.threshold_real_numpy(snr[stilde.analyze],
                                                self.snr_threshold)

            return [], 0, 0, [], [], [], [], 0, 0, 0
        logging.info("%s points above threshold", str(len(idx)))

        logging.info("%s clustered points", str(len(idx)))
        # erased self.
        u_vals, coa_phase = self._maximized_extrinsic_params\
            (I_plus.data, I_cross.data, hplus_cross_corr,
             indices=idx+stilde.analyze.start, hpnorm=Iplus_norm,
             hcnorm=Icross_norm)

        return snr, Iplus_corr, Icross_corr, idx, snrv, u_vals, coa_phase,\
                                      hplus_cross_corr, Iplus_norm, Icross_norm

    def _maximized_snr(self, hplus, hcross, hphccorr, **kwargs):
        return compute_max_snr_over_sky_loc_stat(hplus, hcross, hphccorr,
                                                 **kwargs)

    def _maximized_extrinsic_params(self, hplus, hcross, hphccorr, **kwargs):
        return compute_u_val_for_sky_loc_stat(hplus, hcross, hphccorr,
                                              **kwargs)

class MatchedFilterSkyMaxControlNoPhase(MatchedFilterSkyMaxControl):
    # Basically the same as normal SkyMaxControl, except we use a slight
    # variation in the internal SNR functions.
        return compute_max_snr_over_sky_loc_stat_no_phase(hplus, hcross,
                                                          hphccorr, **kwargs)

        return compute_u_val_for_sky_loc_stat_no_phase(hplus, hcross, hphccorr,
                                                       **kwargs)

def make_frequency_series(vec):
    """Return a frequency series of the input vector.

    If the input is a frequency series it is returned, else if the input
    vector is a real time series it is fourier transformed and returned as a
    frequency series.

    vector : TimeSeries or FrequencySeries

    Returns
    -------
    Frequency Series: FrequencySeries
        A frequency domain version of the input vector.
    """
    if isinstance(vec, FrequencySeries):
        return vec
    if isinstance(vec, TimeSeries):
        N = len(vec)
        n = N // 2 + 1
        delta_f = 1.0 / N / vec.delta_t
        vectilde =  FrequencySeries(zeros(n, dtype=complex_same_precision_as(vec)),
                                    delta_f=delta_f, copy=False)
        fft(vec, vectilde)
        return vectilde
    else:
        raise TypeError("Can only convert a TimeSeries to a FrequencySeries")

def sigmasq_series(htilde, psd=None, low_frequency_cutoff=None,
            high_frequency_cutoff=None):
    """Return a cumulative sigmasq frequency series.

    Return a frequency series containing the accumulated power in the input
    up to that frequency.

    htilde : TimeSeries or FrequencySeries
        The input vector
    psd : {None, FrequencySeries}, optional
        The psd used to weight the accumulated power.
    low_frequency_cutoff : {None, float}, optional
        The frequency to begin accumulating power. If None, start at the beginning
        of the vector.
    high_frequency_cutoff : {None, float}, optional
        The frequency to stop considering accumulated power. If None, continue
        until the end of the input vector.

    Returns
    -------
        A frequency series containing the cumulative sigmasq.
    """
    htilde = make_frequency_series(htilde)
    N = (len(htilde)-1) * 2
    norm = 4.0 * htilde.delta_f
    kmin, kmax = get_cutoff_indices(low_frequency_cutoff,
                                   high_frequency_cutoff, htilde.delta_f, N)

    sigma_vec = FrequencySeries(zeros(len(htilde), dtype=real_same_precision_as(htilde)),
                                delta_f = htilde.delta_f, copy=False)

    mag = htilde.squared_norm()

    if psd is not None:
        mag /= psd

    sigma_vec[kmin:kmax] = mag[kmin:kmax].cumsum()

    return sigma_vec*norm

def sigmasq(htilde, psd = None, low_frequency_cutoff=None,
    """Return the loudness of the waveform. This is defined (see Duncan
    Brown's thesis) as the unnormalized matched-filter of the input waveform,
    htilde, with itself. This quantity is usually referred to as (sigma)^2
    and is then used to normalize matched-filters with the data.

        The input vector containing a waveform.
        The frequency to begin considering waveform power.
        The frequency to stop considering waveform power.

    Returns
    -------
    sigmasq: float
    """
    ht = htilde[kmin:kmax]

    if psd:
        try:
            numpy.testing.assert_almost_equal(ht.delta_f, psd.delta_f)
        except AssertionError:
            raise ValueError('Waveform does not have same delta_f as psd')

    if psd is None:
        sq = ht.inner(ht)
    else:
        sq = ht.weighted_inner(ht, psd[kmin:kmax])

    return sq.real * norm

def sigma(htilde, psd = None, low_frequency_cutoff=None,
        high_frequency_cutoff=None):
    """ Return the sigma of the waveform. See sigmasq for more details.

    Returns
    -------
    """
    return sqrt(sigmasq(htilde, psd, low_frequency_cutoff, high_frequency_cutoff))

def get_cutoff_indices(flow, fhigh, df, N):
    """
    Gets the indices of a frequency series at which to stop an overlap
    calculation.

    flow: float
        The frequency (in Hz) of the lower index.
    fhigh: float
        The frequency (in Hz) of the upper index.
    df: float
        The frequency step (in Hz) of the frequency series.
    N: int
        The number of points in the **time** series. Can be odd
        or even.

    Returns
    -------
    kmin: int
    kmax: int
    """
    if flow:
        kmin = int(flow / df)
        if kmin < 0:
            err_msg = "Start frequency cannot be negative. "
            err_msg += "Supplied value and kmin {} and {}".format(flow, kmin)
    else:
        kmin = 1
    if fhigh:
        kmax = int(fhigh / df )
        if kmax > int((N + 1)/2.):
            kmax = int((N + 1)/2.)
    else:
        # int() truncates towards 0, so this is
        # equivalent to the floor of the float
        kmax = int((N + 1)/2.)

    if kmax <= kmin:
        err_msg = "Kmax cannot be less than or equal to kmin. "
        err_msg += "Provided values of freqencies (min,max) were "
        err_msg += "{} and {} ".format(flow, fhigh)
        err_msg += "corresponding to (kmin, kmax) of "
        err_msg += "{} and {}.".format(kmin, kmax)

    return kmin,kmax

def matched_filter_core(template, data, psd=None, low_frequency_cutoff=None,
                  high_frequency_cutoff=None, h_norm=None, out=None, corr_out=None):
    """ Return the complex snr and normalization.

    Return the complex snr, along with its associated normalization of the template,
    matched filtered against the data.

    template : TimeSeries or FrequencySeries
        The template waveform
    data : TimeSeries or FrequencySeries
        The strain data to be filtered.
    psd : {FrequencySeries}, optional
        The noise weighting of the filter.
        The frequency to begin the filter calculation. If None, begin at the
        first frequency after DC.
        The frequency to stop the filter calculation. If None, continue to the
        the nyquist frequency.
    h_norm : {None, float}, optional
        The template normalization. If none, this value is calculated internally.
    out : {None, Array}, optional
        An array to use as memory for snr storage. If None, memory is allocated
        internally.
    corr_out : {None, Array}, optional
        An array to use as memory for correlation storage. If None, memory is allocated
        internally. If provided, management of the vector is handled externally by the
        caller. No zero'ing is done internally.

    Returns
    -------
    snr : TimeSeries
        A time series containing the complex snr.
    corrrelation: FrequencySeries
        A frequency series containing the correlation vector.
        The normalization of the complex snr.
    """
    htilde = make_frequency_series(template)
    stilde = make_frequency_series(data)

    if len(htilde) != len(stilde):
        raise ValueError("Length of template and data must match")

    N = (len(stilde)-1) * 2
                                   high_frequency_cutoff, stilde.delta_f, N)

    if corr_out is not None:
        qtilde = corr_out
    else:
        qtilde = zeros(N, dtype=complex_same_precision_as(data))

        _q = zeros(N, dtype=complex_same_precision_as(data))
    elif (len(out) == N) and type(out) is Array and out.kind =='complex':
        _q = out
    else:
        raise TypeError('Invalid Output Vector: wrong length or dtype')

    correlate(htilde[kmin:kmax], stilde[kmin:kmax], qtilde[kmin:kmax])

        if isinstance(psd, FrequencySeries):
            try:
                numpy.testing.assert_almost_equal(stilde.delta_f, psd.delta_f)
            except AssertionError:
                raise ValueError("PSD delta_f does not match data")
            qtilde[kmin:kmax] /= psd[kmin:kmax]
        else:
            raise TypeError("PSD must be a FrequencySeries")

    ifft(qtilde, _q)

    if h_norm is None:
        h_norm = sigmasq(htilde, psd, low_frequency_cutoff, high_frequency_cutoff)

    norm = (4.0 * stilde.delta_f) / sqrt( h_norm)

    return (TimeSeries(_q, epoch=stilde._epoch, delta_t=stilde.delta_t, copy=False),
           FrequencySeries(qtilde, epoch=stilde._epoch, delta_f=stilde.delta_f, copy=False),
           norm)

def smear(idx, factor):
    """
    This function will take as input an array of indexes and return every
    unique index within the specified factor of the inputs.

    E.g.: smear([5,7,100],2) = [3,4,5,6,7,8,9,98,99,100,101,102]

    idx : numpy.array of ints
        The indexes to be smeared.
    factor : idx
        The factor by which to smear out the input array.

    Returns
    --------
    new_idx : numpy.array of ints
        The smeared array of indexes.
    """

    s = [idx]
    for i in range(factor+1):
        a = i - factor/2
        s += [idx + a]
    return numpy.unique(numpy.concatenate(s))

def matched_filter(template, data, psd=None, low_frequency_cutoff=None,
                  high_frequency_cutoff=None, sigmasq=None):
    """ Return the complex snr.

    Return the complex snr, along with its associated normalization of the
    template, matched filtered against the data.

    psd : FrequencySeries
    sigmasq : {None, float}, optional
        The template normalization. If none, this value is calculated

    Returns
    -------
    """
    snr, _, norm = matched_filter_core(template, data, psd=psd,
            low_frequency_cutoff=low_frequency_cutoff,
            high_frequency_cutoff=high_frequency_cutoff, h_norm=sigmasq)
    return snr * norm

_snr = None
def match(vec1, vec2, psd=None, low_frequency_cutoff=None,
          high_frequency_cutoff=None, v1_norm=None, v2_norm=None):
    """ Return the match between the two TimeSeries or FrequencySeries.

    Return the match between two waveforms. This is equivalent to the overlap
    maximized over time and phase.

    vec1 : TimeSeries or FrequencySeries
    vec2 : TimeSeries or FrequencySeries
    psd : Frequency Series
        A power spectral density to weight the overlap.
        The frequency to begin the match.
        The frequency to stop the match.
    v1_norm : {None, float}, optional
        The normalization of the first waveform. This is equivalent to its
        sigmasq value. If None, it is internally calculated.
    v2_norm : {None, float}, optional
        The normalization of the second waveform. This is equivalent to its

    Returns
    -------
    match: float
    index: int
        The number of samples to shift to get the match.
    """

    htilde = make_frequency_series(vec1)
    stilde = make_frequency_series(vec2)

    global _snr
    if _snr is None or _snr.dtype != htilde.dtype or len(_snr) != N:
        _snr = zeros(N,dtype=complex_same_precision_as(vec1))
    snr, _, snr_norm = matched_filter_core(htilde,stilde,psd,low_frequency_cutoff,
                             high_frequency_cutoff, v1_norm, out=_snr)
    maxsnr, max_id = snr.abs_max_loc()
    if v2_norm is None:
        v2_norm = sigmasq(stilde, psd, low_frequency_cutoff, high_frequency_cutoff)
    return maxsnr * snr_norm / sqrt(v2_norm), max_id

def overlap(vec1, vec2, psd=None, low_frequency_cutoff=None,
          high_frequency_cutoff=None, normalized=True):
    """ Return the overlap between the two TimeSeries or FrequencySeries.

        The frequency to begin the overlap.
        The frequency to stop the overlap.
    normalized : {True, boolean}, optional
        Set if the overlap is normalized. If true, it will range from 0 to 1.

    Returns
    -------
    overlap: float
    """

    return overlap_cplx(vec1, vec2, psd=psd, \
            low_frequency_cutoff=low_frequency_cutoff,\
            high_frequency_cutoff=high_frequency_cutoff,\
            normalized=normalized).real

def overlap_cplx(vec1, vec2, psd=None, low_frequency_cutoff=None,
    """Return the complex overlap between the two TimeSeries or FrequencySeries.

    Returns
    -------
    overlap: complex
    """

            high_frequency_cutoff, stilde.delta_f, (len(stilde)-1) * 2)

    if psd:
        inner = (htilde[kmin:kmax]).weighted_inner(stilde[kmin:kmax], psd[kmin:kmax])
    else:
        inner = (htilde[kmin:kmax]).inner(stilde[kmin:kmax])

    if normalized:
        sig1 = sigma(vec1, psd=psd, low_frequency_cutoff=low_frequency_cutoff,
                     high_frequency_cutoff=high_frequency_cutoff)
        sig2 = sigma(vec2, psd=psd, low_frequency_cutoff=low_frequency_cutoff,
        norm = 1 / sig1 / sig2
    else:
        norm = 1

    return 4 * htilde.delta_f * inner * norm

def quadratic_interpolate_peak(left, middle, right):
    """ Interpolate the peak and offset using a quadratic approximation

    left : numpy array
        Values at a relative bin value of [-1]
    middle : numpy array
        Values at a relative bin value of [0]
    right : numpy array
        Values at a relative bin value of [1]

    Returns
    -------
    bin_offset : numpy array
        Array of bins offsets, each in the range [-1/2, 1/2]
    peak_values : numpy array
        Array of the estimated peak values at the interpolated offset
    """
    bin_offset = 1.0/2.0 * (left - right) / (left - 2 * middle + right)
    peak_value = middle + 0.25 * (left - right) * bin_offset
    return bin_offset, peak_value

class LiveBatchMatchedFilter(object):

    """Calculate SNR and signal consistency tests in a batched progression"""

    def __init__(self, templates, snr_threshold, chisq_bins, sg_chisq,
                 maxelements=2**27,
                 snr_abort_threshold=None,
                 newsnr_threshold=None,
                 max_triggers_in_batch=None):
        """Create a batched matchedfilter instance

        templates: list of `FrequencySeries`
            List of templates from the FilterBank class.
        snr_threshold: float
            Minimum value to record peaks in the SNR time series.
        chisq_bins: str
            Str that determines how the number of chisq bins varies as a
            function of the template bank parameters.
        sg_chisq: pycbc.vetoes.SingleDetSGChisq
            Instance of the sg_chisq class to calculate sg_chisq with.
        maxelements: {int, 2**27}
            Maximum size of a batched fourier transform.
        snr_abort_threshold: {float, None}
            If the SNR is above this threshold, do not record any triggers.
        newsnr_threshold: {float, None}
            Only record triggers that have a re-weighted NewSNR above this
        threshold.
        max_triggers_in_batch: {int, None}
            Record X number of the loudest triggers by newsnr in each mpi
        process group. Signal consistency values will also only be calculated
        for these triggers.
        """
        self.snr_abort_threshold = snr_abort_threshold
        self.newsnr_threshold = newsnr_threshold
        self.max_triggers_in_batch = max_triggers_in_batch

        from pycbc import vetoes
        self.power_chisq = vetoes.SingleDetPowerChisq(chisq_bins, None)
        self.sg_chisq = sg_chisq

        durations = numpy.array([1.0 / t.delta_f for t in templates])

        lsort = durations.argsort()
        durations = durations[lsort]
        templates = [templates[li] for li in lsort]

        # Figure out how to chunk together the templates into groups to process
        _, counts = numpy.unique(durations, return_counts=True)
        tsamples = [(len(t) - 1) * 2 for t in templates]
        grabs = maxelements / numpy.unique(tsamples)

        chunks = numpy.array([])
        num = 0
        for count, grab in zip(counts, grabs):
            chunks = numpy.append(chunks, numpy.arange(num, count + num, grab))
            chunks = numpy.append(chunks, [count + num])
            num += count
        chunks = numpy.unique(chunks).astype(numpy.uint32)

        # We now have how many templates to grab at a time.
        self.chunks = chunks[1:] - chunks[0:-1]

        self.out_mem = {}
        self.cout_mem = {}
        self.ifts = {}
        chunk_durations = [durations[i] for i in chunks[:-1]]
        self.chunk_tsamples = [tsamples[int(i)] for i in chunks[:-1]]
        samples = self.chunk_tsamples * self.chunks

        # Create workspace memory for correlate and snr
        mem_ids = [(a, b) for a, b in zip(chunk_durations, self.chunks)]
        mem_types = set(zip(mem_ids, samples))

        self.tgroups, self.mids = [], []
        for i, size in mem_types:
            dur, count = i
            self.out_mem[i] = zeros(size, dtype=numpy.complex64)
            self.cout_mem[i] = zeros(size, dtype=numpy.complex64)
            self.ifts[i] = IFFT(self.cout_mem[i], self.out_mem[i],
                                nbatch=count,
                                size=len(self.cout_mem[i]) // count)

        # Split the templates into their processing groups
        for dur, count in mem_ids:
            tgroup = templates[0:count]
            self.tgroups.append(tgroup)
            self.mids.append((dur, count))
            templates = templates[count:]

        # Associate the snr and corr memory block to each template
        self.corr = []
        for i, tgroup in enumerate(self.tgroups):
            psize = self.chunk_tsamples[i]
            s = 0
            e = psize
            mid = self.mids[i]
            for htilde in tgroup:
                htilde.out = self.out_mem[mid][s:e]
                htilde.cout = self.cout_mem[mid][s:e]
                s += psize
                e += psize
            self.corr.append(BatchCorrelator(tgroup, [t.cout for t in tgroup], len(tgroup[0])))

    def set_data(self, data):
        """Set the data reader object to use"""
        self.data = data
        self.block_id = 0

    def combine_results(self, results):
        """Combine results from different batches of filtering"""
        result = {}
        for key in results[0]:
            result[key] = numpy.concatenate([r[key] for r in results])

    def process_data(self, data_reader):
        """Process the data for all of the templates"""
        self.set_data(data_reader)
        return self.process_all()

    def process_all(self):
        """Process every batch group and return as single result"""
        results = []
        veto_info = []
        while 1:
            result, veto = self._process_batch()
            if result is False: return False
            if result is None: break
            results.append(result)
            veto_info += veto

        result = self.combine_results(results)

        if self.max_triggers_in_batch:
            sort = result['snr'].argsort()[::-1][:self.max_triggers_in_batch]
            for key in result:
                result[key] = result[key][sort]

            tmp = veto_info
            veto_info = [tmp[i] for i in sort]

        result = self._process_vetoes(result, veto_info)

    def _process_vetoes(self, results, veto_info):
        """Calculate signal based vetoes"""
        chisq = numpy.array(numpy.zeros(len(veto_info)), numpy.float32, ndmin=1)
        dof = numpy.array(numpy.zeros(len(veto_info)), numpy.uint32, ndmin=1)
        sg_chisq = numpy.array(numpy.zeros(len(veto_info)), numpy.float32,
                               ndmin=1)
        results['chisq'] = chisq
        results['chisq_dof'] = dof
        results['sg_chisq'] = sg_chisq

        keep = []
        for i, (snrv, norm, l, htilde, stilde) in enumerate(veto_info):
            correlate(htilde, stilde, htilde.cout)
            c, d = self.power_chisq.values(htilde.cout, snrv,
                                           norm, stilde.psd, [l], htilde)
            chisq[i] = c[0] / d[0]
            dof[i] = d[0]

            sgv = self.sg_chisq.values(stilde, htilde, stilde.psd,
                                       snrv, norm, c, d, [l])
            if sgv is not None:
                sg_chisq[i] = sgv[0]

            if self.newsnr_threshold:
                newsnr = ranking.newsnr(results['snr'][i], chisq[i])
                if newsnr >= self.newsnr_threshold:
                    keep.append(i)

        if self.newsnr_threshold:
            keep = numpy.array(keep, dtype=numpy.uint32)
            for key in results:
                results[key] = results[key][keep]

    def _process_batch(self):
        """Process only a single batch group of data"""
        if self.block_id == len(self.tgroups):
            return None, None

        tgroup = self.tgroups[self.block_id]
        psize = self.chunk_tsamples[self.block_id]
        mid = self.mids[self.block_id]
        stilde = self.data.overwhitened_data(tgroup[0].delta_f)
        psd = stilde.psd

        valid_end = int(psize - self.data.trim_padding)
        valid_start = int(valid_end - self.data.blocksize * self.data.sample_rate)

        seg = slice(valid_start, valid_end)

        self.corr[self.block_id].execute(stilde)
        self.ifts[mid].execute()

        self.block_id += 1

        snr = numpy.zeros(len(tgroup), dtype=numpy.complex64)
        time = numpy.zeros(len(tgroup), dtype=numpy.float64)
        templates = numpy.zeros(len(tgroup), dtype=numpy.uint64)
        sigmasq = numpy.zeros(len(tgroup), dtype=numpy.float32)

        time[:] = self.data.start_time

        result = {}
        tkeys = tgroup[0].params.dtype.names
        for key in tkeys:
            result[key] = []

        # Find the peaks in our SNR times series from the various templates
        i = 0
        for htilde in tgroup:
            if hasattr(htilde, 'time_offset'):
                if 'time_offset' not in result:
                    result['time_offset'] = []

            l = htilde.out[seg].abs_arg_max()

            sgm = htilde.sigmasq(psd)
            norm = 4.0 * htilde.delta_f / (sgm ** 0.5)

            l += valid_start
            snrv = numpy.array([htilde.out[l]])

            # If nothing is above threshold we can exit this template
            s = abs(snrv[0]) * norm
            if s < self.snr_threshold:
                continue

            time[i] += float(l - valid_start) / self.data.sample_rate

            # We have an SNR so high that we will drop the entire analysis
            # of this chunk of time!
            if self.snr_abort_threshold is not None and s > self.snr_abort_threshold:
                logging.info("We are seeing some *really* high SNRs, lets"
                             " assume they aren't signals and just give up")
                return False, []

            veto_info.append((snrv, norm, l, htilde, stilde))

            snr[i] = snrv[0] * norm
            sigmasq[i] = sgm
            templates[i] = htilde.id
            if not hasattr(htilde, 'dict_params'):
                htilde.dict_params = {}
                for key in tkeys:
                    htilde.dict_params[key] = htilde.params[key]

            for key in tkeys:
                result[key].append(htilde.dict_params[key])

                result['time_offset'].append(htilde.time_offset)

            i += 1

        result['snr'] = abs(snr[0:i])
        result['coa_phase'] = numpy.angle(snr[0:i])
        result['end_time'] = time[0:i]
        result['template_id'] = templates[0:i]
        result['sigmasq'] = sigmasq[0:i]

            result[key] = numpy.array(result[key])

        if 'time_offset' in result:
            result['time_offset'] = numpy.array(result['time_offset'])

        return result, veto_info

def followup_event_significance(ifo, data_reader, bank,
                                template_id, coinc_times,
                                coinc_threshold=0.005,
                                lookback=150, duration=0.095):
    """ Followup an event in another detector and determine its significance
    """
    from pycbc.waveform import get_waveform_filter_length_in_time
    tmplt = bank.table[template_id]
    length_in_time = get_waveform_filter_length_in_time(tmplt['approximant'],
                                                        tmplt)

    # calculate onsource time range
    from pycbc.detector import Detector
    onsource_start = -numpy.inf
    onsource_end = numpy.inf
    fdet = Detector(ifo)

    for cifo in coinc_times:
        time = coinc_times[cifo]
        dtravel =  Detector(cifo).light_travel_time_to_detector(fdet)
        if time - dtravel > onsource_start:
            onsource_start = time - dtravel
        if time + dtravel < onsource_end:
            onsource_end = time + dtravel

    # Source must be within this time window to be considered a possible
    # coincidence
    onsource_start -= coinc_threshold
    onsource_end += coinc_threshold

    # Calculate how much time needed to calculate significance
    trim_pad = (data_reader.trim_padding * data_reader.strain.delta_t)
    bdur = int(lookback + 2.0 * trim_pad + length_in_time)
    if bdur > data_reader.strain.duration * .75:
        bdur = data_reader.strain.duration * .75

    # Require all strain be valid within lookback time
    if data_reader.state is not None:
        state_start_time = data_reader.strain.end_time \
                - data_reader.reduced_pad * data_reader.strain.delta_t - bdur
        if not data_reader.state.is_extent_valid(state_start_time, bdur):
            return None, None, None, None

    # We won't require that all DQ checks be valid for now, except at
    # onsource time.
    if data_reader.dq is not None:
        dq_start_time = onsource_start - duration / 2.0
        dq_duration = onsource_end - onsource_start + duration
        if not data_reader.dq.is_extent_valid(dq_start_time, dq_duration):

    # Calculate SNR time series for this duration
    htilde = bank.get_template(template_id, min_buffer=bdur)
    stilde = data_reader.overwhitened_data(htilde.delta_f)

    sigma2 = htilde.sigmasq(stilde.psd)
    snr, _, norm = matched_filter_core(htilde, stilde, h_norm=sigma2)

    # Find peak in on-source and determine p-value
    onsrc = snr.time_slice(onsource_start, onsource_end)
    peak = onsrc.abs_arg_max()
    peak_time = peak * snr.delta_t + onsrc.start_time
    peak_value = abs(onsrc[peak])

    bstart = float(snr.start_time) + length_in_time + trim_pad
    bkg = abs(snr.time_slice(bstart, onsource_start)).numpy()

    window = int((onsource_end - onsource_start) * snr.sample_rate)
    nsamples = int(len(bkg) / window)

    peaks = bkg[:nsamples*window].reshape(nsamples, window).max(axis=1)
    pvalue = (1 + (peaks >= peak_value).sum()) / float(1 + nsamples)

    # Return recentered source SNR for bayestar, along with p-value, and trig
    peak_full = int((peak_time - snr.start_time) / snr.delta_t)
    half_dur_samples = int(snr.sample_rate * duration / 2)
    snr_slice = slice(peak_full - half_dur_samples,
                      peak_full + half_dur_samples + 1)
    baysnr = snr[snr_slice]

    logging.info('Adding %s to candidate, pvalue %s, %s samples', ifo,
                 pvalue, nsamples)

    return baysnr * norm, peak_time, pvalue, sigma2

def compute_followup_snr_series(data_reader, htilde, trig_time,
                                duration=0.095, check_state=True,
                                coinc_window=0.05):
    """Given a StrainBuffer, a template frequency series and a trigger time,
    compute a portion of the SNR time series centered on the trigger for its
    rapid sky localization and followup.

    If the trigger time is too close to the boundary of the valid data segment
    the SNR series is calculated anyway and might be slightly contaminated by
    filter and wrap-around effects. For reasonable durations this will only
    affect a small fraction of the triggers and probably in a negligible way.

    data_reader : StrainBuffer
        The StrainBuffer object to read strain data from.

    htilde : FrequencySeries
        The frequency series containing the template waveform.

    trig_time : {float, lal.LIGOTimeGPS}
        The trigger time.

    duration : float (optional)
        Duration of the computed SNR series in seconds. If omitted, it defaults
        to twice the Earth light travel time plus 10 ms of timing uncertainty.

    check_state : boolean
        If True, and the detector was offline or flagged for bad data quality
        at any point during the inspiral, then return (None, None) instead.

    coinc_window : float (optional)
        Maximum possible time between coincident triggers at different
        detectors. This is needed to properly determine data padding.

    Returns
    -------
        The portion of SNR around the trigger. None if the detector is offline
        or has bad data quality, and check_state is True.
    """
    if check_state:
        # was the detector observing for the full amount of involved data?
        state_start_time = trig_time - duration / 2 - htilde.length_in_time
        state_end_time = trig_time + duration / 2
        state_duration = state_end_time - state_start_time
        if data_reader.state is not None:
            if not data_reader.state.is_extent_valid(state_start_time,
                                                     state_duration):
                return None

        # was the data quality ok for the full amount of involved data?
        dq_start_time = state_start_time - data_reader.dq_padding
        dq_duration = state_duration + 2 * data_reader.dq_padding
        if data_reader.dq is not None:
            if not data_reader.dq.is_extent_valid(dq_start_time, dq_duration):

    snr, _, norm = matched_filter_core(htilde, stilde,
                                          h_norm=htilde.sigmasq(stilde.psd))

    valid_end = int(len(snr) - data_reader.trim_padding)
    valid_start = int(valid_end - data_reader.blocksize * snr.sample_rate)

    coinc_samples = int(snr.sample_rate * coinc_window)
    valid_start -= half_dur_samples + coinc_samples
    valid_end += half_dur_samples
    if valid_start < 0 or valid_end > len(snr)-1:
        raise ValueError(('Requested SNR duration ({0} s)'
                          ' too long').format(duration))

    # Onsource slice for Bayestar followup
    onsource_idx = float(trig_time - snr.start_time) * snr.sample_rate
    onsource_idx = int(round(onsource_idx))
    onsource_slice = slice(onsource_idx - half_dur_samples,
                           onsource_idx + half_dur_samples + 1)
    return snr[onsource_slice] * norm

__all__ = ['match', 'matched_filter', 'sigmasq', 'sigma', 'get_cutoff_indices',
           'sigmasq_series', 'make_frequency_series', 'overlap',
           'overlap_cplx', 'matched_filter_core', 'correlate',
           'MatchedFilterControl', 'LiveBatchMatchedFilter',
           'MatchedFilterSkyMaxControl', 'MatchedFilterSkyMaxControlNoPhase',
           'compute_max_snr_over_sky_loc_stat_no_phase',
           'compute_max_snr_over_sky_loc_stat',
           'compute_followup_snr_series',
           'compute_u_val_for_sky_loc_stat_no_phase',
           'compute_u_val_for_sky_loc_stat',
           'followup_event_significance']

# Copyright (C) 2017 Ian Harry
#
#

    z.data[:] = numpy.conjugate(x.data)[:]
    z *= y

# Copyright (C) 2017  Hunter A. Gabbard, Andrew Lundgren,
#                     Duncan Macleod, Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#

"""
This module retrives a timeseries and then calculates
the q-transform of that time series
"""

from six.moves import range
from numpy import ceil, log, exp
from pycbc.types.timeseries import FrequencySeries, TimeSeries
from pycbc.fft import ifft

def qplane(qplane_tile_dict, fseries, return_complex=False):
    """Performs q-transform on each tile for each q-plane and selects
       tile with the maximum energy. Q-transform can then
       be interpolated to a desired frequency and time resolution.

    qplane_tile_dict:
        Dictionary containing a list of q-tile tupples for each q-plane
    fseries: 'pycbc FrequencySeries'
        frequency-series data set
    return_complex: {False, bool}
        Return the raw complex series instead of the normalized power.

    Returns
    -------
    q : float
        The q of the maximum q plane
    times : numpy.ndarray
        The time that the qtransform is sampled.
    freqs : numpy.ndarray
        The frequencies that the qtransform is samled.
    qplane : numpy.ndarray (2d)
        The two dimensional interpolated qtransform of this time series.
    """
    # store q-transforms for each q in a dict
    qplanes = {}
    max_energy, max_key = None, None
    for i, q in enumerate(qplane_tile_dict):
        energies = []
        for f0 in qplane_tile_dict[q]:
            energy = qseries(fseries, q, f0, return_complex=return_complex)
            menergy = abs(energy).max()
            energies.append(energy)

            if i == 0 or menergy > max_energy:
                max_energy = menergy
                max_key = q

        qplanes[q] = energies

    # record q-transform output for peak q
    plane = qplanes[max_key]
    frequencies = qplane_tile_dict[max_key]
    times = plane[0].sample_times.numpy()
    plane = numpy.array([v.numpy() for v in plane])
    return max_key, times, frequencies, numpy.array(plane)

def qtiling(fseries, qrange, frange, mismatch=0.2):
    """Iterable constructor of QTile tuples

    qrange:
        upper and lower bounds of q range
    frange:
        upper and lower bounds of frequency range
    mismatch:
        percentage of desired fractional mismatch

    Returns
    -------
    qplane_tile_dict: 'dict'
        dictionary containing Q-tile tuples for a set of Q-planes
    """
    qplane_tile_dict = {}
    qs = list(_iter_qs(qrange, deltam_f(mismatch)))
    for q in qs:
        qtilefreq = _iter_frequencies(q, frange, mismatch, fseries.duration)
        qplane_tile_dict[q] = numpy.array(list(qtilefreq))

    return qplane_tile_dict

def deltam_f(mismatch):
    """Fractional mismatch between neighbouring tiles

    mismatch: 'float'

    Returns
    -------
    :type: 'float'
    """
    return 2 * (mismatch / 3.) ** (1/2.)

def _iter_qs(qrange, deltam):
    """Iterate over the Q values

    qrange:
    deltam:
        Fractional mismatch between neighbouring tiles

    Returns
    -------
    Q-value:
        Q value for Q-tile
    """

    # work out how many Qs we need
    cumum = log(float(qrange[1]) / qrange[0]) / 2**(1/2.)
    nplanes = int(max(ceil(cumum / deltam), 1))
    dq = cumum / nplanes
    for i in range(nplanes):
        yield qrange[0] * exp(2**(1/2.) * dq * (i + .5))
    return

def _iter_frequencies(q, frange, mismatch, dur):
    """Iterate over the frequencies of this 'QPlane'

    q:
        q value
    frange: 'list'
    mismatch:
    dur:
        duration of timeseries in seconds

    Returns
    -------
    frequencies:
        Q-Tile frequency
    """
    # work out how many frequencies we need
    minf, maxf = frange
    fcum_mismatch = log(float(maxf) / minf) * (2 + q**2)**(1/2.) / 2.
    nfreq = int(max(1, ceil(fcum_mismatch / deltam_f(mismatch))))
    fstep = fcum_mismatch / nfreq
    fstepmin = 1. / dur
    # for each frequency, yield a QTile
    for i in range(nfreq):
        yield (float(minf) *
               exp(2 / (2 + q**2)**(1/2.) * (i + .5) * fstep) //
               fstepmin * fstepmin)
    return

def qseries(fseries, Q, f0, return_complex=False):
    """Calculate the energy 'TimeSeries' for the given fseries

    Q:
        q value
    f0:
        central frequency

    Returns
    -------
    energy: '~pycbc.types.TimeSeries'
        A 'TimeSeries' of the normalized energy from the Q-transform of
        this tile against the data.
    """
    # normalize and generate bi-square window
    qprime = Q / 11**(1/2.)
    norm = numpy.sqrt(315. * qprime / (128. * f0))
    window_size = 2 * int(f0 / qprime * fseries.duration) + 1
    xfrequencies = numpy.linspace(-1., 1., window_size)

    start = int((f0 - (f0 / qprime)) * fseries.duration)
    end = int(start + window_size)
    center = (start + end) // 2

    windowed = fseries[start:end] * (1 - xfrequencies ** 2) ** 2 * norm

    tlen = (len(fseries)-1) * 2
    windowed.resize(tlen)
    windowed.roll(-center)

    # calculate the time series for this q -value
    windowed = FrequencySeries(windowed, delta_f=fseries.delta_f,
                            epoch=fseries.start_time)
    ctseries = TimeSeries(zeros(tlen, dtype=numpy.complex128),
                            delta_t=fseries.delta_t)
    ifft(windowed, ctseries)

    if return_complex:
        return ctseries
    else:
        energy = ctseries.squared_norm()
        medianenergy = numpy.median(energy.numpy())
        return  energy / float(medianenergy)

#
#

#
#
#                                   Preamble
#
#
import lal
import scipy.signal
from pycbc.types import TimeSeries, Array, zeros, FrequencySeries, real_same_precision_as
from pycbc.types import complex_same_precision_as
from pycbc.fft import ifft, fft

_resample_func = {numpy.dtype('float32'): lal.ResampleREAL4TimeSeries,
                 numpy.dtype('float64'): lal.ResampleREAL8TimeSeries}

def lfilter(coefficients, timeseries):
    """ Apply filter coefficients to a time series

    coefficients: numpy.ndarray
        Filter coefficients to apply
    timeseries: numpy.ndarray
        Time series to be filtered.

    Returns
    -------
    tseries: numpy.ndarray
        filtered array
    """
    from pycbc.filter import correlate

    # If there aren't many points just use the default scipy method
    if len(timeseries) < 2**7:
        if hasattr(timeseries, 'numpy'):
            timeseries = timeseries.numpy()
        series = scipy.signal.lfilter(coefficients, 1.0, timeseries)
        return series
    else:
        cseries = (Array(coefficients[::-1] * 1)).astype(timeseries.dtype)
        cseries.resize(len(timeseries))
        cseries.roll(len(timeseries) - len(coefficients) + 1)
        timeseries = Array(timeseries, copy=False)

        flen = len(cseries) // 2 + 1
        ftype = complex_same_precision_as(timeseries)

        cfreq = zeros(flen, dtype=ftype)
        tfreq = zeros(flen, dtype=ftype)

        fft(Array(cseries), cfreq)
        fft(Array(timeseries), tfreq)

        cout = zeros(flen, ftype)
        out = zeros(len(timeseries), dtype=timeseries)

        correlate(cfreq, tfreq, cout)
        ifft(cout, out)

        return out.numpy()  / len(out)

def fir_zero_filter(coeff, timeseries):
    """Filter the timeseries with a set of FIR coefficients

    coeff: numpy.ndarray
        FIR coefficients. Should be and odd length and symmetric.
    timeseries: pycbc.types.TimeSeries

    Returns
    -------
    filtered_series: pycbc.types.TimeSeries
        Return the filtered timeseries, which has been properly shifted to account
    for the FIR filter delay and the corrupted regions zeroed out.
    """
    # apply the filter
    series = lfilter(coeff, timeseries.numpy())

    # reverse the time shift caused by the filter,
    # corruption regions contain zeros
    # If the number of filter coefficients is odd, the central point *should*
    # be included in the output so we only zero out a region of len(coeff) - 1
    data = numpy.zeros(len(timeseries))
    data[len(coeff)//2:len(data)-len(coeff)//2] = series[(len(coeff) // 2) * 2:]

def resample_to_delta_t(timeseries, delta_t, method='butterworth'):
    """Resmple the time_series to delta_t

    Resamples the TimeSeries instance time_series to the given time step,
    delta_t. Only powers of two and real valued time series are supported
    at this time. Additional restrictions may apply to particular filter
    methods.

    time_series: TimeSeries
        The time series to be resampled
    delta_t: float
        The desired time step

    Returns
    -------
    Time Series: TimeSeries
        A TimeSeries that has been resampled to delta_t.

    Raises
    ------
    TypeError:
        time_series is not an instance of TimeSeries.
        time_series is not real valued

    Examples
    --------

    >>> h_plus_sampled = resample_to_delta_t(h_plus, 1.0/2048)
    """
    if not isinstance(timeseries,TimeSeries):
        raise TypeError("Can only resample time series")

    if timeseries.kind is not 'real':
        raise TypeError("Time series must be real")

    if timeseries.sample_rate_close(1.0 / delta_t):
        return timeseries * 1

    if method == 'butterworth':
        lal_data = timeseries.lal()
        _resample_func[timeseries.dtype](lal_data, delta_t)
        data = lal_data.data.data

    elif method == 'ldas':
        factor = int(round(delta_t / timeseries.delta_t))
        numtaps = factor * 20 + 1

        # The kaiser window has been testing using the LDAS implementation
        # and is in the same configuration as used in the original lalinspiral
        filter_coefficients = scipy.signal.firwin(numtaps, 1.0 / factor,
                                                  window=('kaiser', 5))

        # apply the filter and decimate
        data = fir_zero_filter(filter_coefficients, timeseries)[::factor]

    else:
        raise ValueError('Invalid resampling method: %s' % method)

    ts = TimeSeries(data, delta_t = delta_t,
                      dtype=timeseries.dtype,
                      epoch=timeseries._epoch)

    # From the construction of the LDAS FIR filter there will be 10 corrupted samples
    # explanation here http://software.ligo.org/docs/lalsuite/lal/group___resample_time_series__c.html
    ts.corrupted_samples = 10
    return ts

_highpass_func = {numpy.dtype('float32'): lal.HighPassREAL4TimeSeries,
                 numpy.dtype('float64'): lal.HighPassREAL8TimeSeries}

def notch_fir(timeseries, f1, f2, order, beta=5.0):
    """ notch filter the time series using an FIR filtered generated from
    the ideal response passed through a time-domain kaiser window (beta = 5.0)

    The suppression of the notch filter is related to the bandwidth and
    the number of samples in the filter length. For a few Hz bandwidth,
    a length corresponding to a few seconds is typically
    required to create significant suppression in the notched band.
    To achieve frequency resolution df at sampling frequency fs,
    order should be at least fs/df.

        The time series to be notched.
    f1: float
        The start of the frequency suppression.
    f2: float
        The end of the frequency suppression.
    order: int
        Number of corrupted samples on each side of the time series
        (Extent of the filter on either side of zero)
    beta: float
        Beta parameter of the kaiser window that sets the side lobe attenuation.
    """
    k1 = f1 / float((int(1.0 / timeseries.delta_t) / 2))
    k2 = f2 / float((int(1.0 / timeseries.delta_t) / 2))
    coeff = scipy.signal.firwin(order * 2 + 1, [k1, k2], window=('kaiser', beta))
    data = fir_zero_filter(coeff, timeseries)
    return TimeSeries(data, epoch=timeseries.start_time, delta_t=timeseries.delta_t)

def lowpass_fir(timeseries, frequency, order, beta=5.0):
    """ Lowpass filter the time series using an FIR filtered generated from
    the ideal response passed through a kaiser window (beta = 5.0)

        The time series to be low-passed.
    frequency: float
        The frequency below which is suppressed.
    order: int
    """
    k = frequency / float((int(1.0 / timeseries.delta_t) / 2))
    coeff = scipy.signal.firwin(order * 2 + 1, k, window=('kaiser', beta))

def highpass_fir(timeseries, frequency, order, beta=5.0):
    """ Highpass filter the time series using an FIR filtered generated from

        The time series to be high-passed.
    order: int
    """
    coeff = scipy.signal.firwin(order * 2 + 1, k, window=('kaiser', beta), pass_zero=False)

def highpass(timeseries, frequency, filter_order=8, attenuation=0.1):
    """Return a new timeseries that is highpassed.

    Return a new time series that is highpassed above the `frequency`.

    filter_order: {8, int}, optional
        The order of the filter to use when high-passing the time series.
    attenuation: {0.1, float}, optional
        The attenuation of the filter.

    Returns
    -------
        A  new TimeSeries that has been high-passed.

    Raises
    ------

    """

    if not isinstance(timeseries, TimeSeries):

    lal_data = timeseries.lal()
    _highpass_func[timeseries.dtype](lal_data, frequency,
                                     1-attenuation, filter_order)

    return TimeSeries(lal_data.data.data, delta_t = lal_data.deltaT,
                      dtype=timeseries.dtype, epoch=timeseries._epoch)

def interpolate_complex_frequency(series, delta_f, zeros_offset=0, side='right'):
    """Interpolate complex frequency series to desired delta_f.

    Return a new complex frequency series that has been interpolated to the
    desired delta_f.

    series : FrequencySeries
        Frequency series to be interpolated.
    delta_f : float
        The desired delta_f of the output
    zeros_offset : optional, {0, int}
        Number of sample to delay the start of the zero padding
    side : optional, {'right', str}
        The side of the vector to zero pad

    Returns
    -------
    interpolated series : FrequencySeries
        A new FrequencySeries that has been interpolated.
    """
    new_n = int( (len(series)-1) * series.delta_f / delta_f + 1)
    old_N = int( (len(series)-1) * 2 )
    new_N = int( (new_n - 1) * 2 )
    time_series = TimeSeries(zeros(old_N), delta_t =1.0/(series.delta_f*old_N),
                             dtype=real_same_precision_as(series))

    ifft(series, time_series)

    time_series.roll(-zeros_offset)
    time_series.resize(new_N)

    if side == 'left':
        time_series.roll(zeros_offset + new_N - old_N)
    elif side == 'right':
        time_series.roll(zeros_offset)

    out_series = FrequencySeries(zeros(new_n), epoch=series.epoch,
                           delta_f=delta_f, dtype=series.dtype)
    fft(time_series, out_series)

    return out_series

__all__ = ['resample_to_delta_t', 'highpass', 'interpolate_complex_frequency', 'highpass_fir', 'lowpass_fir', 'notch_fir', 'fir_zero_filter']

#
#
#
from pycbc.types import float32, complex64
from .simd_correlate_cython import ccorrf_simd, ccorrf_parallel

"""
This module interfaces to C functions for multiplying
the complex conjugate of one vector by a second vector, writing the output
to a third vector. They do this multi-threaded and with SIMD vectorization.

The code defined here, and the other calling that function,
are imported and used in the CPUCorrelator class defined in
matchedfilter_cpu.py.

Two functions are defined in the 'support' C/Cython module:

ccorrf_simd: Runs on a single core, but vectorized
ccorrf_parallel: Runs multicore, but not explicitly vectorized.
                 Parallelized using OpenMP, and calls ccorrf_simd
"""

def correlate_simd(ht, st, qt):
    htilde = _np.array(ht.data, copy=False, dtype=float32)
    stilde = _np.array(st.data, copy=False, dtype=float32)
    qtilde = _np.array(qt.data, copy=False, dtype=float32)
    arrlen = len(htilde)
    ccorrf_simd(htilde, stilde, qtilde, arrlen)

# We need a segment size (number of complex elements) such that *three* segments
# of that size will fit in the L2 cache. We also want it to be a power of two.
# We are dealing with single-precision complex numbers, which each require 8 bytes.
#
# Our kernel is written to assume a complex correlation of single-precision vectors,
# so that's all we support here.  Note that we are assuming that the correct target
# is that the vectors should fit in L2 cache.  Figuring out cache topology dynamically
# is a harder problem than we attempt to solve here.

    # Since we need 3 vectors fitting in L2 cache, divide by 3
    # We find the nearest power-of-two that fits, and the length
    # of the single-precision complex array that fits into that size.
    pow2 = int(_np.log(opt.LEVEL2_CACHE_SIZE/3.0)/_np.log(2.0))
    default_segsize = pow(2, pow2)/_np.dtype(_np.complex64).itemsize
else:
    default_segsize = 8192

def correlate_parallel(ht, st, qt):
    htilde = _np.array(ht.data, copy=False, dtype=complex64)
    stilde = _np.array(st.data, copy=False, dtype=complex64)
    qtilde = _np.array(qt.data, copy=False, dtype=complex64)
    segsize = default_segsize
    ccorrf_parallel(htilde, stilde, qtilde, arrlen, segsize)

# Copyright (C) 2014  Christopher M. Biwer
#
#
#

#
#
#                                   Preamble
#
#

from scipy.signal import zpk2sos, sosfilt

def filter_zpk(timeseries, z, p, k):
    """Return a new timeseries that was filtered with a zero-pole-gain filter.
    The transfer function in the s-domain looks like:
    .. math::
    \\frac{H(s) = (s - s_1) * (s - s_3) * ... * (s - s_n)}{(s - s_2) * (s - s_4) * ... * (s - s_m)}, m >= n

    The zeroes, and poles entered in Hz are converted to angular frequency,
    along the imaginary axis in the s-domain s=i*omega.  Then the zeroes, and
    poles are bilinearly transformed via:
    .. math::
    z(s) = \\frac{(1 + s*T/2)}{(1 - s*T/2)}

    Where z is the z-domain value, s is the s-domain value, and T is the
    sampling period.  After the poles and zeroes have been bilinearly
    transformed, then the second-order sections are found and filter the data
    using scipy.

    timeseries: TimeSeries
        The TimeSeries instance to be filtered.
    z: array
        Array of zeros to include in zero-pole-gain filter design.
        In units of Hz.
    p: array
        Array of poles to include in zero-pole-gain filter design.
    k: float
        Gain to include in zero-pole-gain filter design. This gain is a
        constant multiplied to the transfer function.

    Returns
    -------
        A  new TimeSeries that has been filtered.

    Examples
    --------
    To apply a 5 zeroes at 100Hz, 5 poles at 1Hz, and a gain of 1e-10 filter
    to a TimeSeries instance, do:
    >>> filtered_data = zpk_filter(timeseries, [100]*5, [1]*5, 1e-10)
    """

    # sanity check type
        raise TypeError("Can only filter TimeSeries instances.")

    # sanity check casual filter
    degree = len(p) - len(z)
    if degree < 0:
        raise TypeError("May not have more zeroes than poles. \
                         Filter is not casual.")

    # cast zeroes and poles as arrays and gain as a float
    z = np.array(z)
    p = np.array(p)
    k = float(k)

    # put zeroes and poles in the s-domain
    # convert from frequency to angular frequency
    z *= -2 * np.pi
    p *= -2 * np.pi

    # get denominator of bilinear transform
    fs = 2.0 * timeseries.sample_rate

    # zeroes in the z-domain
    z_zd = (1 + z/fs) / (1 - z/fs)

    # any zeros that were at infinity are moved to the Nyquist frequency
    z_zd = z_zd[np.isfinite(z_zd)]
    z_zd = np.append(z_zd, -np.ones(degree))

    # poles in the z-domain
    p_zd = (1 + p/fs) / (1 - p/fs)

    # gain change in z-domain
    k_zd = k * np.prod(fs - z) / np.prod(fs - p)

    # get second-order sections
    sos = zpk2sos(z_zd, p_zd, k_zd)

    # filter
    filtered_data = sosfilt(sos, timeseries.numpy())

    return TimeSeries(filtered_data, delta_t = timeseries.delta_t,

# Copyright (C) 2014 Andrew Miller, Alex Nitz, Tito Dal Canton, Christopher M. Biwer
#
#
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Generals
#
"""
This modules contains functions for reading in data from frame files or caches
"""

import lalframe, logging
import lal
import os.path, glob, time
import gwdatafind
from six.moves.urllib.parse import urlparse
from pycbc.types import TimeSeries, zeros

# map LAL series types to corresponding functions and Numpy types
_fr_type_map = {
    lal.S_TYPE_CODE: [
        lalframe.FrStreamReadREAL4TimeSeries, numpy.float32,
        lal.CreateREAL4TimeSeries,
        lalframe.FrStreamGetREAL4TimeSeriesMetadata,
        lal.CreateREAL4Sequence,
        lalframe.FrameAddREAL4TimeSeriesProcData
    ],
    lal.D_TYPE_CODE: [
        lalframe.FrStreamReadREAL8TimeSeries, numpy.float64,
        lal.CreateREAL8TimeSeries,
        lalframe.FrStreamGetREAL8TimeSeriesMetadata,
        lal.CreateREAL8Sequence,
        lalframe.FrameAddREAL8TimeSeriesProcData
    ],
    lal.C_TYPE_CODE: [
        lalframe.FrStreamReadCOMPLEX8TimeSeries, numpy.complex64,
        lal.CreateCOMPLEX8TimeSeries,
        lalframe.FrStreamGetCOMPLEX8TimeSeriesMetadata,
        lal.CreateCOMPLEX8Sequence,
        lalframe.FrameAddCOMPLEX8TimeSeriesProcData
    ],
    lal.Z_TYPE_CODE: [
        lalframe.FrStreamReadCOMPLEX16TimeSeries, numpy.complex128,
        lal.CreateCOMPLEX16TimeSeries,
        lalframe.FrStreamGetCOMPLEX16TimeSeriesMetadata,
        lal.CreateCOMPLEX16Sequence,
        lalframe.FrameAddCOMPLEX16TimeSeriesProcData
    ],
    lal.U4_TYPE_CODE: [
        lalframe.FrStreamReadUINT4TimeSeries, numpy.uint32,
        lal.CreateUINT4TimeSeries,
        lalframe.FrStreamGetUINT4TimeSeriesMetadata,
        lal.CreateUINT4Sequence,
        lalframe.FrameAddUINT4TimeSeriesProcData
    ],
    lal.I4_TYPE_CODE: [
        lalframe.FrStreamReadINT4TimeSeries, numpy.int32,
        lal.CreateINT4TimeSeries,
        lalframe.FrStreamGetINT4TimeSeriesMetadata,
        lal.CreateINT4Sequence,
        lalframe.FrameAddINT4TimeSeriesProcData
    ],
}

def _read_channel(channel, stream, start, duration):
    """ Get channel using lalframe """
    channel_type = lalframe.FrStreamGetTimeSeriesType(channel, stream)
    read_func = _fr_type_map[channel_type][0]
    d_type = _fr_type_map[channel_type][1]
    data = read_func(stream, channel, start, duration, 0)
    return TimeSeries(data.data.data, delta_t=data.deltaT, epoch=start,
                      dtype=d_type)

def _is_gwf(file_path):
    """Test if a file is a frame file by checking if its contents begins with
    the magic string 'IGWD'."""
    try:
        with open(file_path, 'rb') as f:
            if f.read(4) == b'IGWD':
                return True
    except IOError:
        pass
    return False

def locations_to_cache(locations, latest=False):
    """ Return a cumulative cache file build from the list of locations

    locations : list
        A list of strings containing files, globs, or cache files used to build
    a combined lal cache file object.
    latest : Optional, {False, Boolean}
        Only return a cache with the most recent frame in the locations.
        If false, all results are returned.

    Returns
    -------
    cache : lal.Cache
        A cumulative lal cache object containing the files derived from the
    list of locations
    """
    cum_cache = lal.Cache()
    for source in locations:
        flist = glob.glob(source)
        if latest:
            def relaxed_getctime(fn):
                # when building a cache from a directory of temporary
                # low-latency frames, files might disappear between
                # the glob() and getctime() calls
                try:
                    return os.path.getctime(fn)
                except OSError:
                    return 0
            flist = [max(flist, key=relaxed_getctime)]

        for file_path in flist:
            dir_name, file_name = os.path.split(file_path)
            _, file_extension = os.path.splitext(file_name)

            if file_extension in [".lcf", ".cache"]:
                cache = lal.CacheImport(file_path)
            elif file_extension == ".gwf" or _is_gwf(file_path):
                cache = lalframe.FrOpen(str(dir_name), str(file_name)).cache
            else:
                raise TypeError("Invalid location name")

            cum_cache = lal.CacheMerge(cum_cache, cache)
    return cum_cache

def read_frame(location, channels, start_time=None,
               end_time=None, duration=None, check_integrity=False,
               sieve=None):
    """Read time series from frame data.

    Using the `location`, which can either be a frame file ".gwf" or a
    frame cache ".gwf", read in the data for the given channel(s) and output
    as a TimeSeries or list of TimeSeries.

    location : string
        A source of gravitational wave frames. Either a frame filename
        (can include pattern), a list of frame files, or frame cache file.
    channels : string or list of strings
        Either a string that contains the channel name or a list of channel
        name strings.
    start_time : {None, LIGOTimeGPS}, optional
        The gps start time of the time series. Defaults to reading from the
        beginning of the available frame(s).
    end_time : {None, LIGOTimeGPS}, optional
        The gps end time of the time series. Defaults to the end of the frame.
        Note, this argument is incompatible with `duration`.
    duration : {None, float}, optional
        The amount of data to read in seconds. Note, this argument is
        incompatible with `end`.
    check_integrity : {True, bool}, optional
        Test the frame files for internal integrity.
    sieve : string, optional
        Selects only frames where the frame URL matches the regular
        expression sieve

    Returns
    -------
    Frame Data: TimeSeries or list of TimeSeries
        A TimeSeries or a list of TimeSeries, corresponding to the data from
        the frame file/cache for a given channel or channels.
    """

    if end_time and duration:
        raise ValueError("end time and duration are mutually exclusive")

    if type(location) is list:
        locations = location
    else:
        locations = [location]

    cum_cache = locations_to_cache(locations)
    if sieve:
        logging.info("Using frames that match regexp: %s", sieve)
        lal.CacheSieve(cum_cache, 0, 0, None, None, sieve)
    if start_time is not None and end_time is not None:
        # Before sieving, check if this is sane. Otherwise it will fail later.
        if (int(math.ceil(end_time)) - int(start_time)) <= 0:
            raise ValueError("Negative or null duration")
        lal.CacheSieve(cum_cache, int(start_time), int(math.ceil(end_time)),
                       None, None, None)

    stream = lalframe.FrStreamCacheOpen(cum_cache)
    stream.mode = lalframe.FR_STREAM_VERBOSE_MODE

    if check_integrity:
        stream.mode = (stream.mode | lalframe.FR_STREAM_CHECKSUM_MODE)

    lalframe.FrSetMode(stream.mode, stream)

    # determine duration of data
    if type(channels) is list:
        first_channel = channels[0]
    else:
        first_channel = channels

    data_length = lalframe.FrStreamGetVectorLength(first_channel, stream)
    channel_type = lalframe.FrStreamGetTimeSeriesType(first_channel, stream)
    create_series_func = _fr_type_map[channel_type][2]
    get_series_metadata_func = _fr_type_map[channel_type][3]
    series = create_series_func(first_channel, stream.epoch, 0, 0,
                                lal.ADCCountUnit, 0)
    get_series_metadata_func(series, stream)
    data_duration = data_length * series.deltaT

    if start_time is None:
        start_time = stream.epoch*1
    if end_time is None:
        end_time = start_time + data_duration

    if type(start_time) is not lal.LIGOTimeGPS:
        start_time = lal.LIGOTimeGPS(start_time)
    if type(end_time) is not lal.LIGOTimeGPS:
        end_time = lal.LIGOTimeGPS(end_time)

    if duration is None:
        duration = float(end_time - start_time)
    else:
        duration = float(duration)

    # lalframe behaves dangerously with invalid duration so catch it here
    if duration <= 0:
        raise ValueError("Negative or null duration")
    #if duration > data_duration:
    #    raise ValueError("Requested duration longer than available data")

        all_data = []
        for channel in channels:
            channel_data = _read_channel(channel, stream, start_time, duration)
            lalframe.FrStreamSeek(stream, start_time)
            all_data.append(channel_data)
        return all_data
    else:
        return _read_channel(channels, stream, start_time, duration)

def datafind_connection(server=None):
    """ Return a connection to the datafind server

    server : {SERVER:PORT, string}, optional
       A string representation of the server and port.
       The port may be ommitted.

    Returns
    --------
    connection
        The open connection to the datafind server.
    """
    return gwdatafind.connect(host=server)

def frame_paths(frame_type, start_time, end_time, server=None, url_type='file'):
    """Return the paths to a span of frame files

    frame_type : string
        The string representation of the frame type (ex. 'H1_ER_C00_L1')
    start_time : int
        The start time that we need the frames to span.
    end_time : int
        The end time that we need the frames to span.
    server : {None, SERVER:PORT string}, optional
        Optional string to specify the datafind server to use. By default an
        attempt is made to use a local datafind server.
    url_type : string
        Returns only frame URLs with a particular scheme or head such
        as "file" or "gsiftp". Default is "file", which queries locally
        stored frames. Option can be disabled if set to None.
    Returns
    -------
    paths : list of paths
        The list of paths to the frame files.

    Examples
    --------
    >>> paths = frame_paths('H1_LDAS_C02_L2', 968995968, 968995968+2048)
    """
    site = frame_type[0]
    connection = datafind_connection(server)
    connection.find_times(site, frame_type,
                          gpsstart=start_time, gpsend=end_time)
    cache = connection.find_frame_urls(site, frame_type, start_time, end_time,urltype=url_type)
    return [urlparse(entry).path for entry in cache]

def query_and_read_frame(frame_type, channels, start_time, end_time,
                         sieve=None, check_integrity=False):

    Query for the locatin of physical frames matching the frame type. Return
    a time series containing the channel between the given start and end times.

        The type of frame file that we are looking for.
    start_time : LIGOTimeGPS or int
    end_time : LIGOTimeGPS or int
    check_integrity : boolean
        Do an expensive checksum of the file before returning.

    Returns
    -------

    Examples
    --------
    >>> ts = query_and_read_frame('H1_LDAS_C02_L2', 'H1:LDAS-STRAIN',
    >>>                               968995968, 968995968+2048)
    """
    # Allows compatibility with our standard tools
    # We may want to place this into a higher level frame getting tool
    if frame_type == 'LOSC_STRAIN':
        from pycbc.frame.losc import read_strain_losc
        if not isinstance(channels, list):
            channels = [channels]
        data = [read_strain_losc(c[:2], start_time, end_time)
                for c in channels]
        return data if len(data) > 1 else data[0]
    if frame_type == 'LOSC':
        from pycbc.frame.losc import read_frame_losc
        return read_frame_losc(channels, start_time, end_time)

    logging.info('querying datafind server')
    paths = frame_paths(frame_type, int(start_time), int(numpy.ceil(end_time)))
    logging.info('found files: %s' % (' '.join(paths)))
    return read_frame(paths, channels,
                      start_time=start_time,
                      end_time=end_time,
                      sieve=sieve,
                      check_integrity=check_integrity)

__all__ = ['read_frame', 'frame_paths',
           'datafind_connection',
           'query_and_read_frame']

def write_frame(location, channels, timeseries):
    """Write a list of time series to a single frame file.

        A frame filename.
        A TimeSeries or list of TimeSeries, corresponding to the data to be
        written to the frame file for a given channel.
    """
    # check if a single channel or a list of channels
    if type(channels) is list and type(timeseries) is list:
        channels = channels
        timeseries = timeseries
    else:
        channels = [channels]
        timeseries = [timeseries]

    # check that timeseries have the same start and end time
    gps_start_times = {series.start_time for series in timeseries}
    gps_end_times = {series.end_time for series in timeseries}
    if len(gps_start_times) != 1 or len(gps_end_times) != 1:
        raise ValueError("Start and end times of TimeSeries must be identical.")

    # check that start, end time, and duration are integers
    gps_start_time = gps_start_times.pop()
    gps_end_time = gps_end_times.pop()
    duration = int(gps_end_time - gps_start_time)
    if gps_start_time % 1 or gps_end_time % 1:
        raise ValueError("Start and end times of TimeSeries must be integer seconds.")

    # create frame
    frame = lalframe.FrameNew(epoch=gps_start_time, duration=duration,
                              project='', run=1, frnum=1,
                              detectorFlags=lal.LALDETECTORTYPE_ABSENT)

    for i,tseries in enumerate(timeseries):
        # get data type
        for seriestype in _fr_type_map.keys():
            if _fr_type_map[seriestype][1] == tseries.dtype:
                create_series_func = _fr_type_map[seriestype][2]
                create_sequence_func = _fr_type_map[seriestype][4]
                add_series_func = _fr_type_map[seriestype][5]
                break

        # add time series to frame
        series = create_series_func(channels[i], tseries.start_time,
                       0, tseries.delta_t, lal.ADCCountUnit,
                       len(tseries.numpy()))
        series.data = create_sequence_func(len(tseries.numpy()))
        series.data.data = tseries.numpy()
        add_series_func(frame, series)

    # write frame
    lalframe.FrameWrite(frame, location)

class DataBuffer(object):

    """A linear buffer that acts as a FILO for reading in frame data
    """

    def __init__(self, frame_src,
                 channel_name,
                 start_time,
                 max_buffer=2048,
                 force_update_cache=True,
                 increment_update_cache=None,
                 dtype=numpy.float64):
        """ Create a rolling buffer of frame data

        ---------
        frame_src: str of list of strings
            Strings that indicate where to read from files from. This can be a
        list of frame files, a glob, etc.
        channel_name: str
            Name of the channel to read from the frame files
        start_time:
            Time to start reading from.
        max_buffer: {int, 2048}, Optional
            Length of the buffer in seconds
        dtype: {dtype, numpy.float32}, Optional
            Data type to use for the interal buffer
        """
        self.frame_src = frame_src
        self.channel_name = channel_name
        self.read_pos = start_time
        self.force_update_cache = force_update_cache
        self.increment_update_cache = increment_update_cache
        self.detector = channel_name.split(':')[0]

        self.update_cache()
        self.channel_type, self.raw_sample_rate = self._retrieve_metadata(self.stream, self.channel_name)

        raw_size = self.raw_sample_rate * max_buffer
        self.raw_buffer = TimeSeries(zeros(raw_size, dtype=dtype),
                                     copy=False,
                                     epoch=start_time - max_buffer,
                                     delta_t=1.0/self.raw_sample_rate)

    def update_cache(self):
        """Reset the lal cache. This can be used to update the cache if the
        result may change due to more files being added to the filesystem,
        for example.
        """
        cache = locations_to_cache(self.frame_src, latest=True)
        stream = lalframe.FrStreamCacheOpen(cache)
        self.stream = stream

    def _retrieve_metadata(stream, channel_name):
        """Retrieve basic metadata by reading the first file in the cache

        stream: lal stream object
            Stream containing a channel we want to learn about
            The name of the channel we want to know the dtype and sample rate of

        Returns
        -------
        channel_type: lal type enum
            Enum value which indicates the dtype of the channel
        sample_rate: int
            The sample rate of the data within this channel
        """
        lalframe.FrStreamGetVectorLength(channel_name, stream)
        channel_type = lalframe.FrStreamGetTimeSeriesType(channel_name, stream)
        create_series_func = _fr_type_map[channel_type][2]
        get_series_metadata_func = _fr_type_map[channel_type][3]
        series = create_series_func(channel_name, stream.epoch, 0, 0,
                            lal.ADCCountUnit, 0)
        get_series_metadata_func(series, stream)
        return channel_type, int(1.0/series.deltaT)

    def _read_frame(self, blocksize):
        """Try to read the block of data blocksize seconds long

        blocksize: int
            The number of seconds to attempt to read from the channel

        Returns
        -------
        data: TimeSeries
            TimeSeries containg 'blocksize' seconds of frame data

        Raises
        ------
        RuntimeError:
            If data cannot be read for any reason
        """
        try:
            read_func = _fr_type_map[self.channel_type][0]
            dtype = _fr_type_map[self.channel_type][1]
            data = read_func(self.stream, self.channel_name,
                             self.read_pos, int(blocksize), 0)
            return TimeSeries(data.data.data, delta_t=data.deltaT,
                              epoch=self.read_pos,
                              dtype=dtype)
        except Exception:
            raise RuntimeError('Cannot read {0} frame data'.format(self.channel_name))

    def null_advance(self, blocksize):
        """Advance and insert zeros

        """
        self.raw_buffer.roll(-int(blocksize * self.raw_sample_rate))
        self.read_pos += blocksize
        self.raw_buffer.start_time += blocksize

    def advance(self, blocksize):
        """Add blocksize seconds more to the buffer, push blocksize seconds
        from the beginning.

        """
        ts = self._read_frame(blocksize)

        self.raw_buffer.roll(-len(ts))
        self.raw_buffer[-len(ts):] = ts[:]
        return ts

    def update_cache_by_increment(self, blocksize):
        """Update the internal cache by starting from the first frame
        and incrementing.

        Guess the next frame file name by incrementing from the first found
        one. This allows a pattern to be used for the GPS folder of the file,
        which is indicated by `GPSX` where x is the number of digits to use.

            Number of seconds to increment the next frame file.
        """
        start = float(self.raw_buffer.end_time)
        end = float(start + blocksize)

        if not hasattr(self, 'dur'):
            fname = glob.glob(self.frame_src[0])[0]
            fname = os.path.splitext(os.path.basename(fname))[0].split('-')

            self.beg = '-'.join([fname[0], fname[1]])
            self.ref = int(fname[2])
            self.dur = int(fname[3])

        fstart = int(self.ref + numpy.floor((start - self.ref) / float(self.dur)) * self.dur)
        starts = numpy.arange(fstart, end, self.dur).astype(numpy.int)

        keys = []
        for s in starts:
            pattern = self.increment_update_cache
            if 'GPS' in pattern:
                n = int(pattern[int(pattern.index('GPS') + 3)])
                pattern = pattern.replace('GPS%s' % n, str(s)[0:n])

            name = '%s/%s-%s-%s.gwf' % (pattern, self.beg, s, self.dur)
            # check that file actually exists, else abort now
            if not os.path.exists(name):
                raise RuntimeError

            keys.append(name)
        cache = locations_to_cache(keys)
        self.channel_type, self.raw_sample_rate = \
            self._retrieve_metadata(self.stream, self.channel_name)

    def attempt_advance(self, blocksize, timeout=10):
        """ Attempt to advance the frame buffer. Retry upon failure, except
        if the frame file is beyond the timeout limit.

        timeout: {int, 10}, Optional
            Number of seconds before giving up on reading a frame

        Returns
        -------
        """
        if self.force_update_cache:
            self.update_cache()

        try:
            if self.increment_update_cache:
                self.update_cache_by_increment(blocksize)

            return DataBuffer.advance(self, blocksize)

        except RuntimeError:
            if pycbc.gps_now() > timeout + self.raw_buffer.end_time:
                # The frame is not there and it should be by now, so we give up
                # and treat it as zeros
                DataBuffer.null_advance(self, blocksize)
            else:
                # I am too early to give up on this frame, so we should try again
                time.sleep(0.1)
                return self.attempt_advance(blocksize, timeout=timeout)

class StatusBuffer(DataBuffer):

    """ Read state vector or DQ information from a frame file """

                       channel_name,
                       start_time,
                       max_buffer=2048,
                       valid_mask=3,
                       force_update_cache=False,
                       increment_update_cache=None,
                       valid_on_zero=False):
        """ Create a rolling buffer of status data from a frame

        ---------
            list of frame files, a glob, etc.
        valid_mask: {int, HOFT_OK | SCIENCE_INTENT}, Optional
            Set of flags that must be on to indicate valid frame data.
        valid_on_zero: bool
            If True, `valid_mask` is ignored and the status is considered
            "good" simply when the channel is zero.
        """
        DataBuffer.__init__(self, frame_src, channel_name, start_time,
                            max_buffer=max_buffer,
                            force_update_cache=force_update_cache,
                            increment_update_cache=increment_update_cache,
                            dtype=numpy.int32)
        self.valid_mask = valid_mask
        self.valid_on_zero = valid_on_zero

    def check_valid(self, values, flag=None):
        """Check if the data contains any non-valid status information

        values: pycbc.types.Array
            Array of status information
        flag: str, optional
            Override the default valid mask with a user defined mask.

        Returns
        -------
        status: boolean
            Returns True if all of the status information if valid,
             False if any is not.
        """
        if self.valid_on_zero:
            valid = values.numpy() == 0
        else:
            if flag is None:
                flag = self.valid_mask
            valid = numpy.bitwise_and(values.numpy(), flag) == flag
        return bool(numpy.all(valid))

    def is_extent_valid(self, start_time, duration, flag=None):
        """Check if the duration contains any non-valid frames

        start_time: int
            Beginning of the duration to check in gps seconds
        duration: int
            Number of seconds after the start_time to check

        Returns
        -------
            False if any is not.
        """
        sr = self.raw_buffer.sample_rate
        s = int((start_time - self.raw_buffer.start_time) * sr)
        e = s + int(duration * sr) + 1
        data = self.raw_buffer[s:e]
        return self.check_valid(data, flag=flag)

    def indices_of_flag(self, start_time, duration, times, padding=0):
        """ Return the indices of the times lying in the flagged region

            Beginning time to request for
            Number of seconds to check.
        padding: float
            Number of seconds to add around flag inactive times to be considered
        inactive as well.

        Returns
        -------
        indices: numpy.ndarray
            Array of indices marking the location of triggers within valid
        time.
        """
        from pycbc.events.veto import indices_outside_times
        s = int((start_time - self.raw_buffer.start_time - padding) * sr) - 1
        e = s + int((duration + padding) * sr) + 1
        stamps = data.sample_times.numpy()

            invalid = data.numpy() != 0
        else:
            invalid = numpy.bitwise_and(data.numpy(), self.valid_mask) \
                    != self.valid_mask

        starts = stamps[invalid] - padding
        ends = starts + 1.0 / sr + padding * 2.0
        idx = indices_outside_times(times, starts, ends)
        return idx

        """ Add blocksize seconds more to the buffer, push blocksize seconds

        Returns
        -------
        """
        try:
            ts = DataBuffer.advance(self, blocksize)
            return self.check_valid(ts)
            self.null_advance(blocksize)

# Copyright (C) 2017 Alex Nitz
#
#
#
"""
This modules contains functions for getting data from the LOSC
"""

_losc_url = "https://www.gw-openscience.org/archive/links/%s/%s/%s/%s/json/"

def get_run(time):
    if 1164556817 <= time <= 1187733618:
        return 'O2_16KHZ_R1'
    if 1126051217 <= time <= 1137254417:
        return 'O1'
    elif 815011213 <= time <= 875318414:
        return 'S5'
    elif 930787215 <= time <= 971568015:
        return 'S6'
    else:
        raise ValueError('Time %s not available in a public dataset' % time)

def _get_channel(time):
        return 'GWOSC-16KHZ_R1_STRAIN'
    else:
        return 'LOSC-STRAIN'

def losc_frame_json(ifo, start_time, end_time):
    """ Get the information about the public data files in a duration of time

    ifo: str
        The name of the IFO to find the information about.
        The gps time in GPS seconds
        The end time in GPS seconds

    Returns
    -------
    info: dict
        A dictionary containing information about the files that span the
        requested times.
    """
    import json
    try:
        from urllib.request import urlopen
    except ImportError:  # python < 3
        from urllib import urlopen
    run = get_run(start_time)
    run2 = get_run(end_time)
    if run != run2:
        raise ValueError('Spanning multiple runs is not currently supported.'
                         'You have requested data that uses '
                         'both %s and %s' % (run, run2))

    url = _losc_url % (run, ifo, int(start_time), int(end_time))

    try:
        return json.loads(urlopen(url).read().decode())
        print(e)
        raise ValueError('Failed to find gwf files for '
            'ifo=%s, run=%s, between %s-%s' % (ifo, run, start_time, end_time))

def losc_frame_urls(ifo, start_time, end_time):
    """ Get a list of urls to losc frame files

    ifo: str

    Returns
    -------
    frame_files: list
    """
    data = losc_frame_json(ifo, start_time, end_time)['strain']
    return [d['url'] for d in data if d['format'] == 'gwf']

def read_frame_losc(channels, start_time, end_time):
    """ Read channels from losc data

    channels: str or list
        The channel name to read or list of channel names.

    Returns
    -------
    ts: TimeSeries
        Returns a timeseries or list of timeseries with the requested data.
    """
    from pycbc.frame import read_frame
    if not isinstance(channels, list):
    ifos = [c[0:2] for c in channels]
    urls = {}
        urls[ifo] = losc_frame_urls(ifo, start_time, end_time)
        if len(urls[ifo]) == 0:
            raise ValueError("No data found for %s so we "
                             "can't produce a time series" % ifo)

    fnames = {ifo:[] for ifo in ifos}
        for url in urls[ifo]:
            fname = get_file(url, cache=True)
            fnames[ifo].append(fname)

    ts = [read_frame(fnames[channel[0:2]], channel,
           start_time=start_time, end_time=end_time) for channel in channels]
    if len(ts) == 1:
        return ts[0]
    else:
        return ts

def read_strain_losc(ifo, start_time, end_time):
    """ Get the strain data from the LOSC data

    ifo: str
        The name of the IFO to read data for. Ex. 'H1', 'L1', 'V1'

    Returns
    -------
        Returns a timeseries with the strain data.
    """
    channel = _get_channel(start_time)
    return read_frame_losc('%s:%s' % (ifo, channel), start_time, end_time)

# Copyright (C) 2019 Alex Nitz
#
#
#
"""
This modules contains functions for reading in data from hdf stores
"""

def read_store(fname, channel, start_time, end_time):
    """ Read time series data from hdf store

    fname: str
        Name of hdf store file
    channel: str
        Channel name to read
        GPS time to start reading from
        GPS time to end time series

    Returns
    -------
    ts: pycbc.types.TimeSeries
        Time series containing the requested data

    """
    fhandle = h5py.File(fname, 'r')
    if channel not in fhandle:
        raise ValueError('Could not find channel name {}'.format(channel))

    # Determine which segment data lies in (can only read contiguous data now)
    starts = fhandle[channel]['segments']['start'][:]
    ends = fhandle[channel]['segments']['end'][:]

    diff = start_time - starts
    loc = numpy.where(diff >= 0)[0]
    sidx = loc[diff[loc].argmin()]

    stime = starts[sidx]
    etime = ends[sidx]

    if stime > start_time:
        raise ValueError("Cannot read data segment before {}".format(stime))

    if etime < end_time:
        raise ValueError("Cannot read data segment past {}".format(etime))

    data = fhandle[channel][str(sidx)]
    sample_rate = len(data) / (etime - stime)

    start = int((start_time - stime) * sample_rate)
    end = int((end_time - stime) * sample_rate)
    return TimeSeries(data[start:end], delta_t=1.0/sample_rate,
                      epoch=start_time)

#
#

#
#
#                                   Preamble
#
#
"""
This modules provides classes and functions for determining when Markov Chains
have burned in.
"""

from abc import ABCMeta, abstractmethod
from six import add_metaclass
from scipy.stats import ks_2samp

from pycbc.io.record import get_vars_from_arg

# The value to use for a burn-in iteration if a chain is not burned in
NOT_BURNED_IN_ITER = -1

#
#
#                              Convenience functions
#
#

def ks_test(samples1, samples2, threshold=0.9):
    """Applies a KS test to determine if two sets of samples are the same.

    The ks test is applied parameter-by-parameter. If the two-tailed p-value
    returned by the test is greater than ``threshold``, the samples are
    considered to be the same.

    samples1 : dict
        Dictionary of mapping parameters to the first set of samples.
    samples2 : dict
        Dictionary of mapping parameters to the second set of samples.
    threshold : float
        The thershold to use for the p-value. Default is 0.9.

    Returns
    -------
    dict :
        Dictionary mapping parameter names to booleans indicating whether the
        given parameter passes the KS test.
    """
    is_the_same = {}
    assert set(samples1.keys()) == set(samples2.keys()), (
        "samples1 and 2 must have the same parameters")
    # iterate over the parameters
    for param in samples1:
        s1 = samples1[param]
        s2 = samples2[param]
        _, p_value = ks_2samp(s1, s2)
        is_the_same[param] = p_value > threshold
    return is_the_same

def max_posterior(lnps_per_walker, dim):
    """Burn in based on samples being within dim/2 of maximum posterior.

    lnps_per_walker : 2D array
        Array of values that are proportional to the log posterior values. Must
        have shape ``nwalkers x niterations``.
    dim : int
        The dimension of the parameter space.

    Returns
    -------
    burn_in_idx : array of int
        The burn in indices of each walker. If a walker is not burned in, its
        index will be be equal to the length of the chain.
    is_burned_in : array of bool
        Whether or not a walker is burned in.
    """
    if len(lnps_per_walker.shape) != 2:
        raise ValueError("lnps_per_walker must have shape "
                         "nwalkers x niterations")
    # find the value to compare against
    max_p = lnps_per_walker.max()
    criteria = max_p - dim/2.
    nwalkers, _ = lnps_per_walker.shape
    burn_in_idx = numpy.empty(nwalkers, dtype=int)
    is_burned_in = numpy.empty(nwalkers, dtype=bool)
    # find the first iteration in each chain where the logpost has exceeded
    # max_p - dim/2
    for ii in range(nwalkers):
        chain = lnps_per_walker[ii, :]
        passedidx = numpy.where(chain >= criteria)[0]
        is_burned_in[ii] = passedidx.size > 0
        if is_burned_in[ii]:
            burn_in_idx[ii] = passedidx[0]
        else:
            burn_in_idx[ii] = NOT_BURNED_IN_ITER
    return burn_in_idx, is_burned_in

def posterior_step(logposts, dim):
    """Finds the last time a chain made a jump > dim/2.

    logposts : array
        1D array of values that are proportional to the log posterior values.
    dim : int

    Returns
    -------
    int
        The index of the last time the logpost made a jump > dim/2. If that
        never happened, returns 0.
    """
    if logposts.ndim > 1:
        raise ValueError("logposts must be a 1D array")
    criteria = dim/2.
    dp = numpy.diff(logposts)
    indices = numpy.where(dp >= criteria)[0]
    if indices.size > 0:
        idx = indices[-1] + 1
    else:
        idx = 0
    return idx

def nacl(nsamples, acls, nacls=5):
    """Burn in based on ACL.

    This applies the following test to determine burn in:

    1. The first half of the chain is ignored.

    2. An ACL is calculated from the second half.

    3. If ``nacls`` times the ACL is < the length of the chain / 2,
       the chain is considered to be burned in at the half-way point.

    nsamples : int
        The number of samples of in the chain(s).
    acls : dict
        Dictionary of parameter -> ACL(s). The ACLs for each parameter may
        be an integer or an array of integers (for multiple chains).
    nacls : int, optional
        The number of ACLs the chain(s) must have gone past the halfway point
        in order to be considered burned in. Default is 5.

    Returns
    -------
    dict
        Dictionary of parameter -> boolean(s) indicating if the chain(s) pass
        the test. If an array of values was provided for the acls, the values
        will be arrays of booleans.
    """
    kstart = int(nsamples / 2.)
    return {param: (nacls * acl) < kstart for (param, acl) in acls.items()}

def evaluate_tests(burn_in_test, test_is_burned_in, test_burn_in_iter):
    """Evaluates burn in data from multiple tests.

    The iteration to use for burn-in depends on the logic in the burn-in
    test string. For example, if the test was 'max_posterior | nacl' and
    max_posterior burned-in at iteration 5000 while nacl burned in at
    iteration 6000, we'd want to use 5000 as the burn-in iteration.
    However, if the test was 'max_posterior & nacl', we'd want to use
    6000 as the burn-in iteration. This function handles all cases by
    doing the following: first, take the collection of burn in iterations
    from all the burn in tests that were applied.  Next, cycle over the
    iterations in increasing order, checking which tests have burned in
    by that point. Then evaluate the burn-in string at that point to see
    if it passes, and if so, what the iteration is. The first point that
    the test passes is used as the burn-in iteration.

    burn_in_test : str
        The test to apply; e.g., ``'max_posterior & nacl'``.
    test_is_burned_in : dict
        Dictionary of test name -> boolean indicating whether a specific burn
        in test has passed.
    test_burn_in_iter : dict
        Dictionary of test name -> int indicating when a specific test burned
        in.

    Returns
    -------
    is_burned_in : bool
        Whether or not the data passes all burn in tests.
    burn_in_iteration :
        The iteration at which all the tests pass. If the tests did not all
        pass (``is_burned_in`` is false), then returns
        :py:data:`NOT_BURNED_IN_ITER`.
    """
    burn_in_iters = numpy.unique(list(test_burn_in_iter.values()))
    burn_in_iters.sort()
    for ii in burn_in_iters:
        test_results = {t: (test_is_burned_in[t] &
                            0 <= test_burn_in_iter[t] <= ii)
                        for t in test_is_burned_in}
        is_burned_in = eval(burn_in_test, {"__builtins__": None},
                            test_results)
        if is_burned_in:
            break
    if not is_burned_in:
        ii = NOT_BURNED_IN_ITER
    return is_burned_in, ii

#
#
#                              Burn in classes
#
#

@add_metaclass(ABCMeta)
class BaseBurnInTests(object):
    """Base class for burn in tests."""
    available_tests = ('halfchain', 'min_iterations', 'max_posterior',
                       'posterior_step', 'nacl',
                       )

    # pylint: disable=unnecessary-pass

    def __init__(self, sampler, burn_in_test, **kwargs):
        self.sampler = sampler
        # determine the burn-in tests that are going to be done
        self.do_tests = get_vars_from_arg(burn_in_test)
        self.burn_in_test = burn_in_test
        self.is_burned_in = False
        self.burn_in_iteration = NOT_BURNED_IN_ITER
        self.test_is_burned_in = {}  # burn in status per test
        self.test_burn_in_iteration = {}  # burn in iter per test
        self.test_aux_info = {}  # any additional information the test stores
        # Arguments specific to each test...
        # for nacl:
        self._nacls = int(kwargs.pop('nacls', 5))
        # for max_posterior and posterior_step
        self._ndim = int(kwargs.pop('ndim', len(sampler.variable_params)))
        # for min iterations
        self._min_iterations = int(kwargs.pop('min_iterations', 0))

    @abstractmethod
    def burn_in_index(self, filename):
        """The burn in index (retrieved from the iteration).

        This is an abstract method because how this is evaluated depends on
        if this is an ensemble MCMC or not.
        """
        pass

    def _getniters(self, filename):
        """Convenience function to get the number of iterations in the file.

        If `niterations` hasn't been written to the file yet, just returns 0.
        """
        with self.sampler.io(filename, 'r') as fp:
            try:
                niters = fp.niterations
                niters = 0
        return niters

    def _getnsamples(self, filename):
        """Convenience function to get the number of samples saved in the file.

        If no samples have been written to the file yet, just returns 0.
        """
            try:
                group = fp[fp.samples_group]
                # we'll just use the first parameter
                params = list(group.keys())
                nsamples = group[params[0]].shape[-1]
            except (KeyError, IndexError):
                nsamples = 0
        return nsamples

    def _index2iter(self, filename, index):
        """Converts the index in some samples at which burn in occurs to the
        iteration of the sampler that corresponds to.
        """
            thin_interval = fp.thinned_by
        return index * thin_interval

    def _iter2index(self, filename, iteration):
        """Converts an iteration to the index it corresponds to.
        """
        return iteration // thin_interval

    def _getlogposts(self, filename):
        """Convenience function for retrieving log posteriors.

        filename : str
            The file to read.

        Returns
        -------
        array
            The log posterior values. They are not flattened, so have dimension
            nwalkers x niterations.
        """
            samples = fp.read_raw_samples(
                ['loglikelihood', 'logprior'], thin_start=0, thin_interval=1,
                flatten=False)
            logposts = samples['loglikelihood'] + samples['logprior']
        return logposts

    def _getacls(self, filename, start_index):
        """Convenience function for calculating acls for the given filename.
        """
        return self.sampler.compute_acl(filename, start_index=start_index)

    def _getaux(self, test):
        """Convenience function for getting auxilary information.

        test : str
            The name of the test to retrieve auxilary information about.

        Returns
        -------
        dict
            The ``test_aux_info[test]`` dictionary. If a dictionary does
            not exist yet for the given test, an empty dictionary will be
            created and saved to ``test_aux_info[test]``.
        """
        try:
            aux = self.test_aux_info[test]
            aux = self.test_aux_info[test] = {}
        return aux

    def halfchain(self, filename):
        """Just uses half the chain as the burn-in iteration.
        """
        niters = self._getniters(filename)
        # this test cannot determine when something will burn in
        # only when it was not burned in in the past
        self.test_is_burned_in['halfchain'] = True
        self.test_burn_in_iteration['halfchain'] = niters//2

    def min_iterations(self, filename):
        """Just checks that the sampler has been run for the minimum number
        of iterations.
        """
        is_burned_in = self._min_iterations < niters
            burn_in_iter = self._min_iterations
        else:
            burn_in_iter = NOT_BURNED_IN_ITER
        self.test_is_burned_in['min_iterations'] = is_burned_in
        self.test_burn_in_iteration['min_iterations'] = burn_in_iter

    def max_posterior(self, filename):
        """Carries out the max posterior test and stores the results."""
        pass

    def posterior_step(self, filename):
        """Carries out the posterior step test and stores the results."""
        pass

    def nacl(self, filename):
        """Carries out the nacl test and stores the results."""
        pass

    def evaluate(self, filename):
        """Performs all tests and evaluates the results to determine if and
        when all tests pass.
        """
        pass

    def write(self, fp, path=None):
        """Writes burn-in info to an open HDF file.

        fp : pycbc.inference.io.base.BaseInferenceFile
            Open HDF file to write the data to. The HDF file should be an
            instance of a pycbc BaseInferenceFile.
        path : str, optional
            Path in the HDF file to write the data to. Default is (None) is
            to write to the path given by the file's ``sampler_group``
            attribute.
        """
        if path is None:
            path = fp.sampler_group
        fp.write_data('burn_in_test', self.burn_in_test, path)
        fp.write_data('is_burned_in', self.is_burned_in, path)
        fp.write_data('burn_in_iteration', self.burn_in_iteration, path)
        testgroup = 'burn_in_tests'
        # write individual test data
        for tst in self.do_tests:
            subpath = '/'.join([path, testgroup, tst])
            fp.write_data('is_burned_in', self.test_is_burned_in[tst], subpath)
            fp.write_data('burn_in_iteration',
                          self.test_burn_in_iteration[tst],
                          subpath)
            # write auxiliary info
            if tst in self.test_aux_info:
                for name, data in self.test_aux_info[tst].items():
                    fp.write_data(name, data, subpath)

    def _extra_tests_from_config(cp, section, tag):
        """For loading class-specific tests."""
        # pylint: disable=unused-argument
        return {}

    def from_config(cls, cp, sampler):
        """Loads burn in from section [sampler-burn_in]."""
        section = 'sampler'
        tag = 'burn_in'
        burn_in_test = cp.get_opt_tag(section, 'burn-in-test', tag)
        kwargs = {}
        if cp.has_option_tag(section, 'nacl', tag):
            kwargs['nacl'] = int(cp.get_opt_tag(section, 'nacl', tag))
        if cp.has_option_tag(section, 'ndim', tag):
            kwargs['ndim'] = int(
                cp.get_opt_tag(section, 'ndim', tag))
        if cp.has_option_tag(section, 'min-iterations', tag):
            kwargs['min_iterations'] = int(
                cp.get_opt_tag(section, 'min-iterations', tag))
        # load any class specific tests
        kwargs.update(cls._extra_tests_from_config(cp, section, tag))
        return cls(sampler, burn_in_test, **kwargs)

class MCMCBurnInTests(BaseBurnInTests):
    """Burn-in tests for collections of independent MCMC chains.

    This differs from EnsembleMCMCBurnInTests in that chains are treated as
    being independent of each other. The ``is_burned_in`` attribute will be
    True if `any` chain passes the burn in tests (whereas in MCMCBurnInTests,
    all chains must pass the burn in tests). In other words, independent
    samples can be collected even if all of the chains are not burned in.
    """
        super(MCMCBurnInTests, self).__init__(sampler, burn_in_test, **kwargs)
        try:
            nchains = sampler.nchains
            nchains = sampler.nwalkers
        self.nchains = nchains
        self.is_burned_in = numpy.zeros(self.nchains, dtype=bool)
        self.burn_in_iteration = numpy.repeat(NOT_BURNED_IN_ITER, self.nchains)

        """The burn in index (retrieved from the iteration)."""
        burn_in_index = self._iter2index(filename, self.burn_in_iteration)
        # don't set if it isn't burned in
        burn_in_index[~self.is_burned_in] = NOT_BURNED_IN_ITER
        return burn_in_index

        """Applies max posterior test."""
        logposts = self._getlogposts(filename)
        burn_in_idx, is_burned_in = max_posterior(logposts, self._ndim)
        # convert index to iterations
        burn_in_iter = self._index2iter(filename, burn_in_idx)
        burn_in_iter[~is_burned_in] = NOT_BURNED_IN_ITER
        # save
        test = 'max_posterior'
        self.test_is_burned_in[test] = is_burned_in
        self.test_burn_in_iteration[test] = burn_in_iter

        """Applies the posterior-step test."""
        burn_in_idx = numpy.array([posterior_step(logps, self._ndim)
                                   for logps in logposts])
        test = 'posterior_step'
        if test not in self.test_is_burned_in:
            self.test_is_burned_in[test] = numpy.ones(self.nchains, dtype=bool)
        self.test_burn_in_iteration[test] = self._index2iter(filename,
                                                             burn_in_idx)

        """Applies the :py:func:`nacl` test."""
        nsamples = self._getnsamples(filename)
        acls = self._getacls(filename, start_index=nsamples//2)
        is_burned_in = nacl(nsamples, acls, self._nacls)
        # stack the burn in results into an nparams x nchains array
        burn_in_per_chain = numpy.stack(list(is_burned_in.values())).all(
            axis=0)
        # store
        test = 'nacl'
        self.test_is_burned_in[test] = burn_in_per_chain
        try:
            burn_in_iter = self.test_burn_in_iteration[test]
            # hasn't been stored yet
            burn_in_iter = numpy.repeat(NOT_BURNED_IN_ITER, self.nchains)
            self.test_burn_in_iteration[test] = burn_in_iter
        burn_in_iter[burn_in_per_chain] = self._index2iter(filename,
                                                           nsamples//2)
        # add the status for each parameter as additional information
        self.test_aux_info[test] = is_burned_in

        """Runs all of the burn-in tests."""
        # evaluate all the tests
            logging.info("Evaluating %s burn-in test", tst)
            getattr(self, tst)(filename)
        # evaluate each chain at a time
        for ci in range(self.nchains):
            # some tests (like halfchain) just store a single bool for all
            # chains
            tibi = {t: r[ci] if isinstance(r, numpy.ndarray) else r
                    for t, r in self.test_is_burned_in.items()}
            tbi = {t: r[ci] if isinstance(r, numpy.ndarray) else r
                   for t, r in self.test_burn_in_iteration.items()}
            is_burned_in, burn_in_iter = evaluate_tests(self.burn_in_test,
                                                        tibi, tbi)
            self.is_burned_in[ci] = is_burned_in
            self.burn_in_iteration[ci] = burn_in_iter
        logging.info("Number of chains burned in: %i of %i",
                     self.is_burned_in.sum(), self.nchains)

        """
        super(MCMCBurnInTests, self).write(fp, path)
        # add number of chains burned in as additional metadata
        fp.write_data('nchains_burned_in', self.is_burned_in.sum(), path)

class MultiTemperedMCMCBurnInTests(MCMCBurnInTests):
    """Adds support for multiple temperatures to
    :py:class:`MCMCBurnInTests`.
    """

        This function is used by the ``n_acl`` burn-in test. That function
        expects the returned ``acls`` dict to just report a single ACL for
        each parameter. Since multi-tempered samplers return an array of ACLs
        for each parameter instead, this takes the max over the array before
        returning.

        Since we calculate the acls, this will also store it to the sampler.

            Name of the file to retrieve samples from.
        start_index : int
            Index to start calculating ACLs.

        Returns
        -------
        dict :
            Dictionary of parameter names -> array giving ACL for each chain.
        """
        acls = super(MultiTemperedMCMCBurnInTests, self)._getacls(
            filename, start_index)
        # acls will have shape ntemps x nchains, flatten to nchains
        return {param: vals.max(axis=0) for (param, vals) in acls.items()}

        This just gets the coldest temperature chain, and returns arrays with
        shape nwalkers x niterations, so the parent class can run the same
        ``posterior_step`` function.
        """
        return _multitemper_getlogposts(self.sampler, filename)

class EnsembleMCMCBurnInTests(BaseBurnInTests):
    """Provides methods for estimating burn-in of an ensemble MCMC."""

                       'posterior_step', 'nacl', 'ks_test',
                       )

        super(EnsembleMCMCBurnInTests, self).__init__(
            sampler, burn_in_test, **kwargs)
        # for kstest
        self._ksthreshold = float(kwargs.pop('ks_threshold', 0.9))

        if self.is_burned_in:
            index = self._iter2index(filename, self.burn_in_iteration)
        else:
            index = NOT_BURNED_IN_ITER
        return index

        """Applies max posterior test to self."""
        all_burned_in = is_burned_in.all()
        if all_burned_in:
            burn_in_iter = self._index2iter(filename, burn_in_idx.max())
        else:
        # store
        self.test_is_burned_in[test] = all_burned_in
        aux = self._getaux(test)
        # additional info
        aux['iteration_per_walker'] = self._index2iter(filename, burn_in_idx)
        aux['status_per_walker'] = is_burned_in

        burn_in_iters = self._index2iter(filename, burn_in_idx)
        self.test_is_burned_in[test] = True
        self.test_burn_in_iteration[test] = burn_in_iters.max()
        # store the iteration per walker as additional info
        aux['iteration_per_walker'] = burn_in_iters

        all_burned_in = all(is_burned_in.values())
            burn_in_iter = self._index2iter(filename, nsamples//2)
        else:
        # store
        # store the status per parameter as additional info
        aux['status_per_parameter'] = is_burned_in

    def ks_test(self, filename):
        """Applies ks burn-in test."""
            # get the samples from the mid point
            samples1 = fp.read_raw_samples(
                ['loglikelihood', 'logprior'], iteration=int(nsamples/2.))
            # get the last samples
            samples2 = fp.read_raw_samples(
                ['loglikelihood', 'logprior'], iteration=-1)
        # do the test
        # is_the_same is a dictionary of params --> bool indicating whether or
        # not the 1D marginal is the same at the half way point
        is_the_same = ks_test(samples1, samples2, threshold=self._ksthreshold)
        is_burned_in = all(is_the_same.values())
            burn_in_iter = self._index2iter(filename, int(nsamples//2))
        else:
        # store
        test = 'ks_test'
        # store the test per parameter as additional info
        aux['status_per_parameter'] = is_the_same

        is_burned_in, burn_in_iter = evaluate_tests(
            self.burn_in_test, self.test_is_burned_in,
            self.test_burn_in_iteration)
        self.is_burned_in = is_burned_in
        self.burn_in_iteration = burn_in_iter
        logging.info("Is burned in: %r", self.is_burned_in)
            logging.info("Burn-in iteration: %i",
                         int(self.burn_in_iteration))

        """Loads the ks test settings from the config file."""
        kwargs = {}
        if cp.has_option_tag(section, 'ks-threshold', tag):
            kwargs['ks_threshold'] = float(
                cp.get_opt_tag(section, 'ks-threshold', tag))

class EnsembleMultiTemperedMCMCBurnInTests(EnsembleMCMCBurnInTests):
    :py:class:`EnsembleMCMCBurnInTests`.
    """

        """
        acls = super(EnsembleMultiTemperedMCMCBurnInTests, self)._getacls(
        # return the max for each parameter
        return {param: vals.max() for (param, vals) in acls.items()}

        """

def _multitemper_getlogposts(sampler, filename):
    """Retrieve log posteriors for multi tempered samplers."""
    with sampler.io(filename, 'r') as fp:
        samples = fp.read_raw_samples(
            ['loglikelihood', 'logprior'], thin_start=0, thin_interval=1,
            temps=0, flatten=False)
        # reshape to drop the first dimension
        for (stat, arr) in samples.items():
            _, nwalkers, niterations = arr.shape
            samples[stat] = arr.reshape((nwalkers, niterations))
        logposts = samples['loglikelihood'] + samples['logprior']
    return logposts

""" The module contains functions for calculating the
Kullback-Leibler divergence.
"""

from scipy import stats

def check_hist_params(samples, hist_min, hist_max, hist_bins):
    """ Checks that the bound values given for the histogram are consistent,
    returning the range if they are or raising an error if they are not.
    Also checks that if hist_bins is a str, it corresponds to a method
    available in numpy.histogram

    samples : numpy.array
        Set of samples to get the min/max if only one of the bounds is given.
    hist_min : numpy.float64
        Minimum value for the histogram.
    hist_max : numpy.float64
        Maximum value for the histogram.
    hist_bins: int or str
        If int, number of equal-width bins to use in numpy.histogram. If str,
        it should be one of the methods to calculate the optimal bin width
        available in numpy.histogram: ['auto', 'fd', 'doane', 'scott', 'stone',
        'rice', 'sturges', 'sqrt']. Default is 'fd' (Freedman Diaconis
        Estimator). This option will be ignored if `kde=True`.

    Returns
    -------
    hist_range : tuple or None
        The bounds (hist_min, hist_max) or None.
    hist_bins : int or str
        Number of bins or method for optimal width bin calculation.
    """

    hist_methods = ['auto', 'fd', 'doane', 'scott', 'stone', 'rice',
                    'sturges', 'sqrt']
    if not hist_bins:
        hist_bins = 'fd'
    elif isinstance(hist_bins, str) and hist_bins not in hist_methods:
        raise ValueError('Method for calculating bins width must be one of'
                         ' {}'.format(hist_methods))

    # No bounds given, return None
    if not hist_min and not hist_max:
        return None, hist_bins

    # One of the bounds is missing
    if hist_min and not hist_max:
        hist_max = samples.max()
    elif hist_max and not hist_min:
        hist_min = samples.min()
    # Both bounds given
    elif hist_min and hist_max and hist_min >= hist_max:
        raise ValueError('hist_min must be lower than hist_max.')

    hist_range = (hist_min, hist_max)

    return hist_range, hist_bins

def compute_pdf(samples, method, bins, hist_min, hist_max):
    """ Computes the probability density function for a set of samples.

        Set of samples to calculate the pdf.
    method : str
        Method to calculate the pdf. Options are 'kde' for the Kernel Density
        Estimator, and 'hist' to use numpy.histogram
    bins : str or int, optional
        This option will be ignored if method is `kde`.
        If int, number of equal-width bins to use when calculating probability
        density function from a set of samples of the distribution. If str, it
        should be one of the methods to calculate the optimal bin width
        Estimator).
    hist_min : numpy.float64, optional
        Minimum of the distributions' values to use. This will be ignored if
        `kde=True`.
    hist_max : numpy.float64, optional
        Maximum of the distributions' values to use. This will be ignored if

    Returns
    -------
    pdf : numpy.array
        Discrete probability distribution calculated from samples.
    """

    if method == 'kde':
        samples_kde = stats.gaussian_kde(samples)
        npts = 10000 if len(samples) <= 10000 else len(samples)
        draw = samples_kde.resample(npts)
        pdf = samples_kde.evaluate(draw)
    elif method == 'hist':
        hist_range, hist_bins = check_hist_params(samples, hist_min,
                                                  hist_max, bins)
        pdf, _ = numpy.histogram(samples, bins=hist_bins,
                                 range=hist_range, density=True)
    else:
        raise ValueError('Method not recognized.')

    return pdf

def entropy(pdf1, base=numpy.e):
    """ Computes the information entropy for a single parameter
    from one probability density function.

    pdf1 : numpy.array
        Probability density function.
    base : {numpy.e, numpy.float64}, optional
        The logarithmic base to use (choose base 2 for information measured
        in bits, default is nats).

    Returns
    -------
    numpy.float64
        The information entropy value.
    """

    return stats.entropy(pdf1, base=base)

def kl(samples1, samples2, pdf1=False, pdf2=False, kde=False,
       bins=None, hist_min=None, hist_max=None, base=numpy.e):
    """ Computes the Kullback-Leibler divergence for a single parameter
    from two distributions.

    samples1 : numpy.array
        Samples or probability density function (for the latter must also set
        `pdf1=True`).
    samples2 : numpy.array
        `pdf2=True`).
    pdf1 : bool
        Set to `True` if `samples1` is a probability density funtion already.
    pdf2 : bool
        Set to `True` if `samples2` is a probability density funtion already.
    kde : bool
        Set to `True` if at least one of `pdf1` or `pdf2` is `False` to
        estimate the probability density function using kernel density
        estimation (KDE).
    bins : int or str, optional
    base : numpy.float64

    Returns
    -------
        The Kullback-Leibler divergence value.
    """
    if pdf1 and pdf2 and kde:
        raise ValueError('KDE can only be used when at least one of pdf1 or '
                         'pdf2 is False.')

    sample_groups = {'P': (samples1, pdf1), 'Q': (samples2, pdf2)}
    pdfs = {}
    for n in sample_groups:
        samples, pdf = sample_groups[n]
        if pdf:
            pdfs[n] = samples
        else:
            method = 'kde' if kde else 'hist'
            pdfs[n] = compute_pdf(samples, method, bins, hist_min, hist_max)

    return stats.entropy(pdfs['P'], qk=pdfs['Q'], base=base)

def js(samples1, samples2, kde=False, bins=None, hist_min=None, hist_max=None,
       base=numpy.e):
    """ Computes the Jensen-Shannon divergence for a single parameter

        Samples.
        Samples.
    kde : bool
        Set to `True` to estimate the probability density function using
        kernel density estimation (KDE).

    Returns
    -------
        The Jensen-Shannon divergence value.
    """

    sample_groups = {'P': samples1, 'Q': samples2}
    pdfs = {}
        samples = sample_groups[n]
        method = 'kde' if kde else 'hist'
        pdfs[n] = compute_pdf(samples, method, bins, hist_min, hist_max)

    pdfs['M'] = (1./2) * (pdfs['P'] + pdfs['Q'])

    js_div = 0
    for pdf in (pdfs['P'], pdfs['Q']):
        js_div += (1./2) * kl(pdf, pdfs['M'], pdf1=True, pdf2=True, base=base)

    return js_div

# Copyright (C) 2019 Steven Reyes
#
#
"""
This modules provides functions for estimating the marginal
likelihood or evidence of a model.
"""
from scipy import integrate

def arithmetic_mean_estimator(log_likelihood):
    """Returns the log evidence via the prior arithmetic mean estimator (AME).

    The logarithm form of AME is used. This is the most basic
    evidence estimator, and often requires O(billions) of samples
    from the prior.

    log_likelihood : 1d array of floats
        The log likelihood of the data sampled from the prior
        distribution.

    Returns
    -------
    float :
        Estimation of the log of the evidence.
    """
    num_samples = len(log_likelihood)
    logl_max = numpy.max(log_likelihood)

    log_evidence = 0.
    for i, _ in enumerate(log_likelihood):
        log_evidence += numpy.exp(log_likelihood[i] - logl_max)

    log_evidence = numpy.log(log_evidence)
    log_evidence += logl_max - numpy.log(num_samples)

    return log_evidence

def harmonic_mean_estimator(log_likelihood):
    """Returns the log evidence via posterior harmonic mean estimator (HME).

    The logarithm form of HME is used. This method is not
    recommended for general use. It is very slow to converge,
    formally, has infinite variance, and very error prone.

    Not recommended for general use.

        The log likelihood of the data sampled from the posterior

    Returns
    -------
    float :
    """
    logl_max = numpy.max(-1.0*log_likelihood)

        log_evidence += numpy.exp(-1.0*log_likelihood[i] + logl_max)

    log_evidence = -1.0*numpy.log(log_evidence)
    log_evidence += logl_max
    log_evidence += numpy.log(num_samples)

def thermodynamic_integration(log_likelihood, betas,
                              method="simpsons"):
    """Returns the log evidence of the model via thermodynamic integration.
    Also returns an estimated standard deviation for the log evidence.

    Current options are integration through the trapezoid rule, a
    first-order corrected trapezoid rule, and Simpson's rule.

    log_likelihood : 3d array of shape (betas, walker, iteration)
        The log likelihood for each temperature separated by
        temperature, walker, and iteration.

    betas : 1d array
        The inverse temperatures used in the MCMC.

    method : {"trapzoid", "trapezoid_corrected", "simpsons"},
             optional.
        The numerical integration method to use for the
        thermodynamic integration. Choices include: "trapezoid",
        "trapezoid_corrected", "simpsons", for the trapezoid rule,
        the first-order correction to the trapezoid rule, and
        Simpson's rule. [Default = "simpsons"]

    Returns
    -------
    log_evidence : float

    mcmc_std : float
        The standard deviation of the log evidence estimate from
        Monte-Carlo spread.
    """
    # Check if the method of integration is in the list of choices
    method_list = ["trapezoid", "trapezoid_corrected", "simpsons"]

    if method not in method_list:
        raise ValueError("Method %s not supported. Expected %s"
                         % (method, method_list))
    # Read in the data and ensure ordering of data.
    # Ascending order sort
    order = numpy.argsort(betas)
    betas = betas[order]
    log_likelihood = log_likelihood[order]

    # Assume log likelihood is given in shape of beta, walker,
    # and iteration.
    log_likelihood = numpy.reshape(log_likelihood,
                                   (len(betas),
                                    len(log_likelihood[0].flatten())))

    average_logl = numpy.average(log_likelihood, axis=1)

    if method in ("trapezoid", "trapezoid_corrected"):
        log_evidence = numpy.trapz(average_logl, betas)

    if method == "trapezoid_corrected":
        # var_correction holds the derivative correction terms
        # See Friel et al. 2014 for expression and derivation.
        # https://link.springer.com/article/10.1007/s11222-013-9397-1
        var_correction = 0
        for i in range(len(betas) - 1):
            delta_beta = betas[i+1] - betas[i]
            pre_fac_var = (1. / 12.) * (delta_beta ** 2.0)
            var_diff = numpy.var(log_likelihood[i+1])
            var_diff -= numpy.var(log_likelihood[i])
            var_correction -= pre_fac_var * var_diff

        # Add the derivative correction term back to the log_evidence
        # from the first if statement.
        log_evidence += var_correction

    elif method == "simpsons":
        # beta -> 0 tends to contribute the least to the integral
        # so we can sacrifice precision there, rather than near
        # beta -> 1. Option even="last" puts trapezoid rule at
        # first few points.
        log_evidence = integrate.simps(average_logl, betas,
                                       even="last")

    # Estimate the Monte Carlo variance of the evidence calculation
    # See (Evans, Annis, 2019.)
    # https://www.sciencedirect.com/science/article/pii/S0022249617302651
    ti_vec = numpy.zeros(len(log_likelihood[0]))

    # Get log likelihood chains by sample and not by temperature.
    logl_per_samp = []
    for i, _ in enumerate(log_likelihood[0]):
        logl_per_samp.append([log_likelihood[x][i] for x in range(len(betas))])

        for i, _ in enumerate(log_likelihood[0]):
            ti_vec[i] = numpy.trapz(logl_per_samp[i], betas)

            ti_vec[i] = integrate.simps(logl_per_samp[i], betas,
                                        even="last")

    # Standard error is sample std / sqrt(number of samples)
    mcmc_std = numpy.std(ti_vec) / numpy.sqrt(float(len(log_likelihood[0])))

    return log_evidence, mcmc_std

def stepping_stone_algorithm(log_likelihood, betas):
    """Returns the log evidence of the model via stepping stone algorithm.

    betas          : 1d array
                     The inverse temperatures used in the MCMC.

    Returns
    -------
    """
    # Reverse order sort
    order = numpy.argsort(betas)[::-1]

    # Assume log likelihood is given in shape of beta,
    # walker, iteration.

    log_rk_pb = numpy.zeros(len(betas) - 1)
    for i in range(len(betas) - 1):
        delta_beta = betas[i] - betas[i+1]
        # Max log likelihood for beta [i+1]
        max_logl_pb = numpy.max(log_likelihood[i+1])
        val_1 = delta_beta * max_logl_pb
        val_2 = delta_beta * (log_likelihood[i+1] - max_logl_pb)
        val_2 = numpy.log(numpy.average(numpy.exp(val_2)))
        log_rk_pb[i] = val_1 + val_2

    log_rk = numpy.sum(log_rk_pb)
    log_evidence = log_rk

    # Calculate the Monte Carlo variation
    mcmc_std = 0
        pre_fact = (delta_beta * log_likelihood[i+1]) - log_rk_pb[i]
        pre_fact = numpy.exp(pre_fact) - 1.0
        val = numpy.sum(pre_fact ** 2)

        mcmc_std += val

    mcmc_std /= float(len(log_likelihood[0])) ** 2.0
    mcmc_std = numpy.sqrt(mcmc_std)

#
#
""" This modules provides functions for evaluating the Gelman-Rubin convergence
diagnostic statistic.
"""

def walk(chains, start, end, step):
    """ Calculates Gelman-Rubin conervergence statistic along chains of data.
    This function will advance along the chains and calculate the
    statistic for each step.

    chains : iterable
        An iterable of numpy.array instances that contain the samples
        for each chain. Each chain has shape (nparameters, niterations).
    start : float
        Start index of blocks to calculate all statistics.
    end : float
        Last index of blocks to calculate statistics.
    step : float
        Step size to take for next block.

    Returns
    -------
    starts : numpy.array
        1-D array of start indexes of calculations.
    ends : numpy.array
        1-D array of end indexes of caluclations.
    stats : numpy.array
        Array with convergence statistic. It has
        shape (nparameters, ncalculations).
    """

    # get number of chains, parameters, and iterations
    chains = numpy.array(chains)
    _, nparameters, _ = chains.shape

    # get end index of blocks
    ends = numpy.arange(start, end, step)
    stats = numpy.zeros((nparameters, len(ends)))

    # get start index of blocks
    starts = numpy.array(len(ends) * [start])

    # loop over end indexes and calculate statistic
    for i, e in enumerate(ends):
        tmp = chains[:, :, 0:e]
        stats[:, i] = gelman_rubin(tmp)

    return starts, ends, stats

def gelman_rubin(chains, auto_burn_in=True):
    """ Calculates the univariate Gelman-Rubin convergence statistic
    which compares the evolution of multiple chains in a Markov-Chain Monte
    Carlo process and computes their difference to determine their convergence.
    The between-chain and within-chain variances are computed for each sampling
    parameter, and a weighted combination of the two is used to determine the
    convergence. As the chains converge, the point scale reduction factor
    should go to 1.

    auto_burn_in : bool
        If True, then only use later half of samples provided.

    Returns
    -------
    psrf : numpy.array
        A numpy.array of shape (nparameters) that has the point estimates of
        the potential scale reduction factor.
    """

    # remove first half of samples
    # this will have shape (nchains, nparameters, niterations)
    if auto_burn_in:
        _, _, niterations = numpy.array(chains).shape
        chains = numpy.array([chain[:, niterations // 2 + 1:]
                              for chain in chains])

    nchains, nparameters, niterations = chains.shape

    # calculate the covariance matrix for each chain
    # this will have shape (nchains, nparameters, nparameters)
    chains_covs = numpy.array([numpy.cov(chain) for chain in chains])
    if nparameters == 1:
        chains_covs = chains_covs.reshape((nchains, 1, 1))

    # calculate W the within-chain variance
    # this will have shape (nparameters, nparameters)
    w = numpy.zeros(chains_covs[0].shape)
    for i, row in enumerate(chains_covs[0]):
        for j, _ in enumerate(row):
            w[i, j] = numpy.mean(chains_covs[:, i, j])
        w = w.reshape((1, 1))

    # calculate B the between-chain variance
    means = numpy.zeros((nparameters, nchains))
    for i, chain in enumerate(chains):
        means[:, i] = numpy.mean(chain, axis=1).transpose()
    b = niterations * numpy.cov(means)
        b = b.reshape((1, 1))

    # get diagonal elements of W and B
    # these will have shape (nparameters)
    w_diag = numpy.diag(w)
    b_diag = numpy.diag(b)

    # get variance for each chain
    # this will have shape (nparameters, nchains)
    var = numpy.zeros((nparameters, nchains))
    for i, chain_cov in enumerate(chains_covs):
        var[:, i] = numpy.diag(chain_cov)

    # get mean of means
    # this will have shape (nparameters)
    mu_hat = numpy.mean(means, axis=1)

    # get variance of variances
    s = numpy.var(var, axis=1)

    # get V the combined variance of all chains
    v = ((niterations - 1.) * w_diag / niterations +
         (1. + 1. / nchains) * b_diag / niterations)

    # get factors in variance of V calculation
    k = 2 * b_diag**2 / (nchains - 1)
    mid_term = numpy.cov(
        var, means**2)[nparameters:2*nparameters, 0:nparameters].T
    end_term = numpy.cov(
        var, means)[nparameters:2*nparameters, 0:nparameters].T
    wb = niterations / nchains * numpy.diag(mid_term - 2 * mu_hat * end_term)

    # get variance of V
    var_v = (
        (niterations - 1.) ** 2 * s +
        (1. + 1. / nchains) ** 2 * k +
        2. * (niterations - 1.) * (1. + 1. / nchains) * wb
    ) / niterations**2

    # get degrees of freedom
    dof = (2. * v**2) / var_v

    # more degrees of freedom factors
    df_adj = (dof + 3.) / (dof + 1.)

    # estimate R
    r2_fixed = (niterations - 1.) / niterations
    r2_random = (1. + 1. / nchains) * (1. / niterations) * (b_diag / w_diag)
    r2_estimate = r2_fixed + r2_random

    # calculate PSRF the potential scale reduction factor
    psrf = numpy.sqrt(r2_estimate * df_adj)

    return psrf

#
#
#
""" Functions for computing the Geweke convergence statistic.
"""

def geweke(x, seg_length, seg_stride, end_idx, ref_start,
           ref_end=None, seg_start=0):
    """ Calculates Geweke conervergence statistic for a chain of data.
    This function will advance along the chain and calculate the

    x : numpy.array
        A one-dimensional array of data.
    seg_length : int
        Number of samples to use for each Geweke calculation.
    seg_stride : int
        Number of samples to advance before next Geweke calculation.
    end_idx : int
        Index of last start.
    ref_start : int
        Index of beginning of end reference segment.
    ref_end : int
        Index of end of end reference segment. Default is None which
        will go to the end of the data array.
    seg_start : int
        What index to start computing the statistic. Default is 0 which
        will go to the beginning of the data array.

    Returns
    -------
        The start index of the first segment in the chain.
        The end index of the first segment in the chain.
        The Geweke convergence diagnostic statistic for the segment.
    """

    # lists to hold statistic and end index
    stats = []
    ends = []

    # get the beginning of all segments
    starts = numpy.arange(seg_start, end_idx, seg_stride)

    # get second segment of data at the end to compare
    x_end = x[ref_start:ref_end]

    # loop over all segments
    for start in starts:

        # find the end of the first segment
        x_start_end = int(start + seg_length)

        # get first segment
        x_start = x[start:x_start_end]

        # compute statistic
        stats.append((x_start.mean() - x_end.mean()) / numpy.sqrt(
            x_start.var() + x_end.var()))

        # store end of first segment
        ends.append(x_start_end)

    return numpy.array(starts), numpy.array(ends), numpy.array(stats)

# Copyright (C) 2016 Collin Capano, Duncan Brown
#
#
#

"""This module contains standard options used for inference-related programs.
"""

import argparse

from pycbc import waveform
from pycbc import distributions

# -----------------------------------------------------------------------------
#
#                Utilities for plotting results
#

class ParseLabelArg(argparse.Action):
    """Argparse action that will parse arguments that can accept labels.

    This assumes that the values set on the command line for its assigned
    argument are strings formatted like ``PARAM[:LABEL]``. When the arguments
    are parsed, the ``LABEL`` bit is stripped off and added to a dictionary
    mapping ``PARAM -> LABEL``. This dictionary is stored to the parsed
    namespace called ``{dest}_labels``, where ``{dest}`` is the argument's
    ``dest`` setting (by default, this is the same as the option string).
    Likewise, the argument's ``dest`` in the parsed namespace is updated so
    that it is just ``PARAM``.

    If no ``LABEL`` is provided, then ``PARAM`` will be used for ``LABEL``.

    This action can work on arguments that have ``nargs != 0`` and ``type`` set
    to ``str``.
    """
    def __init__(self, type=str, nargs=None,
                 **kwargs):  # pylint: disable=redefined-builtin
        # check that type is string
        if type != str:
            raise ValueError("the type for this action must be a string")
        if nargs == 0:
            raise ValueError("nargs must not be 0 for this action")
        super(ParseLabelArg, self).__init__(type=type, nargs=nargs,
                                            **kwargs)

    def __call__(self, parser, namespace, values, option_string=None):
        singlearg = isinstance(values, string_types)
        if singlearg:
            values = [values]
        params = []
        labels = {}
        for param in values:
            psplit = param.split(':')
            if len(psplit) == 2:
                param, label = psplit
            else:
                label = param
            labels[param] = label
            params.append(param)
        # update the namespace
            params = params[0]
        setattr(namespace, self.dest, params)
        setattr(namespace, '{}_labels'.format(self.dest), labels)

class ParseParametersArg(ParseLabelArg):
    """Argparse action that will parse parameters and labels from an opton.

    Does the same as ``ParseLabelArg``, with the additional functionality that
    if ``LABEL`` is a known parameter in ``pycbc.waveform.parameters``, then
    the label attribute there will be used in the labels dictionary.
    Otherwise, ``LABEL`` will be used.

    Examples
    --------
    Create a parser and add two arguments that use this action (note that the
    first argument accepts multiple inputs while the second only accepts a
    single input):

    >>> import argparse
    >>> parser = argparse.ArgumentParser()
    >>> parser.add_argument('--parameters', type=str, nargs="+",
                            action=ParseParametersArg)
    >>> parser.add_argument('--z-arg', type=str, action=ParseParametersArg)

    Parse a command line that uses these options:

    >>> import shlex
    >>> cli = "--parameters 'mass1+mass2:mtotal' ra ni --z-arg foo:bar"
    >>> opts = parser.parse_args(shlex.split(cli))
    >>> opts.parameters
    ['mass1+mass2', 'ra', 'ni']
    >>> opts.parameters_labels
    {'mass1+mass2': '$M~(\\mathrm{M}_\\odot)$', 'ni': 'ni', 'ra': '$\\alpha$'}
    >>> opts.z_arg
    'foo'
    >>> opts.z_arg_labels
    {'foo': 'bar'}

    In the above, the first argument to ``--parameters`` was ``mtotal``. Since
    this is a recognized parameter in ``pycbc.waveform.parameters``, the label
    dictionary contains the latex string associated with the ``mtotal``
    parameter. A label was not provided for the second argument, and so ``ra``
    was used. Since ``ra`` is also a recognized parameter, its associated latex
    string was used in the labels dictionary. Since ``ni`` and ``bar`` (the
    label for ``z-arg``) are not recognized parameters, they were just used
    as-is in the labels dictionaries.
    """
        super(ParseParametersArg, self).__call__(parser, namespace, values,
                                                 option_string=option_string)
        # try to replace the labels with a label from waveform.parameters
        labels = getattr(namespace, '{}_labels'.format(self.dest))
        for param, label in labels.items():
            try:
                label = getattr(waveform.parameters, label).label
                labels[param] = label
            except AttributeError:
                pass

def add_injsamples_map_opt(parser):
    """Adds option to parser to specify a mapping between injection parameters
    an sample parameters.
    """
    parser.add_argument('--injection-samples-map', nargs='+',
                        metavar='INJECTION_PARAM:SAMPLES_PARAM',
                        help='Rename/apply functions to the injection '
                             'parameters and name them the same as one of the '
                             'parameters in samples. This can be used if the '
                             'injection parameters are not the same as the '
                             'samples parameters. INJECTION_PARAM may be a '
                             'function of the injection parameters; '
                             'SAMPLES_PARAM must a name of one of the '
                             'parameters in the samples group.')

def add_plot_posterior_option_group(parser):
    """Adds the options needed to configure plots of posterior results.

        ArgumentParser instance.
    """
    pgroup = parser.add_argument_group("Options for what plots to create and "
                                       "their formats.")
    pgroup.add_argument('--plot-marginal', action='store_true', default=False,
                        help="Plot 1D marginalized distributions on the "
                             "diagonal axes.")
    pgroup.add_argument('--marginal-percentiles', nargs='+', default=None,
                        type=float,
                        help="Percentiles to draw lines at on the 1D "
                             "histograms.")
    pgroup.add_argument("--plot-scatter", action='store_true', default=False,
                        help="Plot each sample point as a scatter plot.")
    pgroup.add_argument("--plot-density", action="store_true", default=False,
                        help="Plot the posterior density as a color map.")
    pgroup.add_argument("--plot-contours", action="store_true", default=False,
                        help="Draw contours showing the 50th and 90th "
                             "percentile confidence regions.")
    pgroup.add_argument('--contour-percentiles', nargs='+', default=None,
                        help="Percentiles to draw contours if different "
                             "than 50th and 90th.")
    # add mins, maxs options
    pgroup.add_argument('--mins', nargs='+', metavar='PARAM:VAL', default=[],
                        help="Specify minimum parameter values to plot. This "
                             "should be done by specifying the parameter name "
                             "followed by the value. Parameter names must be "
                             "the same as the PARAM argument in --parameters "
                             "(or, if no parameters are provided, the same as "
                             "the parameter name specified in the variable "
                             "args in the input file. If none provided, "
                             "the smallest parameter value in the posterior "
                             "will be used.")
    pgroup.add_argument('--maxs', nargs='+', metavar='PARAM:VAL', default=[],
                        help="Same as mins, but for the maximum values to "
                             "plot.")
    # add expected parameters options
    pgroup.add_argument('--expected-parameters', nargs='+',
                        metavar='PARAM:VAL',
                        default=[],
                        help="Specify expected parameter values to plot. If "
                             "provided, a cross will be plotted in each axis "
                             "that an expected parameter is provided. "
                             "Parameter names must be "
                             "args in the input file.")
    pgroup.add_argument('--expected-parameters-color', default='r',
                        help="What to color the expected-parameters cross. "
                             "Default is red.")
    pgroup.add_argument('--plot-injection-parameters', action='store_true',
                        default=False,
                        help="Get the expected parameters from the injection "
                             "in the input file. There must be only a single "
                             "injection in the file to work. Any values "
                             "specified by expected-parameters will override "
                             "the values obtained for the injection.")
    pgroup.add_argument('--pick-injection-by-time', action='store_true',
                        help="In the case of multiple injections, pick one"
                             " for plotting based on its proximity in time.")
    add_injsamples_map_opt(pgroup)
    return pgroup

def plot_ranges_from_cli(opts):
    """Parses the mins and maxs arguments from the `plot_posterior` option
    group.

    opts : ArgumentParser
        The parsed arguments from the command line.

    Returns
    -------
    mins : dict
        Dictionary of parameter name -> specified mins. Only parameters that
        were specified in the --mins option will be included; if no parameters
        were provided, will return an empty dictionary.
    maxs : dict
        Dictionary of parameter name -> specified maxs. Only parameters that
    """
    mins = {}
    for x in opts.mins:
        x = x.split(':')
        if len(x) != 2:
            raise ValueError("option --mins not specified correctly; see help")
        mins[x[0]] = float(x[1])
    maxs = {}
    for x in opts.maxs:
            raise ValueError("option --maxs not specified correctly; see help")
        maxs[x[0]] = float(x[1])
    return mins, maxs

def expected_parameters_from_cli(opts):
    """Parses the --expected-parameters arguments from the `plot_posterior`
    option group.

    Returns
    -------
    dict
        Dictionary of parameter name -> expected value. Only parameters that
        were specified in the --expected-parameters option will be included; if
        no parameters were provided, will return an empty dictionary.
    """
    expected = {}
    for x in opts.expected_parameters:
            raise ValueError("option --expected-paramters not specified "
                             "correctly; see help")
        expected[x[0]] = float(x[1])
    return expected

def add_scatter_option_group(parser):
    """Adds the options needed to configure scatter plots.

    """
    scatter_group = parser.add_argument_group("Options for configuring the "
                                              "scatter plot.")

    scatter_group.add_argument(
        '--z-arg', type=str, default=None, action=ParseParametersArg,
        help='What to color the scatter points by. Syntax is the same as the '
             'parameters option.')
        "--vmin", type=float, help="Minimum value for the colorbar.")
        "--vmax", type=float, help="Maximum value for the colorbar.")
        "--scatter-cmap", type=str, default='plasma',
        help="Specify the colormap to use for points. Default is plasma.")

    return scatter_group

def add_density_option_group(parser):
    """Adds the options needed to configure contours and density colour map.

    """
    density_group = parser.add_argument_group("Options for configuring the "
                                              "contours and density color map")

    density_group.add_argument(
        "--density-cmap", type=str, default='viridis',
        help="Specify the colormap to use for the density. "
             "Default is viridis.")
        "--contour-color", type=str, default=None,
        help="Specify the color to use for the contour lines. Default is "
             "white for density plots and black for scatter plots.")
        '--use-kombine-kde', default=False, action="store_true",
        help="Use kombine's KDE for determining contours. "
             "Default is to use scipy's gaussian_kde.")

    return density_group

def prior_from_config(cp, prior_section='prior'):
    """Loads a prior distribution from the given config file.

        The config file to read.
    sections : list of str, optional
        The sections to retrieve the prior from. If ``None`` (the default),
        will look in sections starting with 'prior'.

    Returns
    -------
    distributions.JointDistribution
        The prior distribution.
    """
    # Read variable and static parameters from the config file
    variable_params, _ = distributions.read_params_from_config(
        cp, prior_section=prior_section, vargs_section='variable_params',
        sargs_section='static_params')
    # Read constraints to apply to priors from the config file
    constraints = distributions.read_constraints_from_config(cp)
    # Get PyCBC distribution instances for each variable parameter in the
    # config file
    dists = distributions.read_distributions_from_config(cp, prior_section)
    # construct class that will return draws from the prior
    return distributions.JointDistribution(variable_params, *dists,
                                           **{"constraints": constraints})

# Copyright (C) 2016 Christopher M. Biwer, Collin Capano
#
#

#
#
#                                   Preamble
#
#
"""This modules defines functions for reading and writing samples that the
inference samplers generate.
"""

import sys
# python 2.7 needs to use StingIO from the StringIO module; this was
# deprecated in python 3
if sys.version_info.major == 2:
    from StringIO import StringIO
else:
    from io import StringIO

from abc import (ABCMeta, abstractmethod)
from six import (add_metaclass, string_types)

from pycbc.io import FieldArray
from pycbc.inject import InjectionSet
from pycbc.io import (dump_state, load_state)
from pycbc.workflow import WorkflowConfigParser
from pycbc.types import FrequencySeries

def format_attr(val):
    """Formats an attr so that it can be read in either python 2 or 3.

    In python 2, strings that are saved as an attribute in an hdf file default
    to unicode. Since unicode was removed in python 3, if you load that file
    in a python 3 environment, the strings will be read as bytes instead, which
    causes a number of issues. This attempts to fix that. If the value is
    a bytes string, then it will be decoded into a string. If the value is
    a numpy array of byte strings, it will convert the array to a list of
    strings.

    val : obj
        The value to format. This will try to apply decoding to the value

    Returns
    -------
    obj
        If ``val`` was a byte string, the value as a ``str``. If the value
        was a numpy array of ``bytes_``, the value as a list of ``str``.
        Otherwise, just returns the value.
    """
    try:
        val = str(val.decode())
        pass
    if isinstance(val, numpy.ndarray) and val.dtype.type == numpy.bytes_:
        val = val.astype(numpy.unicode_).tolist()
    return val

class BaseInferenceFile(h5py.File):
    """Base class for all inference hdf files.

    This is a subclass of the h5py.File object. It adds functions for
    handling reading and writing the samples from the samplers.

    path : str
        The path to the HDF file.
    mode : {None, str}
        The mode to open the file, eg. "w" for write and "r" for read.
    """

    name = None
    samples_group = 'samples'
    sampler_group = 'sampler_info'
    data_group = 'data'
    injections_group = 'injections'
    config_group = 'config_file'

    def __init__(self, path, mode=None, **kwargs):
        super(BaseInferenceFile, self).__init__(path, mode, **kwargs)
        # check that file type matches self
        try:
            filetype = self.attrs['filetype']
            if mode == 'w':
                # first time creating the file, add this class's name
                filetype = self.name
                self.attrs['filetype'] = filetype
            else:
                filetype = None
        try:
            filetype = str(filetype.decode())
            pass
        if filetype != self.name:
            raise ValueError("This file has filetype {}, whereas this class "
                             "is named {}. This indicates that the file was "
                             "not written by this class, and so cannot be "
                             "read by this class.".format(filetype, self.name))

    def __getattr__(self, attr):
        """Things stored in ``.attrs`` are promoted to instance attributes.

        Note that properties will be called before this, so if there are any
        properties that share the same name as something in ``.attrs``, that
        property will get returned.
        """
        return self.attrs[attr]

    def write_samples(self, samples, **kwargs):
        """This should write all of the provided samples.

        This function should be used to write both samples and model stats.

        samples : dict
            Samples should be provided as a dictionary of numpy arrays.
            Any other keyword args the sampler needs to write data.
        """
        pass

    def parse_parameters(self, parameters, array_class=None):
        """Parses a parameters arg to figure out what fields need to be loaded.

        parameters : (list of) strings
            The parameter(s) to retrieve. A parameter can be the name of any
            field in ``samples_group``, a virtual field or method of
            ``FieldArray`` (as long as the file contains the necessary fields
            to derive the virtual field or method), and/or a function of
            these.
        array_class : array class, optional
            The type of array to use to parse the parameters. The class must
            have a ``parse_parameters`` method. Default is to use a
            ``FieldArray``.

        Returns
        -------
        list :
            A list of strings giving the fields to load from the file.
        """
        # get the type of array class to use
        if array_class is None:
            array_class = FieldArray
        # get the names of fields needed for the given parameters
        possible_fields = self[self.samples_group].keys()
        return array_class.parse_parameters(parameters, possible_fields)

    def read_samples(self, parameters, array_class=None, **kwargs):
        """Reads samples for the given parameter(s).

        The ``parameters`` can be the name of any dataset in ``samples_group``,
        a virtual field or method of ``FieldArray`` (as long as the file
        contains the necessary fields to derive the virtual field or method),
        and/or any numpy function of these.

        The ``parameters`` are parsed to figure out what datasets are needed.
        Only those datasets will be loaded, and will be the base-level fields
        of the returned ``FieldArray``.

        The ``static_params`` are also added as attributes of the returned
        ``FieldArray``.

            The parameter(s) to retrieve.
        array_class : FieldArray-like class, optional
            The type of array to return. The class must have ``from_kwargs``
            and ``parse_parameters`` methods. If None, will return a
            All other keyword arguments are passed to ``read_raw_samples``.

        Returns
        -------
        FieldArray :
            The samples as a ``FieldArray``.
        """
        loadfields = array_class.parse_parameters(parameters, possible_fields)
        samples = self.read_raw_samples(loadfields, **kwargs)
        # convert to FieldArray
        samples = array_class.from_kwargs(**samples)
        # add the static params and attributes
        addatrs = (list(self.static_params.items()) +
                   list(self[self.samples_group].attrs.items()))
        for (p, val) in addatrs:
            setattr(samples, format_attr(p), format_attr(val))
        return samples

    def read_raw_samples(self, fields, **kwargs):
        """Low level function for reading datasets in the samples group.

        This should return a dictionary of numpy arrays.
        """
        pass

    def extra_args_parser(parser=None, skip_args=None, **kwargs):
        """Provides a parser that can be used to parse sampler-specific command
        line options for loading samples.

        This is optional. Inheriting classes may override this if they want to
        implement their own options.

        parser : argparse.ArgumentParser, optional
            Instead of creating a parser, add arguments to the given one. If
            none provided, will create one.
        skip_args : list, optional
            Don't include the given options. Options should be given as the
            option string, minus the '--'. For example,
            ``skip_args=['iteration']`` would cause the ``--iteration``
            argument not to be included.
            All other keyword arguments are passed to the parser that is
            created.

        Returns
        -------
        parser : argparse.ArgumentParser or None
            If this class adds extra arguments, an argument parser with the
            extra arguments. Otherwise, will just return whatever was passed
            for the ``parser`` argument (default is None).
        actions : list of argparse.Action
            List of the actions that were added.
        """
        return parser, []

    def _get_optional_args(args, opts, err_on_missing=False, **kwargs):
        """Convenience function to retrieve arguments from an argparse
        namespace.

        args : list of str
            List of arguments to retreive.
        opts : argparse.namespace
            Namespace to retreive arguments for.
        err_on_missing : bool, optional
            If an argument is not found in the namespace, raise an
            AttributeError. Otherwise, just pass. Default is False.
            All other keyword arguments are added to the return dictionary.
            Any keyword argument that is the same as an argument in ``args``
            will override what was retrieved from ``opts``.

        Returns
        -------
        dict :
            Dictionary mapping arguments to values retrieved from ``opts``. If
            keyword arguments were provided, these will also be included in the
            dictionary.
        """
        parsed = {}
        for arg in args:
            try:
                parsed[arg] = getattr(opts, arg)
            except AttributeError as e:
                if err_on_missing:
                    raise AttributeError(e)
                else:
                    continue
        parsed.update(kwargs)
        return parsed

    def samples_from_cli(self, opts, parameters=None, **kwargs):
        """Reads samples from the given command-line options.

        opts : argparse Namespace
            The options with the settings to use for loading samples (the sort
            of thing returned by ``ArgumentParser().parse_args``).
        parameters : (list of) str, optional
            A list of the parameters to load. If none provided, will try to
            get the parameters to load from ``opts.parameters``.
            All other keyword arguments are passed to ``read_samples``. These
            will override any options with the same name.

        Returns
        -------
            Array of the loaded samples.
        """
        if parameters is None and opts.parameters is None:
            parameters = self.variable_params
        elif parameters is None:
            parameters = opts.parameters
        # parse optional arguments
        _, extra_actions = self.extra_args_parser()
        extra_args = [act.dest for act in extra_actions]
        kwargs = self._get_optional_args(extra_args, opts, **kwargs)
        return self.read_samples(parameters, **kwargs)

    @property
    def static_params(self):
        """Returns a dictionary of the static_params. The keys are the argument
        names, values are the value they were set to.
        """
        return {arg: self.attrs[arg] for arg in self.attrs["static_params"]}

    @property
    def effective_nsamples(self):
        """Returns the effective number of samples stored in the file.
        """
        try:
            return self.attrs['effective_nsamples']
            return 0

    def write_effective_nsamples(self, effective_nsamples):
        """Writes the effective number of samples stored in the file."""
        self.attrs['effective_nsamples'] = effective_nsamples

    @property
    def thin_start(self):
        """The default start index to use when reading samples.

        Unless overridden by sub-class attribute, just returns 0.
        """
        return 0

    @property
    def thin_interval(self):
        """The default interval to use when reading samples.

        Unless overridden by sub-class attribute, just returns 1.
        """
        return 1

    @property
    def thin_end(self):
        """The defaut end index to use when reading samples.

        Unless overriden by sub-class attribute, just return ``None``.
        """

    @property
    def cmd(self):
        """Returns the (last) saved command line.

        If the file was created from a run that resumed from a checkpoint, only
        the last command line used is returned.

        Returns
        -------
        cmd : string
            The command line that created this InferenceFile.
        """
        cmd = self.attrs["cmd"]
        if isinstance(cmd, numpy.ndarray):
            cmd = cmd[-1]
        return cmd

    def write_logevidence(self, lnz, dlnz):
        """Writes the given log evidence and its error.

        Results are saved to file's 'log_evidence' and 'dlog_evidence'
        attributes.

        lnz : float
            The log of the evidence.
        dlnz : float
            The error in the estimate of the log evidence.
        """
        self.attrs['log_evidence'] = lnz
        self.attrs['dlog_evidence'] = dlnz

    @property
    def log_evidence(self):
        """Returns the log of the evidence and its error, if they exist in the
        file. Raises a KeyError otherwise.
        """
        return self.attrs["log_evidence"], self.attrs["dlog_evidence"]

    def write_random_state(self, group=None, state=None):
        """Writes the state of the random number generator from the file.

        The random state is written to ``sampler_group``/random_state.

        group : str
            Name of group to write random state to.
        state : tuple, optional
            Specify the random state to write. If None, will use
            ``numpy.random.get_state()``.
        """
        group = self.sampler_group if group is None else group
        dataset_name = "/".join([group, "random_state"])
        if state is None:
            state = numpy.random.get_state()
        s, arr, pos, has_gauss, cached_gauss = state
        if dataset_name in self:
            self[dataset_name][:] = arr
        else:
            self.create_dataset(dataset_name, arr.shape, fletcher32=True,
                                dtype=arr.dtype)
        self[dataset_name].attrs["s"] = s
        self[dataset_name].attrs["pos"] = pos
        self[dataset_name].attrs["has_gauss"] = has_gauss
        self[dataset_name].attrs["cached_gauss"] = cached_gauss

    def read_random_state(self, group=None):
        """Reads the state of the random number generator from the file.

        group : str
            Name of group to read random state from.

        Returns
        -------
        tuple
            A tuple with 5 elements that can be passed to numpy.set_state.
        """
        arr = self[dataset_name][:]
        s = self[dataset_name].attrs["s"]
        pos = self[dataset_name].attrs["pos"]
        has_gauss = self[dataset_name].attrs["has_gauss"]
        cached_gauss = self[dataset_name].attrs["cached_gauss"]
        return s, arr, pos, has_gauss, cached_gauss

    def write_strain(self, strain_dict, group=None):
        """Writes strain for each IFO to file.

        strain : {dict, FrequencySeries}
            A dict of FrequencySeries where the key is the IFO.
        group : {None, str}
            The group to write the strain to. If None, will write to the top
            level.
        """
        subgroup = self.data_group + "/{ifo}/strain"
        if group is None:
            group = subgroup
        else:
            group = '/'.join([group, subgroup])
        for ifo, strain in strain_dict.items():
            self[group.format(ifo=ifo)] = strain
            self[group.format(ifo=ifo)].attrs['delta_t'] = strain.delta_t
            self[group.format(ifo=ifo)].attrs['start_time'] = \
                float(strain.start_time)

    def write_stilde(self, stilde_dict, group=None):
        """Writes stilde for each IFO to file.

        stilde : {dict, FrequencySeries}
            level.
        """
        subgroup = self.data_group + "/{ifo}/stilde"
        else:
        for ifo, stilde in stilde_dict.items():
            self[group.format(ifo=ifo)] = stilde
            self[group.format(ifo=ifo)].attrs['delta_f'] = stilde.delta_f
            self[group.format(ifo=ifo)].attrs['epoch'] = float(stilde.epoch)

    def write_psd(self, psds, group=None):
        """Writes PSD for each IFO to file.

        psds : {dict, FrequencySeries}
            The group to write the psd to. Default is ``data_group``.
        """
        subgroup = self.data_group + "/{ifo}/psds/0"
        else:
        for ifo in psds:
            self[group.format(ifo=ifo)] = psds[ifo]
            self[group.format(ifo=ifo)].attrs['delta_f'] = psds[ifo].delta_f

    def write_injections(self, injection_file):
        """Writes injection parameters from the given injection file.

        Everything in the injection file is copied to ``injections_group``.

        injection_file : str
            Path to HDF injection file.
        """
        logging.info("Writing injection file to output")
        try:
            with h5py.File(injection_file, "r") as fp:
                super(BaseInferenceFile, self).copy(fp, self.injections_group)
        except IOError:
            logging.warn("Could not read %s as an HDF file", injection_file)

    def read_injections(self):
        """Gets injection parameters.

        Returns
        -------
        FieldArray
            Array of the injection parameters.
        """
        injset = InjectionSet(self.filename, hdf_group=self.injections_group)
        injections = injset.table.view(FieldArray)
        # close the new open filehandler to self
        injset._injhandler.filehandler.close()
        return injections

    def write_command_line(self):
        """Writes command line to attributes.

        The command line is written to the file's ``attrs['cmd']``. If this
        attribute already exists in the file (this can happen when resuming
        from a checkpoint), ``attrs['cmd']`` will be a list storing the current
        command line and all previous command lines.
        """
        cmd = [" ".join(sys.argv)]
        try:
            previous = self.attrs["cmd"]
            if isinstance(previous, str):
                # convert to list
                previous = [previous]
            elif isinstance(previous, numpy.ndarray):
                previous = previous.tolist()
            previous = []
        self.attrs["cmd"] = cmd + previous

    def get_slice(thin_start=None, thin_interval=None, thin_end=None):
        """Formats a slice to retrieve a thinned array from an HDF file.

        thin_start : float or int, optional
            The starting index to use. If provided, the ``int`` will be taken.
        thin_interval : float or int, optional
            The interval to use. If provided the ceiling of it will be taken.
        thin_end : float or int, optional
            The end index to use. If provided, the ``int`` will be taken.

        Returns
        -------
        slice :
            The slice needed.
        """
        if thin_start is not None:
            thin_start = int(thin_start)
        if thin_interval is not None:
            thin_interval = int(numpy.ceil(thin_interval))
        if thin_end is not None:
            thin_end = int(thin_end)
        return slice(thin_start, thin_end, thin_interval)

    def copy_metadata(self, other):
        """Copies all metadata from this file to the other file.

        Metadata is defined as everything in the top-level ``.attrs``.

        other : InferenceFile
            An open inference file to write the data to.
        """
        logging.info("Copying metadata")
        # copy attributes
        for key in self.attrs.keys():
            other.attrs[key] = self.attrs[key]

    def copy_info(self, other, ignore=None):
        """Copies "info" from this file to the other.

        "Info" is defined all groups that are not the samples group.

        other : output file
            The output file. Must be an hdf file.
        ignore : (list of) str
            Don't copy the given groups.
        """
        logging.info("Copying info")
        # copy non-samples/stats data
        if ignore is None:
            ignore = []
        if isinstance(ignore, string_types):
            ignore = [ignore]
        ignore = set(ignore + [self.samples_group])
        copy_groups = set(self.keys()) - ignore
        for key in copy_groups:
            super(BaseInferenceFile, self).copy(key, other)

    def copy_samples(self, other, parameters=None, parameter_names=None,
                     read_args=None, write_args=None):
        """Should copy samples to the other files.

            An open inference file to write to.
        parameters : list of str, optional
            List of parameters to copy. If None, will copy all parameters.
        parameter_names : dict, optional
            Rename one or more parameters to the given name. The dictionary
            should map parameter -> parameter name. If None, will just use the
            original parameter names.
        read_args : dict, optional
            Arguments to pass to ``read_samples``.
        write_args : dict, optional
            Arguments to pass to ``write_samples``.
        """
        # select the samples to copy
        logging.info("Reading samples to copy")
        if parameters is None:
        # if list of desired parameters is different, rename
        if set(parameters) != set(self.variable_params):
            other.attrs['variable_params'] = parameters
        if read_args is None:
            read_args = {}
        samples = self.read_samples(parameters, **read_args)
        logging.info("Copying {} samples".format(samples.size))
        # if different parameter names are desired, get them from the samples
        if parameter_names:
            arrs = {pname: samples[p] for p, pname in parameter_names.items()}
            arrs.update({p: samples[p] for p in parameters if
                         p not in parameter_names})
            samples = FieldArray.from_kwargs(**arrs)
            other.attrs['variable_params'] = samples.fieldnames
        logging.info("Writing samples")
        if write_args is None:
            write_args = {}
        other.write_samples({p: samples[p] for p in samples.fieldnames},
                            **write_args)

    def copy(self, other, ignore=None, parameters=None, parameter_names=None,
             read_args=None, write_args=None):
        """Copies metadata, info, and samples in this file to another file.

        other : str or InferenceFile
            The file to write to. May be either a string giving a filename,
            or an open hdf file. If the former, the file will be opened with
            the write attribute (note that if a file already exists with that
            name, it will be deleted).
        ignore : (list of) strings
            Don't copy the given groups. If the samples group is included, no
            samples will be copied.
            List of parameters in the samples group to copy. If None, will copy
            all parameters.

        Returns
        -------
        InferenceFile
            The open file handler to other.
        """
        if not isinstance(other, h5py.File):
            # check that we're not trying to overwrite this file
            if other == self.name:
                raise IOError("destination is the same as this file")
            other = self.__class__(other, 'w')
        # metadata
        self.copy_metadata(other)
        # info
            ignore = []
        self.copy_info(other, ignore=ignore)
        # samples
        if self.samples_group not in ignore:
            self.copy_samples(other, parameters=parameters,
                              parameter_names=parameter_names,
                              read_args=read_args,
                              write_args=write_args)
            # if any down selection was done, re-set the default
            # thin-start/interval/end
            p = tuple(self[self.samples_group].keys())[0]
            my_shape = self[self.samples_group][p].shape
            p = tuple(other[other.samples_group].keys())[0]
            other_shape = other[other.samples_group][p].shape
            if my_shape != other_shape:
                other.attrs['thin_start'] = 0
                other.attrs['thin_interval'] = 1
                other.attrs['thin_end'] = None
        return other

    def write_kwargs_to_attrs(cls, attrs, **kwargs):
        """Writes the given keywords to the given ``attrs``.

        If any keyword argument points to a dict, the keyword will point to a
        list of the dict's keys. Each key is then written to the attrs with its
        corresponding value.

        attrs : an HDF attrs
            The ``attrs`` of an hdf file or a group in an hdf file.
            The keywords to write.
        """
        for arg, val in kwargs.items():
            if val is None:
                val = str(None)
            if isinstance(val, dict):
                attrs[arg] = list(val.keys())
                # just call self again with the dict as kwargs
                cls.write_kwargs_to_attrs(attrs, **val)
            else:
                attrs[arg] = val

    def write_data(self, name, data, path=None, append=False):
        """Convenience function to write data.

        Given ``data`` is written as a dataset with ``name`` in ``path``.
        If the dataset or path do not exist yet, the dataset and path will
        be created.

        name : str
            The name to associate with the data. This will be the dataset
            name (if data is array-like) or the key in the attrs.
        data : array, dict, or atomic
            The data to write. If a dictionary, a subgroup will be created
            for each key, and the values written there. This will be done
            recursively until an array or atomic (e.g., float, int, str), is
            found. Otherwise, the data is written to the given name.
            Write to the given path. Default (None) will write to the top
            level. If the path does not exist in the file, it will be
            created.
        append : bool, optional
            Append the data to what is currently in the file if ``path/name``
            already exists in the file, and if it does not, create the dataset
            so that its last dimension can be resized. The data can only
            be appended along the last dimension, and if it already exists in
            the data, it must be resizable along this dimension. If ``False``
            (the default) what is in the file will be overwritten, and the
            given data must have the same shape.
        """
            path = '/'
        try:
            group = self[path]
            # create the group
            self.create_group(path)
        if isinstance(data, dict):
            # call myself for each key, value pair in the dictionary
            for key, val in data.items():
                self.write_data(key, val, path='/'.join([path, name]),
                                append=append)
        # if appending, we need to resize the data on disk, or, if it doesn't
        # exist yet, create a dataset that is resizable along the last
        # dimension
        elif append:
            # cast the data to an array if it isn't already one
            if isinstance(data, (list, tuple)):
                data = numpy.array(data)
            if not isinstance(data, numpy.ndarray):
                data = numpy.array([data])
            dshape = data.shape
            ndata = dshape[-1]
            try:
                startidx = group[name].shape[-1]
                group[name].resize(dshape[-1]+group[name].shape[-1],
                                   axis=len(group[name].shape)-1)
                # dataset doesn't exist yet
                group.create_dataset(name, dshape,
                                     maxshape=tuple(list(dshape)[:-1]+[None]),
                                     dtype=data.dtype, fletcher32=True)
                startidx = 0
            group[name][..., startidx:startidx+ndata] = data[..., :]
        else:
            try:
                group[name][()] = data
                group[name] = data

    def write_config_file(self, cp):
        """Writes the given config file parser.

        File is stored as a pickled buffer array to ``config_parser/{index}``,
        where ``{index}`` is an integer corresponding to the number of config
        files that have been saved. The first time a save is called, it is
        stored to ``0``, and incremented from there.

        cp : ConfigParser
            Config parser to save.
        """
        # get the index of the last saved file
        try:
            index = list(map(int, self[self.config_group].keys()))
            index = []
        if index == []:
            # hasn't been written yet
            index = 0
        else:
            index = max(index) + 1
        # we'll store the config file as a text file that is pickled
        out = StringIO()
        cp.write(out)
        # now pickle it
        dump_state(out, self, path=self.config_group, dsetname=str(index))

    def read_config_file(self, return_cp=True, index=-1):
        """Reads the config file that was used.

        A ``ValueError`` is raised if no config files have been saved, or if
        the requested index larger than the number of stored config files.

        return_cp : bool, optional
            If true, returns the loaded config file as
            :py:class:`pycbc.workflow.configuration.WorkflowConfigParser`
            type. Otherwise will return as string buffer. Default is True.
        index : int, optional
            The config file to load. If ``write_config_file`` has been called
            multiple times (as would happen if restarting from a checkpoint),
            there will be config files stored. Default (-1) is to load the
            last saved file.

        Returns
        -------
        WorkflowConfigParser or StringIO :
            The parsed config file.
        """
        # get the stored indices
        try:
            indices = sorted(map(int, self[self.config_group].keys()))
            index = indices[index]
            raise ValueError("no config files saved in hdf")
        except IndexError:
            raise ValueError("no config file matches requested index")
        cf = load_state(self, path=self.config_group, dsetname=str(index))
        cf.seek(0)
        if return_cp:
            cp = WorkflowConfigParser()
            cp.read_file(cf)
            return cp
        return cf

    def read_data(self):
        """Loads the data stored in the file as a FrequencySeries.

        Only works for models that store data as a frequency series in
        ``data/DET/stilde``. A ``KeyError`` will be raised if the model used
        did not store data in that path.

        Returns
        -------
        dict :
            Dictionary of detector name -> FrequencySeries.
        """
        data = {}
        fmt = 'data/{}/stilde'
        for det in self['data'].keys():
            group = self[fmt.format(det)]
            data[det] = FrequencySeries(
                group[()], delta_f=group.attrs['delta_f'],
                epoch=group.attrs['epoch'])
        return data

    def read_psds(self):
        """Loads the PSDs stored in the file as a FrequencySeries.

        Only works for models that store PSDs in
        ``data/DET/psds/0``. A ``KeyError`` will be raised if the model used
        did not store PSDs in that path.

        Returns
        -------
        dict :
        """
        psds = {}
        fmt = 'data/{}/psds/0'
            psds[det] = FrequencySeries(
                group[()], delta_f=group.attrs['delta_f'])
        return psds

#
#

#
#
#                                   Preamble
#
#
"""Provides I/O that is specific to MCMC samplers.
"""

from __future__ import (absolute_import, division)

class CommonMCMCMetadataIO(object):
    """Provides functions for reading/writing MCMC metadata to file.

    The functions here are common to both standard MCMC (in which chains
    are independent) and ensemble MCMC (in which chains/walkers share
    information).
    """
    def write_resume_point(self):
        """Keeps a list of the number of iterations that were in a file when a
        run was resumed from a checkpoint."""
        try:
            resume_pts = self.attrs["resume_points"].tolist()
            resume_pts = []
        try:
            niterations = self.niterations
            niterations = 0
        resume_pts.append(niterations)
        self.attrs["resume_points"] = resume_pts

    def write_niterations(self, niterations):
        """Writes the given number of iterations to the sampler group."""
        self[self.sampler_group].attrs['niterations'] = niterations

    @property
    def niterations(self):
        """Returns the number of iterations the sampler was run for."""
        return self[self.sampler_group].attrs['niterations']

    @property
    def nwalkers(self):
        """Returns the number of walkers used by the sampler.

        Alias of ``nchains``.
        """
        try:
            return self[self.sampler_group].attrs['nwalkers']
            return self[self.sampler_group].attrs['nchains']

    @property
    def nchains(self):
        """Returns the number of chains used by the sampler.

        Alias of ``nwalkers``.
        """
        try:

    def _thin_data(self, group, params, thin_interval):
        """Thins data on disk by the given interval.

        This makes no effort to record the thinning interval that is applied.

        group : str
            The group where the datasets to thin live.
        params : list
            The list of dataset names to thin.
        thin_interval : int
            The interval to thin the samples on disk by.
        """
        samples = self.read_raw_samples(params, thin_start=0,
                                        thin_interval=thin_interval,
                                        thin_end=None, flatten=False,
                                        group=group)
        # now resize and write the data back to disk
        fpgroup = self[group]
        for param in params:
            data = samples[param]
            # resize the arrays on disk
            fpgroup[param].resize(data.shape)
            # and write
            fpgroup[param][:] = data

    def thin(self, thin_interval):
        """Thins the samples on disk to the given thinning interval.

        The interval must be a multiple of the file's current ``thinned_by``.

            The interval the samples on disk should be thinned by.
        """
        # get the new interval to thin by
        new_interval = thin_interval / self.thinned_by
        if new_interval % 1:
            raise ValueError("thin interval ({}) must be a multiple of the "
                             "current thinned_by ({})"
                             .format(thin_interval, self.thinned_by))
        new_interval = int(new_interval)
        # now thin the data on disk
        params = list(self[self.samples_group].keys())
        self._thin_data(self.samples_group, params, new_interval)
        # store the interval that samples were thinned by
        self.thinned_by = thin_interval

    @property
    def thinned_by(self):
        """Returns interval samples have been thinned by on disk.

        This looks for ``thinned_by`` in the samples group attrs. If none is
        found, will just return 1.
        """
        try:
            thinned_by = self.attrs['thinned_by']
            thinned_by = 1
        return thinned_by

    @thinned_by.setter
    def thinned_by(self, thinned_by):
        """Sets the thinned_by attribute.

        This is the interval that samples have been thinned by on disk. The
        given value is written to
        ``self[self.samples_group].attrs['thinned_by']``.
        """
        self.attrs['thinned_by'] = int(thinned_by)

    def last_iteration(self, parameter=None, group=None):
        """Returns the iteration of the last sample of the given parameter.

        parameter : str, optional
            The name of the parameter to get the last iteration for. If
            None provided, will just use the first parameter in ``group``.
        group : str, optional
            The name of the group to get the last iteration from. Default is
            the ``samples_group``.
        """
            group = self.samples_group
        if parameter is None:
            try:
                parameter = list(self[group].keys())[0]
            except (IndexError, KeyError):
                # nothing has been written yet, just return 0
                return 0
        try:
            lastiter = self[group][parameter].shape[-1]
            # no samples have been written, just return 0
            lastiter = 0
        # account for thinning
        return lastiter * self.thinned_by

    def iterations(self, parameter):
        """Returns the iteration each sample occurred at."""
        return numpy.arange(0, self.last_iteration(parameter), self.thinned_by)

    def write_sampler_metadata(self, sampler):
        """Writes the sampler's metadata."""
        self.attrs['sampler'] = sampler.name
        try:
            self[self.sampler_group].attrs['nchains'] = sampler.nchains
            self[self.sampler_group].attrs['nwalkers'] = sampler.nwalkers
        # write the model's metadata
        sampler.model.write_metadata(self)

    @property
    def is_burned_in(self):
        """Returns whether or not chains are burned in.

        Raises a ``ValueError`` if no burn in tests were done.
        """
        try:
            return self[self.sampler_group]['is_burned_in'][()]
            raise ValueError("No burn in tests were performed")

    @property
    def burn_in_iteration(self):
        """Returns the burn in iteration of all the chains.

        """
        try:
            return self[self.sampler_group]['burn_in_iteration'][()]

    @property
    def burn_in_index(self):
        """Returns the burn in index.

        This is the burn in iteration divided by the file's ``thinned_by``.
        Requires the class that this is used with has a ``burn_in_iteration``
        attribute.
        """
        return self.burn_in_iteration // self.thinned_by

    @property
    def act(self):
        """The autocorrelation time (ACT).

        This is the ACL times the file's thinned by. Raises a ``ValueError``
        if the ACT has not been calculated.
        """
        try:
            return self[self.sampler_group]['act'][()]
            raise ValueError("ACT has not been calculated")

    @act.setter
    def act(self, act):
        """Writes the autocorrelation time(s).

        ACT(s) are written to the ``sample_group`` as a dataset with name
        ``act``.

        act : array or int
            ACT(s) to write.
        """
        # pylint: disable=no-member
        self.write_data('act', act, path=self.sampler_group)

    @property
    def raw_acts(self):
        """Dictionary of parameter names -> raw autocorrelation time(s).

        Depending on the sampler, the autocorrelation times may be floats,
        or [ntemps x] [nchains x] arrays.

        Raises a ``ValueError`` is no raw acts have been set.
        """
        try:
            group = self[self.sampler_group]['raw_acts']
            raise ValueError("ACTs have not been calculated")
        acts = {}
        for param in group:
            acts[param] = group[param][()]
        return acts

    @raw_acts.setter
    def raw_acts(self, acts):
        """Writes the raw autocorrelation times.

        The ACT of each parameter is saved to
        ``[sampler_group]/raw_acts/{param}']``. Works for all types of MCMC
        samplers (independent chains, ensemble, parallel tempering).

        acts : dict
            A dictionary of ACTs keyed by the parameter.
        """
        path = self.sampler_group + '/raw_acts'
        for param in acts:
            self.write_data(param, acts[param], path=path)

    @property
    def acl(self):
        """The autocorrelation length (ACL) of the samples.

        This is the autocorrelation time (ACT) divided by the file's
        ``thinned_by`` attribute. Raises a ``ValueError`` if the ACT has not
        been calculated.
        """
        return self.act / self.thinned_by

    @acl.setter
    def acl(self, acl):
        """Sets the autocorrelation length (ACL) of the samples.

        This will convert the given value(s) to autocorrelation time(s) and
        save to the ``act`` attribute; see that attribute for details.
        """
        self.act = acl * self.thinned_by

    @property
    def raw_acls(self):
        """Dictionary of parameter names -> raw autocorrelation length(s).

        Depending on the sampler, the autocorrelation lengths may be floats,

        The ACLs are the autocorrelation times (ACT) divided by the file's
        ``thinned_by`` attribute. Raises a ``ValueError`` is no raw acts have
        been set.
        """
        return {p: self.raw_acts[p] / self.thinned_by for p in self.raw_acts}

    @raw_acls.setter
    def raw_acls(self, acls):
        """Sets the raw autocorrelation lengths.

        The given ACLs are converted to autocorrelation times (ACTs) and saved
        to the ``raw_acts`` attribute; see that attribute for details.

        acls : dict
            A dictionary of ACLs keyed by the parameter.
        """
        self.raw_acts = {p: acls[p] * self.thinned_by for p in acls}

    def _update_sampler_history(self):
        """Writes the number of iterations, effective number of samples,
        autocorrelation times, and burn-in iteration to the history.
        """
        path = '/'.join([self.sampler_group, 'checkpoint_history'])
        # write the current number of iterations
        self.write_data('niterations', self.niterations, path=path,
                        append=True)
        self.write_data('effective_nsamples', self.effective_nsamples,
                        path=path, append=True)
        # write the act: we'll make sure that this is 2D, so that the acts
        # can be appened along the last dimension
        try:
            act = self.act
            # no acts were calculate
            act = None
        if act is not None:
            act = act.reshape(tuple(list(act.shape)+[1]))
            self.write_data('act', act, path=path, append=True)
        # write the burn in iteration in the same way
        try:
            burn_in = self.burn_in_iteration
            # no burn in tests were done
            burn_in = None
        if burn_in is not None:
            burn_in = burn_in.reshape(tuple(list(burn_in.shape)+[1]))
            self.write_data('burn_in_iteration', burn_in, path=path,
                            append=True)

        """Create a parser to parse sampler-specific arguments for loading
        samples.

            Don't parse the given options. Options should be given as the
            created.

        Returns
        -------
        parser : argparse.ArgumentParser
            An argument parser with th extra arguments added.
            A list of the actions that were added.
        """
        if parser is None:
            parser = argparse.ArgumentParser(**kwargs)
        elif kwargs:
            raise ValueError("No other keyword arguments should be provded if "
                             "a parser is provided.")
        if skip_args is None:
            skip_args = []
        actions = []
        if 'thin-start' not in skip_args:
            act = parser.add_argument(
                "--thin-start", type=int, default=None,
                help="Sample number to start collecting samples. If "
                     "none provided, will use the input file's `thin_start` "
                     "attribute.")
            actions.append(act)
        if 'thin-interval' not in skip_args:
                "--thin-interval", type=int, default=None,
                help="Interval to use for thinning samples. If none provided, "
                     "will use the input file's `thin_interval` attribute.")
        if 'thin-end' not in skip_args:
                "--thin-end", type=int, default=None,
                help="Sample number to stop collecting samples. If "
                     "none provided, will use the input file's `thin_end` "
        if 'iteration' not in skip_args:
                "--iteration", type=int, default=None,
                help="Only retrieve the given iteration. To load "
                     "the last n-th sampe use -n, e.g., -1 will "
                     "load the last iteration. This overrides "
                     "the thin-start/interval/end options.")
        if 'walkers' not in skip_args and 'chains' not in skip_args:
                "--walkers", "--chains", type=int, nargs="+", default=None,
                help="Only retrieve samples from the listed "
                     "walkers/chains. Default is to retrieve from all "
                     "walkers/chains.")
        return parser, actions

class MCMCMetadataIO(object):
    """Provides functions for reading/writing metadata to file for MCMCs in
    which all chains are independent of each other.

    Overrides the ``BaseInference`` file's ``thin_start`` and ``thin_interval``
    attributes. Instead of integers, these return arrays.
    """
    @property
        """Returns the default thin start to use for reading samples.

        If burn-in tests were done, this will return the burn-in index of every
        chain that has burned in. The start index for chains that have not
        burned in will be greater than the number of samples, so that those
        chains return no samples. If no burn-in tests were done, returns 0
        for all chains.
        """
        try:
            thin_start = self.burn_in_index
            # replace any that have not been burned in with the number
            # of iterations; this will cause those chains to not return
            # any samples
            thin_start[~self.is_burned_in] = \
                int(numpy.ceil(self.niterations/self.thinned_by))
            return thin_start
            # no burn in, just return array of zeros
            return numpy.zeros(self.nchains, dtype=int)

    @property
        """Returns the default thin interval to use for reading samples.

        If a finite ACL exists in the file, will return that. Otherwise,
        returns 1.
        """
        try:
            acl = self.acl
            return numpy.ones(self.nchains, dtype=int)
        # replace any infs with the number of samples
        acl[numpy.isinf(acl)] = self.niterations / self.thinned_by
        return numpy.ceil(acl).astype(int)

class EnsembleMCMCMetadataIO(object):
    """Provides functions for reading/writing metadata to file for ensemble
    MCMCs.
    """
    @property

        If burn-in tests were done, returns the burn in index. Otherwise,
        returns 0.
        """
        try:
            return self.burn_in_index
            # no burn in, just return 0
            return 0

    @property

        returns 1.
        """
        try:
            acl = 1
        if numpy.isfinite(acl):
        else:
            acl = 1
        return acl

def write_samples(fp, samples, parameters=None, last_iteration=None,
                  samples_group=None, thin_by=None):
    """Writes samples to the given file.

    This works for both standard MCMC and ensemble MCMC samplers without
    parallel tempering.

    Results are written to ``samples_group/{vararg}``, where ``{vararg}``
    is the name of a model params. The samples are written as an
    ``nwalkers x niterations`` array. If samples already exist, the new
    samples are appended to the current.

    If the current samples on disk have been thinned (determined by the
    ``thinned_by`` attribute in the samples group), then the samples will
    be thinned by the same amount before being written. The thinning is
    started at the sample in ``samples`` that occured at the iteration
    equal to the last iteration on disk plus the ``thinned_by`` interval.
    If this iteration is larger than the iteration of the last given
    sample, then none of the samples will be written.

    fp : BaseInferenceFile
        Open file handler to write files to. Must be an instance of
        BaseInferenceFile with CommonMCMCMetadataIO methods added.
    samples : dict
        The samples to write. Each array in the dictionary should have
        shape nwalkers x niterations.
    parameters : list, optional
        Only write the specified parameters to the file. If None, will
        write all of the keys in the ``samples`` dict.
    last_iteration : int, optional
        The iteration of the last sample. If the file's ``thinned_by``
        attribute is > 1, this is needed to determine where to start
        thinning the samples such that the interval between the last sample
        currently on disk and the first new sample is the same as all of
        the other samples.
    samples_group : str, optional
        Which group to write the samples to. Default (None) will result
        in writing to "samples".
    thin_by : int, optional
        Override the ``thinned_by`` attribute in the file with the given
        value. **Only set this if you are using this function to write
        something other than inference samples!**
    """
    nwalkers, nsamples = list(samples.values())[0].shape
    assert all(p.shape == (nwalkers, nsamples)
               for p in samples.values()), (
           "all samples must have the same shape")
    if samples_group is None:
        samples_group = fp.samples_group
        parameters = samples.keys()
    # thin the samples
    samples = thin_samples_for_writing(fp, samples, parameters,
                                       last_iteration, samples_group,
                                       thin_by=thin_by)
    # loop over number of dimensions
    group = samples_group + '/{name}'
    for param in parameters:
        dataset_name = group.format(name=param)
        data = samples[param]
        # check that there's something to write after thinning
        if data.shape[1] == 0:
            # nothing to write, move along
            continue
        try:
            fp_nsamples = fp[dataset_name].shape[-1]
            istart = fp_nsamples
            istop = istart + data.shape[1]
            if istop > fp_nsamples:
                # resize the dataset
                fp[dataset_name].resize(istop, axis=1)
            # dataset doesn't exist yet
            istart = 0
            fp.create_dataset(dataset_name, (nwalkers, istop),
                              maxshape=(nwalkers, None),
                              dtype=data.dtype,
                              fletcher32=True)
        fp[dataset_name][:, istart:istop] = data

def ensemble_read_raw_samples(fp, fields, thin_start=None,
                              thin_interval=None, thin_end=None,
                              iteration=None, walkers=None, flatten=True,
                              group=None):
    """Base function for reading samples from ensemble MCMC files without

        BaseInferenceFile with EnsembleMCMCMetadataIO methods added.
    fields : list
        The list of field names to retrieve.
    thin_start : int, optional
        Start reading from the given iteration. Default is to start from
        the first iteration.
    thin_interval : int, optional
        Only read every ``thin_interval`` -th sample. Default is 1.
    thin_end : int, optional
        Stop reading at the given iteration. Default is to end at the last
        iteration.
    iteration : int, optional
        Only read the given iteration. If this provided, it overrides
        the ``thin_(start|interval|end)`` options.
    walkers : (list of) int, optional
        Only read from the given walkers. Default (``None``) is to read all.
    flatten : bool, optional
        Flatten the samples to 1D arrays before returning. Otherwise, the
        returned arrays will have shape (requested walkers x
        requested iteration(s)). Default is True.
    group : str, optional
        The name of the group to read sample datasets from. Default is
        the file's ``samples_group``.

    Returns
    -------
    dict
        A dictionary of field name -> numpy array pairs.
    """
    if isinstance(fields, string_types):
        fields = [fields]
    # walkers to load
    widx, nwalkers = _ensemble_get_walker_index(fp, walkers)
    # get the slice to use
    get_index = _ensemble_get_index(fp, thin_start, thin_interval, thin_end,
                                    iteration)
    # load
    if group is None:
        group = fp.samples_group
    group = group + '/{name}'
    arrays = {}
    for name in fields:
        arr = fp[group.format(name=name)][widx, get_index]
        niterations = arr.shape[-1] if iteration is None else 1
        if flatten:
            arr = arr.flatten()
        else:
            # ensure that the returned array is 2D
            arr = arr.reshape((nwalkers, niterations))
        arrays[name] = arr
    return arrays

def _ensemble_get_walker_index(fp, walkers=None):
    """Convenience function to determine which walkers to load.

    Returns
    -------
    widx : array or slice
        The walker indices to load.
    nwalkers : int
        The number of walkers that will be loaded.
    """
    if walkers is not None:
        widx = numpy.zeros(fp.nwalkers, dtype=bool)
        widx[walkers] = True
        nwalkers = widx.sum()
    else:
        widx = slice(None, None)
        nwalkers = fp.nwalkers
    return widx, nwalkers

def _ensemble_get_index(fp, thin_start=None, thin_interval=None, thin_end=None,
                        iteration=None):
    """Determines the sample indices to retrieve for an ensemble MCMC.

    Returns
    -------
    slice or int
        The indices to retrieve.
    """
    if iteration is not None:
        get_index = int(iteration)
    else:
        if thin_start is None:
            thin_start = fp.thin_start
        if thin_interval is None:
            thin_interval = fp.thin_interval
        if thin_end is None:
            thin_end = fp.thin_end
        get_index = fp.get_slice(thin_start=thin_start,
                                 thin_interval=thin_interval,
                                 thin_end=thin_end)
    return get_index

def _get_index(fp, chains, thin_start=None, thin_interval=None, thin_end=None,
               iteration=None):
    """Determines the sample indices to retrieve for an MCMC with independent
    chains.

        Open file handler to read samples from. Must be an instance of
    chains : array of int
        The chains to load.
    thin_start : array or int, optional
        Start reading from the given sample. May either provide an array
        indicating the start index for each chain, or an integer. If the
        former, the array must have the same length as the number of chains
        that will be retrieved. If the latter, the given value will be used
        for all chains. Default (None) is to use the file's ``thin_start``
    thin_interval : array or int, optional
        Only read every ``thin_interval``-th sample. May either provide an
        array indicating the interval to use for each chain, or an integer. If
        the former, the array must have the same length as the number of chains
        that will be retrieved. If the latter, the given value will be used for
        all chains. Default (None) is to use the file's ``thin_interval``
    thin_end : array or int, optional
        Stop reading at the given sample index. May either provide an
        array indicating the end index to use for each chain, or an integer. If
        all chains. Default (None) is to use the the file's ``thin_end``
        Only read the given iteration from all chains. If provided, it
        overrides the ``thin_(start|interval|end)`` options.

    Returns
    -------
    get_index : list of slice or int
    """
    nchains = len(chains)
    # convenience function to get the right thin start/interval/end
        get_index = [int(iteration)]*nchains
    else:
        # get the slice arguments
        thin_start = _format_slice_arg(thin_start, fp.thin_start, chains)
        thin_interval = _format_slice_arg(thin_interval, fp.thin_interval,
                                          chains)
        thin_end = _format_slice_arg(thin_end, fp.thin_end, chains)
        # the slices to use for each chain
        get_index = [fp.get_slice(thin_start=thin_start[ci],
                                  thin_interval=thin_interval[ci],
                                  thin_end=thin_end[ci])
                     for ci in range(nchains)]

def _format_slice_arg(value, default, chains):
    """Formats a start/interval/end argument for picking out chains.

    value : None, int, array or list of int
        The thin-start/interval/end value to format. ``None`` indicates the
        user did not specify anything, in which case ``default`` will be used.
        If an integer, then it will be repeated to match the length of
        ``chains```. If an array or list, it must have the same length as
        ``chains``.
    default : array
        What to use instead if ``value`` is ``None``.
        The index values of chains that will be loaded.

    Returns
    -------
    array
        Array giving the value to use for each chain in ``chains``. The array
        will have the same length as ``chains``.
    """
    if value is None and default is None:
        # no value provided, and default is None, just return Nones with the
        # same length as chains
        value = [None]*len(chains)
    elif value is None:
        # use the default, with the desired values extracted
        value = default[chains]
    elif isinstance(value, (int, numpy.int_)):
        # a single integer was provided, repeat into an array
        value = numpy.repeat(value, len(chains))
    elif len(value) != len(chains):
        # a list of values was provided, but the length does not match the
        # chains, raise an error
        raise ValueError("Number of requested thin-start/interval/end values "
                         "({}) does not match number of requested chains ({})"
                         .format(len(value), len(chains)))

def thin_samples_for_writing(fp, samples, parameters, last_iteration,
                             group, thin_by=None):
    """Thins samples for writing to disk.

    The thinning interval to use is determined by the given file handler's
    ``thinned_by`` attribute. If that attribute is 1, just returns the samples.

    fp : CommonMCMCMetadataIO instance
        The file the sampels will be written to. Needed to determine the
        thin interval used on disk.
        Dictionary mapping parameter names to arrays of (unthinned) samples.
        The arrays are thinned along their last dimension.
    parameters : list of str
        The parameters to thin in ``samples`` before writing. All listed
        parameters must be in ``samples``.
    last_iteration : int
        The iteration that the last sample in ``samples`` occurred at. This is
        needed to figure out where to start the thinning in ``samples``, such
        that the interval between the last sample on disk and the first new
        sample is the same as all of the other samples.
    group : str
        The name of the group that the samples will be written to. This is
        needed to determine what the last iteration saved on disk was.
        Override the ``thinned_by`` attribute in the file for with the given
        value. **Only do this if you are thinning something other than
        inference samples!**

    Returns
    -------
    dict :
        Dictionary of the thinned samples to write.
    """
    if thin_by is None:
        thin_by = fp.thinned_by
    if thin_by > 1:
        if last_iteration is None:
            raise ValueError("File's thinned_by attribute is > 1 ({}), "
                             "but last_iteration not provided."
                             .format(thin_by))
        thinned_samples = {}
        for param in parameters:
            nsamples = data.shape[-1]
            # To figure out where to start:
            # the last iteration in the file + the file's thinning interval
            # gives the iteration of the next sample that should be written;
            # last_iteration - nsamples gives the iteration of the first
            # sample in samples. Subtracting the latter from the former - 1
            # (-1 to convert from iteration to index) therefore gives the index
            # in the samples data to start using samples.
            thin_start = fp.last_iteration(param, group) + thin_by \
                - (last_iteration - nsamples) - 1
            thinned_samples[param] = data[..., thin_start::thin_by]
    else:
        thinned_samples = samples
    return thinned_samples

def nsamples_in_chain(start_iter, interval, niterations):
    """Calculates the number of samples in an MCMC chain given a thinning
    start, end, and interval.

    This function will work with either python scalars, or numpy arrays.

    start_iter : (array of) int
        Start iteration. If negative, will count as being how many iterations
        to start before the end; otherwise, counts how many iterations to
        start before the beginning. If this is larger than niterations, will
        just return 0.
    interval : (array of) int
        Thinning interval.
    niterations : (array of) int
        The number of iterations.

    Returns
    -------
    numpy.int_
        The number of samples in a chain, >= 0.
    """
    # this is written in a slightly wonky way so that it will work with either
    # python scalars or numpy arrays; it is equivalent to:
    #    if start_iter < 0:
    #        count = min(abs(start_iter), niterations)
    #    else:
    #        count = max(niterations - start_iter, 0)
    slt0 = start_iter < 0
    sgt0 = start_iter >= 0
    count = slt0*abs(start_iter) + sgt0*(niterations - start_iter)
    # ensure count is in [0, niterations]
    cgtn = count > niterations
    cok = (count >= 0) & (count <= niterations)
    count = cgtn*niterations + cok*count
    return numpy.ceil(count / interval).astype(int)

# Copyright (C) 2018 Collin Capano
#
#

#
#
#                                   Preamble
#
#
"""Provides I/O support for multi-tempered sampler.
"""

from .base_mcmc import (CommonMCMCMetadataIO, thin_samples_for_writing,
                        _ensemble_get_index, _ensemble_get_walker_index,
                        _get_index)

class ParseTempsArg(argparse.Action):
    """Argparse action that will parse temps argument.

    If the provided argument is 'all', sets 'all' in the namespace dest. If a
    a sequence of numbers are provided, converts those numbers to ints before
    saving to the namespace.
    """
    def __init__(self, type=str, **kwargs): # pylint: disable=redefined-builtin
        super(ParseTempsArg, self).__init__(type=type, **kwargs)

        if values[0] == 'all':
            # check that only a single value was provided
            if len(values) > 1:
                raise ValueError("if provide 'all', should not specify any "
                                 "other temps")
            temps = 'all'
        else:
            temps = []
            for val in values:
                try:
                    val = int(val)
                    pass
                temps.append(val)
            if singlearg:
                temps = temps[0]
        setattr(namespace, self.dest, temps)

class CommonMultiTemperedMetadataIO(CommonMCMCMetadataIO):
    """Adds support for reading/writing multi-tempered metadata to
    :py:class:`~pycbc.inference.io.base_mcmc.CommonMCMCMetadatIO`.
    """
    @property
    def ntemps(self):
        """Returns the number of temperatures used by the sampler."""
        return self[self.sampler_group].attrs['ntemps']

        """Adds writing ntemps to file.
        """
        super(CommonMultiTemperedMetadataIO, self).write_sampler_metadata(
            sampler)
        self[self.sampler_group].attrs["ntemps"] = sampler.ntemps

        """Adds --temps to MCMCIO parser.
        """
        parser, actions = CommonMCMCMetadataIO.extra_args_parser(
            parser=parser, skip_args=skip_args, **kwargs)
        if 'temps' not in skip_args:
                "--temps", nargs="+", default=0, action=ParseTempsArg,
                help="Get the given temperatures. May provide either a "
                     "sequence of integers specifying the temperatures to "
                     "plot, or 'all' for all temperatures. Default is to only "
                     "plot the coldest (= 0) temperature chain.")

    This works both for standard MCMC and ensemble MCMC samplers with

    ``ntemps x nwalkers x niterations`` array.

        BaseInferenceFile with CommonMultiTemperedMetadataIO methods added.
        shape ntemps x nwalkers x niterations.
        thinning the samples to match what has already been stored on disk.
    """
    ntemps, nwalkers, niterations = tuple(samples.values())[0].shape
    assert all(p.shape == (ntemps, nwalkers, niterations)
        parameters = list(samples.keys())
        if data.shape[2] == 0:
            continue
        try:
            fp_niterations = fp[dataset_name].shape[-1]
            istart = fp_niterations
            istop = istart + data.shape[2]
            if istop > fp_niterations:
                fp[dataset_name].resize(istop, axis=2)
            istart = 0
            fp.create_dataset(dataset_name, (ntemps, nwalkers, istop),
                              maxshape=(ntemps, nwalkers, None),
        fp[dataset_name][:, :, istart:istop] = data

def read_raw_samples(fp, fields,
                     thin_start=None, thin_interval=None, thin_end=None,
                     iteration=None, temps='all', chains=None,
                     flatten=True, group=None):
    """Base function for reading samples from a collection of independent
    MCMC chains file with parallel tempering.

    This may collect differing numbering of samples from each chains,
    depending on the thinning settings for each chain. If not flattened the
    returned array will have dimensions requested temps x requested chains x
    max samples, where max samples is the largest number of samples retrieved
    from a single chain. Chains that retrieve fewer samples will be padded with
    ``numpy.nan``. If flattened, the NaNs are removed prior to returning.

    temps : 'all' or (list of) int, optional
        The temperature index (or list of indices) to retrieve. To retrieve
        all temperates pass 'all', or a list of all of the temperatures.
        Default is 'all'.
    chains : (list of) int, optional
        Only read from the given chains. Default is to read all.
        Remove NaNs and flatten the samples to 1D arrays before returning.
        Otherwise, the returned arrays will have shape (requested temps x
        requested chains x max requested iteration(s)), with chains that return
        fewer samples padded with NaNs. Default is True.

    Returns
    -------
    dict
    """
    # chains to load
    if chains is None:
        chains = numpy.arange(fp.nchains)
    elif not isinstance(chains, (list, numpy.ndarray)):
        chains = numpy.array([chains]).astype(int)
    get_index = _get_index(fp, chains, thin_start, thin_interval, thin_end,
                           iteration)
    # load the samples
    arrays = {}
        dset = group.format(name=name)
        # get the temperatures to load
        tidx, selecttemps, ntemps = _get_temps_index(temps, fp, dset)
        alist = []
        maxiters = 0
        for ii, cidx in enumerate(chains):
            idx = get_index[ii]
            # load the data
            thisarr = fp[dset][tidx, cidx, idx]
            if thisarr.size == 0:
                # no samples were loaded; skip this chain
                alist.append(None)
                continue
            if isinstance(idx, (int, numpy.int_)):
                # make sure the last dimension corresponds to iteration
                thisarr = thisarr.reshape(list(thisarr.shape)+[1])
            # pull out the temperatures we need
            if selecttemps:
                thisarr = thisarr[temps, ...]
            # make sure its 2D
            thisarr = thisarr.reshape(ntemps, thisarr.shape[-1])
            alist.append(thisarr)
            maxiters = max(maxiters, thisarr.shape[-1])
        # stack into a single array
        arr = numpy.full((ntemps, len(chains), maxiters), numpy.nan,
                         dtype=fp[dset].dtype)
        for ii, thisarr in enumerate(alist):
            if thisarr is not None:
                arr[:, ii, :thisarr.shape[-1]] = thisarr
            # flatten and remove nans
            arr = arr[~numpy.isnan(arr)]

                              iteration=None, temps='all', walkers=None,
                              flatten=True, group=None):
    """Base function for reading samples from ensemble MCMC file with

        returned arrays will have shape (requested temps x
        requested walkers x requested iteration(s)). Default is True.

    Returns
    -------
    dict
    """
    # load
    arrays = {}
        arr = fp[dset][tidx, widx, get_index]
        if selecttemps:
            arr = arr[temps, ...]
        else:
            # ensure that the returned array is 3D
            arr = arr.reshape((ntemps, nwalkers, niterations))

def _get_temps_index(temps, fp, dataset):
    """Convenience function to determine which temperatures to load.

    temps : 'all' or (list of) int
    dataset : str
        The name of the dataset that samples will be loaded from.

    Returns
    -------
    tidx : slice or list of int
        The temperature indices to load from the file.
    selecttemps : bool
        Whether specific temperatures need to be pulled out of the samples
        array after it is loaded from the file.
    ntemps : int
        The number of temperatures that will be loaded.
    """
    if temps == 'all':
        # all temperatures were requested; just need to know how many
        ntemps = fp[dataset].shape[0]
        tidx = slice(None, None)
        selecttemps = False
    elif isinstance(temps, (int, numpy.int_)):
        # only a single temperature is requested
        ntemps = 1
        tidx = temps
    else:
        # a select set of temperatures are requested
        ntemps = len(temps)
        selecttemps = True
    return tidx, selecttemps, ntemps

# Copyright (C) 2019 Sumit Kumar, Collin Capano
#
#

#
#
#                                   Preamble
#
#
"""Provides IO for the dynesty sampler.
"""
from .base_sampler import BaseSamplerFile
from .posterior import read_raw_samples_from_file, write_samples_to_file

class BaseNestedSamplerFile(BaseSamplerFile):
    """Class to handle file IO for the nested samplers cpnest and dynesty."""

    name = 'base_nest_file'

        return read_raw_samples_from_file(self, fields, **kwargs)

        pass

        """
        Writes the given number of iterations to the sampler group.
        """

        """
        Adds writing betas to MultiTemperedMCMCIO.
        """
        if self.sampler_group not in self.keys():
            # create the sampler group
            self.create_group(self.sampler_group)
        self[self.sampler_group].attrs['nlivepoints'] = sampler.nlive

    def write_samples(self, samples, parameters=None):
        """Writes samples to the given file.

        Results are written to ``samples_group/{vararg}``, where ``{vararg}``
        is the name of a model params. The samples are written as an
        array of length ``niterations``.

            The samples to write. Each array in the dictionary should have
            length niterations.
        parameters : list, optional
            Only write the specified parameters to the file. If None, will
            write all of the keys in the ``samples`` dict.
        """
        # since we're just writing a posterior use
        # PosteriorFile's write_samples
        write_samples_to_file(self, samples, parameters=parameters)

# Copyright (C) 2019 Collin Capano
#
#

"""Provides abstract base class for all samplers."""

import time

from .base_hdf import BaseInferenceFile

class BaseSamplerFile(BaseInferenceFile):
    """Base HDF class for all samplers.

    This adds abstract methods ``write_resume_point`` and
    ``write_sampler_metadata`` to :py:class:`BaseInferenceFile`.
    """
    def write_run_start_time(self):
        """Writes the current (UNIX) time to the file.

        Times are stored as a list in the file's ``attrs``, with name
        ``run_start_time``. If the attrbute already exists, the current time
        is appended. Otherwise, the attribute will be created and time added.
        """
        attrname = "run_start_time"
        try:
            times = self.attrs[attrname].tolist()
            times = []
        times.append(time.time())
        self.attrs[attrname] = times

    @property
    def run_start_time(self):
        """The (UNIX) time pycbc inference began running.

        If the run resumed from a checkpoint, the time the last checkpoint
        started is reported.
        """
        return self.attrs['run_start_time'][-1]

    def write_run_end_time(self):
        """"Writes the curent (UNIX) time as the ``run_end_time`` attribute.
        """
        self.attrs["run_end_time"] = time.time()

    @property
    def run_end_time(self):
        """The (UNIX) time pycbc inference finished.
        """
        return self.attrs["run_end_time"]

        """Should write the point that a sampler starts up.

        How the resume point is indexed is up to the sampler. For example,
        MCMC samplers use the number of iterations that are stored in the
        checkpoint file.
        """
        pass

        """This should write the given sampler's metadata to the file.

        This should also include the model's metadata.
        """
        pass

    def update_checkpoint_history(self):
        """Writes a copy of relevant metadata to the file's checkpoint history.

        All data are written to ``sampler_info/checkpoint_history``. If the
        group does not exist yet, it will be created.

        This function writes the current time and the time since the last
        checkpoint to the file. It will also call
        :py:func:`_update_sampler_history` to write sampler-specific history.
        """
        try:
            history = self[path]
            # assume history doesn't exist yet
        # write the checkpoint time
        current_time = time.time()
        self.write_data('checkpoint_time', current_time, path=path,
        # get the amount of time since the last checkpoint
        checkpoint_times = history['checkpoint_time'][()]
        if len(checkpoint_times) == 1:
            # this is the first checkpoint, get the run time for comparison
            lasttime = self.run_start_time
        else:
            lasttime = checkpoint_times[-2]
            # if a resume happened since the last checkpoint, use the resume
            # time instad
            if lasttime < self.run_start_time:
                lasttime = self.run_start_time
        self.write_data('checkpoint_dt', current_time-lasttime, path=path,
        # write any sampler-specific history
        self._update_sampler_history()

        """Writes sampler-specific history to the file.

        This function does nothing. Classes that inherit from it may override
        it to add any extra information they would like written. This is
        called by :py:func:`update_checkpoint_history`.
        """
        pass

    def validate(self):
        """Runs a validation test.

        This checks that a samples group exist, and that there are more than
        one sample stored to it.

        Returns
        -------
        bool :
            Whether or not the file is valid as a checkpoint file.
        """
        try:
            group = '{}/{}'.format(self.samples_group, self.variable_params[0])
            checkpoint_valid = self[group].size != 0
            checkpoint_valid = False
        return checkpoint_valid

# Copyright (C) 2019 Collin Capano, Sumit Kumar
#
#

#
#
#                                   Preamble
#
#
"""

from pycbc.io.hdf import (dump_state, load_state)

from .base_nested_sampler import BaseNestedSamplerFile
from .posterior import write_samples_to_file, read_raw_samples_from_file

class CommonNestedMetadataIO(object):
    """Provides functions for reading/writing dynesty metadata to file.
    """

        samples.
            created.
        Returns
        -------
        """

        if 'raw_samples' not in skip_args:
                "--raw-samples", action='store_true', default=False,
                help="Extract raw samples rather than a posterior. "
                     "Raw samples are the unweighted samples obtained from "
                     "the nested sampler. Default value is False, which means "
                     "raw samples are weighted by the log-weight array "
                     "obtained from the sampler, giving an estimate of the "
                     "posterior.")
        if 'seed' not in skip_args:
                "--seed", type=int, default=0,
                help="Set the random-number seed used for extracting the "
                     "posterior samples. This is needed because the "
                     "unweighted samples are randomly shuffled to produce "
                     "a posterior. Default is 0. Ignored if raw-samples are "
                     "extracted instead.")

class DynestyFile(CommonNestedMetadataIO, BaseNestedSamplerFile):
    """Class to handle file IO for the ``dynesty`` sampler."""

    name = 'dynesty_file'

    def read_raw_samples(self, fields, raw_samples=False, seed=0):
        """Reads samples from a dynesty file and constructs a posterior.

        fields : list of str
            The names of the parameters to load. Names must correspond to
            dataset names in the file's ``samples`` group.
        raw_samples : bool, optional
            Return the raw (unweighted) samples instead of the estimated
            posterior samples. Default is False.
        seed : int, optional
            When extracting the posterior, samples are randomly shuffled. To
            make this reproduceable, numpy's random generator seed is set with
            the given value prior to the extraction. Default is 0.

        Returns
        -------
        dict :
            Dictionary of parameter names -> samples.
        """
        samples = read_raw_samples_from_file(self, fields)
        logwt = read_raw_samples_from_file(self, ['logwt'])['logwt']
        loglikelihood = read_raw_samples_from_file(
            self, ['loglikelihood'])['loglikelihood']
        logz = self.attrs.get('log_evidence')
        if not raw_samples:
            weights = numpy.exp(logwt - logz)
            N = len(weights)
            positions = (numpy.random.random() + numpy.arange(N)) / N
            idx = numpy.zeros(N, dtype=numpy.int)
            cumulative_sum = numpy.cumsum(weights)
            cumulative_sum /= cumulative_sum[-1]
            i, j = 0, 0
            while i < N:
                if positions[i] < cumulative_sum[j]:
                    idx[i] = j
                    i += 1
                else:
                    j += 1
            try:
                rng = numpy.random.default_rng(seed)
                # numpy pre-1.17 uses RandomState
                # Py27: delete this after we drop python 2.7 support
                rng = numpy.random.RandomState(seed)
            rng.shuffle(idx)
            post = {'loglikelihood': loglikelihood[idx]}
            for i, param in enumerate(fields):
                post[param] = samples[param][idx]
            return post
        else:
            return samples

    def write_pickled_data_into_checkpoint_file(self, state):
        """Dump the sampler state into checkpoint file
        """
        if 'sampler_info/saved_state' not in self:
            self.create_group('sampler_info/saved_state')
        dump_state(state, self, path='sampler_info/saved_state')

    def read_pickled_data_from_checkpoint_file(self):
        """Load the sampler state (pickled) from checkpoint file
        """
        return load_state(self, path='sampler_info/saved_state')

    def write_raw_samples(self, data, parameters=None):
        """Write the nested samples to the file
        """
        if 'samples' not in self:
            self.create_group('samples')
        write_samples_to_file(self, data, parameters=parameters,
                              group='samples')

           This checks that a samples group exist, and that pickeled data can
           be loaded.
        Returns
        -------
        bool :
        """
        try:
            if 'sampler_info/saved_state' in self:
                load_state(self, path='sampler_info/saved_state')
            checkpoint_valid = True

#
#

#
#
#                                   Preamble
#
#
"""Provides IO for the emcee sampler.
"""

from .base_mcmc import (EnsembleMCMCMetadataIO, CommonMCMCMetadataIO,
                        write_samples, ensemble_read_raw_samples)

class EmceeFile(EnsembleMCMCMetadataIO, CommonMCMCMetadataIO, BaseSamplerFile):
    """Class to handle file IO for the ``emcee`` sampler."""

    name = 'emcee_file'

        r"""Writes samples to the given file.

        Calls :py:func:`base_mcmc.write_samples`. See that function for
        details.

            shape nwalkers x niterations.
            All other keyword arguments are passed to
            :py:func:`base_mcmc.write_samples`.
        """
        write_samples(self, samples, **kwargs)

        r"""Base function for reading samples.

        Calls :py:func:`base_mcmc.ensemble_read_raw_samples`. See that function
        for details.

        fields : list
            The list of field names to retrieve.
            :py:func:`base_mcmc.ensemble_read_raw_samples`.

        Returns
        -------
        dict
            A dictionary of field name -> numpy array pairs.
        """
        return ensemble_read_raw_samples(self, fields, **kwargs)

    def read_acceptance_fraction(self, walkers=None):
        """Reads the acceptance fraction.

        walkers : (list of) int, optional
            The walker index (or a list of indices) to retrieve. If None,
            samples from all walkers will be obtained.

        Returns
        -------
        array
            Array of acceptance fractions with shape (requested walkers,).
        """
        group = self.sampler_group + '/acceptance_fraction'
        if walkers is None:
            wmask = numpy.ones(self.nwalkers, dtype=bool)
        else:
            wmask = numpy.zeros(self.nwalkers, dtype=bool)
            wmask[walkers] = True
        return self[group][wmask]

    def write_acceptance_fraction(self, acceptance_fraction):
        """Write acceptance_fraction data to file. Results are written to
        the ``[sampler_group]/acceptance_fraction``.

        acceptance_fraction : numpy.ndarray
            Array of acceptance fractions to write.
        """
        try:
            self[group][:] = acceptance_fraction
            # dataset doesn't exist yet, create it
            self[group] = acceptance_fraction

#
#

"""Provides I/O support for emcee_pt.
"""

from .base_mcmc import EnsembleMCMCMetadataIO
from .base_multitemper import (CommonMultiTemperedMetadataIO,
                               write_samples,
                               ensemble_read_raw_samples)

class EmceePTFile(EnsembleMCMCMetadataIO, CommonMultiTemperedMetadataIO,
                  BaseSamplerFile):

    name = 'emcee_pt_file'

    @property
    def betas(self):
        """The betas that were used."""
        return self[self.sampler_group].attrs["betas"]

        Calls :py:func:`base_multitemper.write_samples`. See that function for
        details.

            shape ntemps x nwalkers x niterations.
            :py:func:`base_multitemper.write_samples`.
        """

        Calls :py:func:`base_multitemper.ensemble_read_raw_samples`. See that
        function for details.

            :py:func:`base_multitemper.ensemble_read_raw_samples`.

        Returns
        -------
        dict
        """

        """Adds writing betas to MultiTemperedMCMCIO.
        """
        super(EmceePTFile, self).write_sampler_metadata(sampler)
        self[self.sampler_group].attrs["betas"] = sampler.betas

    def read_acceptance_fraction(self, temps=None, walkers=None):

        temps : (list of) int, optional
            The temperature index (or a list of indices) to retrieve. If None,
            acfs from all temperatures and all walkers will be retrieved.

        Returns
        -------
        array
            Array of acceptance fractions with shape (requested temps,
            requested walkers).
        """
        else:
        if temps is None:
            tmask = numpy.ones(self.ntemps, dtype=bool)
        else:
            tmask = numpy.zeros(self.ntemps, dtype=bool)
            tmask[temps] = True
        return self[group][:][numpy.ix_(tmask, wmask)]

        """Write acceptance_fraction data to file.

        Results are written to ``[sampler_group]/acceptance_fraction``; the
        resulting dataset has shape (ntemps, nwalkers).

            Array of acceptance fractions to write. Must have shape
            ntemps x nwalkers.
        """
        # check
        assert acceptance_fraction.shape == (self.ntemps, self.nwalkers), (
            "acceptance fraction must have shape ntemps x nwalker")
        try:

# Copyright (C) 2019  Collin Capano
#
#

"""This module provides IO classes for epsie samplers.
"""

from pickle import UnpicklingError
from epsie import load_state

from .base_mcmc import MCMCMetadataIO
                               read_raw_samples)

class EpsieFile(MCMCMetadataIO, CommonMultiTemperedMetadataIO,
                BaseSamplerFile):
    """Class to handle IO for Epsie's parallel-tempered sampler."""

    name = 'epsie_file'

    @property
        """Alias for nwalkers."""
        return self.nwalkers

    @property
        return self[self.sampler_group]['betas'][()]

    @property
    def swap_interval(self):
        """The interval that temperature swaps occurred at."""
        return self[self.sampler_group].attrs['swap_interval']

    @swap_interval.setter
    def swap_interval(self, swap_interval):
        """Stores the swap interval to the sampler group's attrs."""
        self[self.sampler_group].attrs['swap_interval'] = swap_interval

        """Adds writing seed and betas to MultiTemperedMCMCIO.
        """
        super(EpsieFile, self).write_sampler_metadata(sampler)
        self[self.sampler_group].attrs['seed'] = sampler.seed
        self.write_data("betas", sampler.betas, path=self.sampler_group)

        Also thins the acceptance ratio and the temperature data, both of
        which are stored in the ``sampler_info`` group.
        """
        # We'll need to know what the new interval to thin by will be
        # so we can properly thin the acceptance ratio and temperatures swaps.
        # We need to do this before calling the base thin, as we need to know
        # what the current thinned by is.
        new_interval = thin_interval // self.thinned_by
        # now thin the samples
        super(EpsieFile, self).thin(thin_interval)
        # thin the acceptance ratio
        self._thin_data(self.sampler_group, ['acceptance_ratio'],
                        new_interval)
        # thin the temperature swaps; since these may not happen every
        # iteration, the thin interval we use for these is different
        ts_group = '/'.join([self.sampler_group, 'temperature_swaps'])
        ts_thin_interval = new_interval // self.swap_interval
        if ts_thin_interval > 1:
            self._thin_data(ts_group, ['swap_index'],
                            ts_thin_interval)
            self._thin_data(ts_group, ['acceptance_ratio'],

        details.

        """

        Calls :py:func:`base_multitemper.read_raw_samples`. See that

            :py:func:`base_multitemper.read_raw_samples`.

        Returns
        -------
        dict
        """
        return read_raw_samples(self, fields, **kwargs)

    def write_acceptance_ratio(self, acceptance_ratio, last_iteration=None):
        """Writes the acceptance ratios to the sampler info group.

        acceptance_ratio : array
            The acceptance ratios to write. Should have shape
            ``ntemps x nchains x niterations``.
        """
        # we'll use the write_samples machinery to write the acceptance ratios
        self.write_samples({'acceptance_ratio': acceptance_ratio},
                           last_iteration=last_iteration,
                           samples_group=self.sampler_group)

    def read_acceptance_ratio(self, temps=None, chains=None):
        """Reads the acceptance ratios.

        Ratios larger than 1 are set back to 1 before returning.

            acceptance ratios from all temperatures and all chains will be
            retrieved.
        chains : (list of) int, optional
            The chain index (or a list of indices) to retrieve. If None,
            ratios from all chains will be obtained.

        Returns
        -------
        array
            Array of acceptance ratios with shape (requested temps,
            requested chains, niterations).
        """
        group = self.sampler_group + '/acceptance_ratio'
        if chains is None:
            wmask = numpy.ones(self.nchains, dtype=bool)
        else:
            wmask = numpy.zeros(self.nchains, dtype=bool)
            wmask[chains] = True
        else:
        all_ratios = self[group][:]
        # make sure values > 1 are set back to 1
        all_ratios[all_ratios > 1] = 1.
        return all_ratios[numpy.ix_(tmask, wmask)]

    def read_acceptance_rate(self, temps=None, chains=None):
        """Reads the acceptance rate.

        This calls :py:func:`read_acceptance_ratio`, then averages the ratios
        over all iterations to get the average rate.

            acceptance rates from all temperatures and all chains will be
            rates from all chains will be obtained.

        Returns
        -------
        array
            requested chains).
        """
        all_ratios = self.read_acceptance_ratio(temps, chains)
        # average over the number of iterations
        all_ratios = all_ratios.mean(axis=-1)
        return all_ratios

        """Alias for :py:func:`read_acceptance_rate`.
        """
        return self.read_acceptance_rate(temps=temps, chains=walkers)

    def write_temperature_data(self, swap_index, acceptance_ratio,
                               swap_interval, last_iteration):
        """Writes temperature swaps and acceptance ratios.

        swap_index : array
            The indices indicating which temperatures were swapped. Should have
            shape ``ntemps x nchains x (niterations/swap_interval)``.
            The array of acceptance ratios between temperatures. Should
            have shape ``(ntemps-1) x nchains x (niterations/swap_interval)``.
            arrays.
        swap_interval : int
            The number of iterations between temperature swaps.
        last_iteration : int
            The iteration of the last sample.
        """
        self.swap_interval = swap_interval
        group = '/'.join([self.sampler_group, 'temperature_swaps'])
        # we'll use the write_samples machinery to write the acceptance ratios;
        # if temperature swaps didn't happen every iteration, then a smaller
        # thinning interval than what is used for the samples should be used
        thin_by = self.thinned_by // swap_interval
        # we'll also tell the write samples that the last "iteration" is the
        # last iteration / the swap interval, to get the spacing correct
        last_iteration = last_iteration // swap_interval
        # we need to write the two arrays separately, since they have different
        # dimensions in temperature
        self.write_samples({'swap_index': swap_index},
                           samples_group=group, thin_by=thin_by)

        """Adds attemp to load checkpoint to validation test."""
        valid = super(EpsieFile, self).validate()
        # try to load the checkpoint
        if valid:
            try:
                load_state(self, self.sampler_group)
            except (KeyError, UnpicklingError):
                # will get this if the state wasn't written, or it was
                # corrupted for some reason
                valid = False
        return valid

        # need this to make sure options called "walkers" are renamed to
        # "chains"
        parsed = BaseSamplerFile._get_optional_args(
            args, opts, err_on_missing=err_on_missing, **kwargs)
        try:
            chains = parsed.pop('walkers')
            parsed['chains'] = chains
            pass

#
#

"""Provides I/O support for multinest.
"""

class MultinestFile(BaseSamplerFile):
    """Class to handle file IO for the ``multinest`` sampler."""

    name = 'multinest_file'

        """
        niterations = len(tuple(samples.values())[0])
        assert all(len(p) == niterations for p in samples.values()), (
            "all samples must have the same shape")
        group = self.samples_group + '/{name}'
            parameters = samples.keys()
        # loop over number of dimensions
            dataset_name = group.format(name=param)
            try:
                fp_niterations = len(self[dataset_name])
                if niterations != fp_niterations:
                    # resize the dataset
                    self[dataset_name].resize(niterations, axis=0)
                self.create_dataset(dataset_name, (niterations,),
                                    maxshape=(None,),
                                    dtype=samples[param].dtype,
                                    fletcher32=True)
            self[dataset_name][:] = samples[param]

    def write_logevidence(self, lnz, dlnz, importance_lnz, importance_dlnz):

        attributes, as well as the importance-weighted versions of these
        stats if they exist.

        lnz : float
        importance_lnz : float, optional
            The importance-weighted log of the evidence.
        importance_dlnz : float, optional
            The error in the importance-weighted estimate of the log evidence.
        """
        if all([e is not None for e in [importance_lnz, importance_dlnz]]):
            self.attrs['importance_log_evidence'] = importance_lnz
            self.attrs['importance_dlog_evidence'] = importance_dlnz

    def read_raw_samples(self, fields, iteration=None):
        if isinstance(fields, string_types):
            fields = [fields]
        # load
        arrays = {}
        for name in fields:
            if iteration is not None:
                arr = self[group.format(name=name)][int(iteration)]
            else:
                arr = self[group.format(name=name)][:]
            arrays[name] = arr
        return arrays

        try:
        try:

    @property

        self[self.sampler_group].attrs['nlivepoints'] = sampler.nlivepoints

#
#

#
#
#                                   Preamble
#
#
"""Provides simplified standard format just for posterior data
"""

class PosteriorFile(BaseInferenceFile):
    """Class to handle file IO for the simplified Posterior file."""

    name = 'posterior_file'

        return write_samples_to_file(self, samples, parameters=parameters)

def read_raw_samples_from_file(fp, fields, **kwargs):
    samples = fp[fp.samples_group]
    return {field: samples[field][:] for field in fields}

def write_samples_to_file(fp, samples, parameters=None, group=None):

    array of length ``niterations``.

    fp : self
        Pass the 'self' from BaseInferenceFile class.
        length niterations.
        """
    # check data dimensions; we'll just use the first array in samples
    arr = list(samples.values())[0]
    if not arr.ndim == 1:
        raise ValueError("samples must be 1D arrays")
    niterations = arr.size
    assert all(len(p) == niterations
        "all samples must have the same shape")
    if group is not None:
        group = group + '/{name}'
    else:
        group = fp.samples_group + '/{name}'
        try:
            fp_niterations = len(fp[dataset_name])
            if niterations != fp_niterations:
                fp[dataset_name].resize(niterations, axis=0)
            fp.create_dataset(dataset_name, (niterations,),
                              maxshape=(None,),
                              dtype=samples[param].dtype,
        fp[dataset_name][:] = samples[param]

# Copyright (C) 2020 Collin Capano
#
#

"""Provides I/O support for ptemcee.
"""

from . import base_mcmc

class PTEmceeFile(EnsembleMCMCMetadataIO, CommonMultiTemperedMetadataIO,
    """Class to handle file IO for the ``ptemcee`` sampler."""

    name = 'ptemcee_file'

    # attributes for setting up an ensemble from file
    _ensemble_attrs = ['jumps_proposed', 'jumps_accepted', 'swaps_proposed',
                       'swaps_accepted', 'logP', 'logl']

        """Adds writing ptemcee-specific metadata to MultiTemperedMCMCIO.
        """
        super(PTEmceeFile, self).write_sampler_metadata(sampler)
        group = self[self.sampler_group]
        group.attrs["starting_betas"] = sampler.starting_betas
        group.attrs["adaptive"] = sampler.adaptive
        group.attrs["adaptation_lag"] = sampler.adaptation_lag
        group.attrs["adaptation_time"] = sampler.adaptation_time
        group.attrs["scale_factor"] = sampler.scale_factor

    @property
    def starting_betas(self):
        """The starting betas that were used."""
        return self[self.sampler_group].attrs["starting_betas"]

    def write_betas(self, betas, last_iteration=None):
        """Writes the betas to sampler group.

        As the betas may change with iterations, this writes the betas as
        a ntemps x niterations array to the file.
        """
        # we'll use the single temperature write_samples to write the betas,
        # so that we get the thinning settings
        base_mcmc.write_samples(self, {'betas': betas},
                                last_iteration=last_iteration,
                                samples_group=self.sampler_group)

    def read_betas(self, thin_start=None, thin_interval=None, thin_end=None,
                   iteration=None):
        """Reads betas from the file.

        thin_start : int, optional
            Start reading from the given iteration. Default is to start from
            the first iteration.
        thin_interval : int, optional
            Only read every ``thin_interval`` -th sample. Default is 1.
        thin_end : int, optional
            Stop reading at the given iteration. Default is to end at the last
            iteration.
        iteration : int, optional
            Only read the given iteration. If this provided, it overrides
            the ``thin_(start|interval|end)`` options.

        Returns
        -------
        array
            A ntemps x niterations array of the betas.
        """
        slc = base_mcmc._ensemble_get_index(self, thin_start=thin_start,
                                            thin_interval=thin_interval,
                                            thin_end=thin_end,
                                            iteration=iteration)
        betas = self[self.sampler_group]['betas'][:]
        return betas[:, slc]

    def write_ensemble_attrs(self, ensemble):
        """Writes ensemble attributes necessary to restart from checkpoint.

        Paramters
        ---------
        ensemble : ptemcee.Ensemble
            The ensemble to write attributes for.
        """
        for attr in self._ensemble_attrs:
            vals = getattr(ensemble, attr)
            try:
                group[attr][:] = vals
                group[attr] = vals

    def read_ensemble_attrs(self):
        """Reads ensemble attributes from the file.

        Returns
        -------
        dict :
            Dictionary of the ensemble attributes.
        """
        return {attr: group[attr][:] for attr in self._ensemble_attrs}

        details.

        """

        Returns
        -------
        dict
        """

#
#
""" This modules defines functions for reading and samples that the
inference samplers generate and are stored in an ASCII TXT file.
"""

class InferenceTXTFile(object):
    """ A class that has extra functions for handling reading the samples
    from posterior-only TXT files.

    path : str
        The path to the TXT file.
        The mode to open the file. Only accepts "r" or "rb" for reading.
    delimiter : str
        Delimiter to use for TXT file. Default is space-delimited.
    """
    name = "txt"
    comments = ""
    delimiter = " "

    def __init__(self, path, mode=None, delimiter=None):
        self.path = path
        self.delimiter = delimiter if delimiter is not None else self.delimiter
        if mode in ["r", "rb"]:
            self.mode = mode
        else:
            raise ValueError("Mode for InferenceTXTFile must be 'r' or 'rb'.")

    def write(cls, output_file, samples, labels, delimiter=None):
        """ Writes a text file with samples.

        output_file : str
            The path of the file to write.
        samples : FieldArray
            Samples to write to file.
        labels : list
            A list of strings to include as header in TXT file.
        delimiter : str
            Delimiter to use in TXT file.
        """
        delimiter = delimiter if delimiter is not None else cls.delimiter
        header = delimiter.join(labels)
        numpy.savetxt(output_file, samples,
                      comments=cls.comments, header=header,
                      delimiter=delimiter)

# Copyright (C) 2019 Collin Capano, Sumit Kumar, Alex Nitz
#
#

#
#
#                                   Preamble
#
#
"""Provides IO for the ultranest sampler.
"""

class UltranestFile(BaseNestedSamplerFile):
    """Class to handle file IO for the ``ultranest`` sampler."""

    name = 'ultranest_file'

# Copyright (C) 2018  Collin Capano
#
#
#

"""I/O utilities for pycbc inference
"""

import os
import shutil
import textwrap
import h5py as _h5py
from pycbc.io.record import (FieldArray, _numpy_function_lib)
from pycbc import waveform as _waveform

from pycbc.inference.option_utils import (ParseLabelArg, ParseParametersArg)
from .emcee import EmceeFile
from .emcee_pt import EmceePTFile
from .ptemcee import PTEmceeFile
from .cpnest import CPNestFile
from .multinest import MultinestFile
from .dynesty import DynestyFile
from .ultranest import UltranestFile
from .posterior import PosteriorFile
from .txt import InferenceTXTFile

filetypes = {
    EmceeFile.name: EmceeFile,
    EmceePTFile.name: EmceePTFile,
    PTEmceeFile.name: PTEmceeFile,
    CPNestFile.name: CPNestFile,
    MultinestFile.name: MultinestFile,
    DynestyFile.name: DynestyFile,
    PosteriorFile.name: PosteriorFile,
    UltranestFile.name: UltranestFile,
}

try:
    from .epsie import EpsieFile
    filetypes[EpsieFile.name] = EpsieFile
    pass

def get_file_type(filename):
    """ Returns I/O object to use for file.

        Name of file.

    Returns
    -------
    file_type : {InferenceFile, InferenceTXTFile}
        The type of inference file object to use.
    """
    txt_extensions = [".txt", ".dat", ".csv"]
    hdf_extensions = [".hdf", ".h5", ".bkup", ".checkpoint"]
    for ext in hdf_extensions:
        if filename.endswith(ext):
            with _h5py.File(filename, 'r') as fp:
                filetype = fp.attrs['filetype']
                try:
                    filetype = str(filetype.decode())
                except AttributeError:
                    pass
            return filetypes[filetype]
    for ext in txt_extensions:
            return InferenceTXTFile
    raise TypeError("Extension is not supported.")

def loadfile(path, mode=None, filetype=None, **kwargs):
    """Loads the given file using the appropriate InferenceFile class.

    If ``filetype`` is not provided, this will try to retreive the ``filetype``
    from the file's ``attrs``. If the file does not exist yet, an IOError will
    be raised if ``filetype`` is not provided.

    path : str
        The filename to load.
    mode : str, optional
        What mode to load the file with, e.g., 'w' for write, 'r' for read,
        'a' for append. Default will default to h5py.File's mode, which is 'a'.
    filetype : str, optional
        Force the file to be loaded with the given class name. This must be
        provided if creating a new file.

    Returns
    -------
    filetype instance
        An open file handler to the file. The class used for IO with the file
        is determined by the ``filetype`` keyword (if provided) or the
        ``filetype`` stored in the file (if not provided).
    """
    if filetype is None:
        # try to read the file to get its filetype
        try:
            fileclass = get_file_type(path)
            # file doesn't exist, filetype must be provided
            raise IOError("The file appears not to exist. In this case, "
                          "filetype must be provided.")
    else:
        fileclass = filetypes[filetype]
    return fileclass(path, mode=mode, **kwargs)

#
#
#                         HDF Utilities
#
#

def check_integrity(filename):
    """Checks the integrity of an InferenceFile.

    Checks done are:

        * can the file open?
        * do all of the datasets in the samples group have the same shape?
        * can the first and last sample in all of the datasets in the samples
          group be read?

    If any of these checks fail, an IOError is raised.

    filename: str
        Name of an InferenceFile to check.

    Raises
    ------
    ValueError
        If the given file does not exist.
    KeyError
        If the samples group does not exist.
    IOError
        If any of the checks fail.
    """
    # check that the file exists
    if not os.path.exists(filename):
        raise ValueError("file {} does not exist".format(filename))
    # if the file is corrupted such that it cannot be opened, the next line
    # will raise an IOError
    with loadfile(filename, 'r') as fp:
        # check that all datasets in samples have the same shape
        parameters = list(fp[fp.samples_group].keys())
        # but only do the check if parameters have been written
        if len(parameters) > 0:
            group = fp.samples_group + '/{}'
            # use the first parameter as a reference shape
            ref_shape = fp[group.format(parameters[0])].shape
            if not all(fp[group.format(param)].shape == ref_shape
                       for param in parameters):
                raise IOError("not all datasets in the samples group have "
                              "the same shape")
            # check that we can read the first/last sample
            firstidx = tuple([0]*len(ref_shape))
            lastidx = tuple([-1]*len(ref_shape))
            _ = fp[group.format(param)][firstidx]
            _ = fp[group.format(param)][lastidx]

def validate_checkpoint_files(checkpoint_file, backup_file,
                              check_nsamples=True):
    """Checks if the given checkpoint and/or backup files are valid.

    The checkpoint file is considered valid if:

        * it passes all tests run by ``check_integrity``;
        * it has at least one sample written to it (indicating at least one
          checkpoint has happened).

    The same applies to the backup file. The backup file must also have the
    same number of samples as the checkpoint file, otherwise, the backup is
    considered invalid.

    If the checkpoint (backup) file is found to be valid, but the backup
    (checkpoint) file is not valid, then the checkpoint (backup) is copied to
    the backup (checkpoint). Thus, this function ensures that checkpoint and
    backup files are either both valid or both invalid.

    checkpoint_file : string
        Name of the checkpoint file.
    backup_file : string
        Name of the backup file.

    Returns
    -------
    checkpoint_valid : bool
        Whether or not the checkpoint (and backup) file may be used for loading
        samples.
    """
    # check if checkpoint file exists and is valid
    try:
        check_integrity(checkpoint_file)
        checkpoint_valid = True
    except (ValueError, KeyError, IOError):
        checkpoint_valid = False

    # backup file
    try:
        check_integrity(backup_file)
        backup_valid = True
        backup_valid = False

    # since we can open the file, run self diagnostics
    if checkpoint_valid:
        with loadfile(checkpoint_file, 'r') as fp:
            checkpoint_valid = fp.validate()
    if backup_valid:
        with loadfile(backup_file, 'r') as fp:
            backup_valid = fp.validate()
    if check_nsamples:
        # This check is not required by nested samplers
        # check that the checkpoint and backup have the same number of samples;
        # if not, assume the checkpoint has the correct number
        if checkpoint_valid and backup_valid:
            with loadfile(checkpoint_file, 'r') as fp:
                group = list(fp[fp.samples_group].keys())[0]
                nsamples = fp[fp.samples_group][group].size
            with loadfile(backup_file, 'r') as fp:
                backup_nsamples = fp[fp.samples_group][group].size
            backup_valid = nsamples == backup_nsamples
        # decide what to do based on the files' statuses
    if checkpoint_valid and not backup_valid:
        # copy the checkpoint to the backup
        logging.info("Backup invalid; copying checkpoint file")
        shutil.copy(checkpoint_file, backup_file)
    elif backup_valid and not checkpoint_valid:
        logging.info("Checkpoint invalid; copying backup file")
        # copy the backup to the checkpoint
        shutil.copy(backup_file, checkpoint_file)
    return checkpoint_valid

#
#
#                         Command-line Utilities
#
#
def get_common_parameters(input_files, collection=None):
    """Gets a list of variable params that are common across all input files.

    If no common parameters are found, a ``ValueError`` is raised.

    input_files : list of str
        List of input files to load.
    collection : str, optional
        What group of parameters to load. Can be the name of a list of
        parameters stored in the files' attrs (e.g., "variable_params"), or
        "all". If "all", will load all of the parameters in the files'
        samples group. Default is to load all.

    Returns
    -------
    list :
        List of the parameter names.
    """
    if collection is None:
        collection = "all"
    parameters = []
    for fn in input_files:
        fp = loadfile(fn, 'r')
        if collection == 'all':
            ps = fp[fp.samples_group].keys()
        else:
            ps = fp.attrs[collection]
        parameters.append(set(ps))
    parameters = list(set.intersection(*parameters))
    if parameters == []:
        raise ValueError("no common parameters found for collection {} in "
                         "files {}".format(collection, ', '.join(input_files)))
    # if using python 3 to read a file created in python 2, need to convert
    # parameters to strs
    try:
        parameters = [p.decode() for p in parameters]
        pass
    return parameters

class NoInputFileError(Exception):
    """Raised in custom argparse Actions by arguments needing input-files when
    no file(s) were provided."""
    pass

class PrintFileParams(argparse.Action):
    """Argparse action that will load input files and print possible parameters
    to screen. Once this is done, the program is forced to exit immediately.

    The behvior is similar to --help, except that the input-file is read.

    .. note::
        The ``input_file`` attribute must be set in the parser namespace before
        this action is called. Otherwise, a ``NoInputFileError`` is raised.
    """
    def __init__(self, skip_args=None, nargs=0, **kwargs):
        if nargs != 0:
            raise ValueError("nargs for this action must be 0")
        super(PrintFileParams, self).__init__(nargs=nargs, **kwargs)
        self.skip_args = skip_args

        # get the input file(s)
        input_files = namespace.input_file
        if input_files is None:
            # see if we should raise an error
            try:
                raise_err = not parser.no_input_file_err
                raise_err = True
            if raise_err:
                raise NoInputFileError("must provide at least one input file")
            else:
                # just return to stop further processing
                return
        filesbytype = {}
        fileparsers = {}
        for fn in input_files:
            fp = loadfile(fn, 'r')
            try:
                filesbytype[fp.name].append(fn)
                filesbytype[fp.name] = [fn]
                # get any extra options
                fileparsers[fp.name], _ = fp.extra_args_parser(
                    skip_args=self.skip_args, add_help=False)
        # now print information about the intersection of all parameters
        parameters = get_common_parameters(input_files, collection='all')
        print("\n"+textwrap.fill("Parameters available with this (these) "
                                 "input file(s):"), end="\n\n")
        print(textwrap.fill(' '.join(sorted(parameters))),
              end="\n\n")
        # information about the pycbc functions
        pfuncs = sorted(FieldArray.functionlib.fget(FieldArray).keys())
        print(textwrap.fill("Available pycbc functions (see "
                            "http://pycbc.org/pycbc/latest/html for "
                            "more details):"), end="\n\n")
        print(textwrap.fill(', '.join(pfuncs)), end="\n\n")
        # numpy funcs
        npfuncs = sorted([name for (name, obj) in _numpy_function_lib.items()
                          if isinstance(obj, numpy.ufunc)])
        print(textwrap.fill("Available numpy functions:"),
        print(textwrap.fill(', '.join(npfuncs)), end="\n\n")
        # misc
        consts = "e euler_gamma inf nan pi"
        print(textwrap.fill("Recognized constants:"),
        print(consts, end="\n\n")
        print(textwrap.fill("Python arthimetic (+ - * / // ** %), "
                            "binary (&, |, etc.), and comparison (>, <, >=, "
                            "etc.) operators may also be used."), end="\n\n")
        # print out the extra arguments that may be used
        outstr = textwrap.fill("The following are additional command-line "
                               "options that may be provided, along with the "
                               "input files that understand them:")
        print("\n"+outstr, end="\n\n")
        for ftype, fparser in fileparsers.items():
            fnames = ', '.join(filesbytype[ftype])
            if fparser is None:
                outstr = textwrap.fill(
                    "File(s) {} use no additional options.".format(fnames))
                print(outstr, end="\n\n")
            else:
                fparser.usage = fnames
                fparser.print_help()
        parser.exit(0)

class ResultsArgumentParser(argparse.ArgumentParser):
    """Wraps argument parser, and preloads arguments needed for loading samples
    from a file.

    This parser class should be used by any program that wishes to use the
    standard arguments for loading samples. It provides functionality to parse
    file specific options. These file-specific arguments are not included in
    the standard ``--help`` (since they depend on what input files are given),
    but can be seen by running ``--file-help/-H``. The ``--file-help`` will
    also print off information about what parameters may be used given the
    input files.

    As with the standard ``ArgumentParser``, running this class's
    ``parse_args`` will result in an error if arguments are provided that are
    not recognized by the parser, nor by any of the file-specific arguments.
    For example, ``parse_args`` would work on the command
    ``--input-file results.hdf --walker 0`` if
    ``results.hdf`` was created by a sampler that recognizes a ``--walker``
    argument, but would raise an error if ``results.hdf`` was created by a
    sampler that does not recognize a ``--walker`` argument. The extra
    arguments that are recognized are determined by the sampler IO class's
    ``extra_args_parser``.

    Some arguments may be excluded from the parser using the ``skip_args``
    optional parameter.

    skip_args : list of str, optional
        Do not add the given arguments to the parser. Arguments should be
        specified as the option string minus the leading '--'; e.g.,
        ``skip_args=['thin-start']`` would cause the ``thin-start`` argument
        to not be included. May also specify sampler-specific arguments. Note
        that ``input-file``, ``file-help``, and ``parameters`` are always
        added.
    defaultparams : {'variable_params', 'all'}, optional
        If no ``--parameters`` provided, which collection of parameters to
        load. If 'all' will load all parameters in the file's
        ``samples_group``. If 'variable_params' or None (the default) will load
        the variable parameters.
    autoparamlabels : bool, optional
        Passed to ``add_results_option_group``; see that function for details.
        All other keyword arguments are passed to ``argparse.ArgumentParser``.
    """
    def __init__(self, skip_args=None, defaultparams=None,
                 autoparamlabels=True, **kwargs):
        super(ResultsArgumentParser, self).__init__(**kwargs)
        # add attribute to communicate to arguments what to do when there is
        # no input files
        self.no_input_file_err = False
        if defaultparams is None:
            defaultparams = 'variable_params'
        self.defaultparams = defaultparams
        # add the results option grup
        self.add_results_option_group(autoparamlabels=autoparamlabels)

    @property
    def actions(self):
        """Exposes the actions this parser can do as a dictionary.

        The dictionary maps the ``dest`` to actions.
        """
        return {act.dest: act for act in self._actions}

    def _unset_required(self):
        """Convenience function to turn off required arguments for first parse.
        """
        self._required_args = [act for act in self._actions if act.required]
        for act in self._required_args:
            act.required = False

    def _reset_required(self):
        """Convenience function to turn required arguments back on.
        """
            act.required = True

    def parse_known_args(self, args=None, namespace=None):
        """Parse args method to handle input-file dependent arguments."""
        # run parse args once to make sure the name space is populated with
        # the input files. We'll turn off raising NoInputFileErrors on this
        # pass
        self.no_input_file_err = True
        self._unset_required()
        opts, extra_opts = super(ResultsArgumentParser, self).parse_known_args(
            args, namespace)
        # now do it again
        self._reset_required()
            args, opts)
        # populate the parameters option if it wasn't specified
        if opts.parameters is None or opts.parameters == ['*']:
            parameters = get_common_parameters(opts.input_file,
                                               collection=self.defaultparams)
            # now call parse parameters action to re-populate the namespace
            self.actions['parameters'](self, opts, parameters)
        # check if we're being greedy or not
        elif '*' in opts.parameters:
            # remove the * from the parameters and the labels
            opts.parameters = [p for p in opts.parameters if p != '*']
            opts.parameters_labels.pop('*', None)
            # add the rest of the parameters not used
            all_params = get_common_parameters(opts.input_file,
            # extract the used parameters from the parameters option
            used_params = FieldArray.parse_parameters(opts.parameters,
                                                      all_params)
            add_params = set(all_params) - set(used_params)
            # repopulate the name space with the additional parameters
            if add_params:
                opts.parameters += list(add_params)
                # update the labels
                opts.parameters_labels.update({p: p for p in add_params})
        # parse the sampler-specific options and check for any unknowns
        unknown = []
        for fn in opts.input_file:
            sampler_parser, _ = fp.extra_args_parser(skip_args=self.skip_args)
            if sampler_parser is not None:
                opts, still_unknown = sampler_parser.parse_known_args(
                    extra_opts, namespace=opts)
                unknown.append(set(still_unknown))
        # the intersection of the unknowns are options not understood by
        # any of the files
        if len(unknown) > 0:
            unknown = set.intersection(*unknown)
        return opts, list(unknown)

    def add_results_option_group(self, autoparamlabels=True):
        """Adds the options used to call pycbc.inference.io.results_from_cli
        function to the parser.

        These are options releated to loading the results from a run of
        pycbc_inference, for purposes of plotting and/or creating tables.

        Any argument strings included in the ``skip_args`` attribute will not
        be added.

        autoparamlabels : bool, optional
            If True, the ``--parameters`` option will use labels from
            ``waveform.parameters`` if a parameter name is the same as a
            parameter there. Otherwise, will just use whatever label is
            provided. Default is True.
        """
        results_reading_group = self.add_argument_group(
            title="Arguments for loading results",
            description="Additional, file-specific arguments may also be "
            "provided, depending on what input-files are given. See "
            "--file-help for details.")
        results_reading_group.add_argument(
            "--input-file", type=str, required=True, nargs="+",
            action=ParseLabelArg, metavar='FILE[:LABEL]',
            help="Path to input HDF file(s). A label may be specified for "
                 "each input file to use for plots when multiple files are "
                 "specified.")
        # advanced help
            "-H", "--file-help",
            action=PrintFileParams, skip_args=self.skip_args,
            help="Based on the provided input-file(s), print all available "
                 "parameters that may be retrieved and all possible functions "
                 "on those parameters. Also print available additional "
                 "arguments that may be passed. This option is like an "
                 "advanced --help: if run, the program will just print the "
                 "information to screen, then exit.")
        if autoparamlabels:
            paramparser = ParseParametersArg
            lblhelp = (
                "If LABEL is the same as a parameter in "
                "pycbc.waveform.parameters, the label "
                "property of that parameter will be used (e.g., if LABEL "
                "were 'mchirp' then {} would be used). "
                .format(_waveform.parameters.mchirp.label))
        else:
            paramparser = ParseLabelArg
            lblhelp = ''
            "--parameters", type=str, nargs="+", metavar="PARAM[:LABEL]",
            action=paramparser,
            help="Name of parameters to load; default is to load all. The "
                 "parameters can be any of the model params or posterior "
                 "stats (loglikelihood, logprior, etc.) in the input file(s), "
                 "derived parameters from them, or any function of them. If "
                 "multiple files are provided, any parameter common to all "
                 "files may be used. Syntax for functions is python; any math "
                 "functions in the numpy libary may be used. Can optionally "
                 "also specify a LABEL for each parameter. If no LABEL is "
                 "provided, PARAM will used as the LABEL. {}"
                 "To see all possible parameters that may be used with the "
                 "given input file(s), as well as all avaiable functions, "
                 "run --file-help, along with one or more input files. "
                 "If '*' is provided in addition to other parameters names, "
                 "then parameters will be loaded in a greedy fashion; i.e., "
                 "all other parameters that exist in the file(s) that are not "
                 "explicitly mentioned will also be loaded. For example, "
                 "if the input-file(s) contains 'srcmass1', "
                 "'srcmass2', and 'distance', and  "
                 "\"'primary_mass(srcmass1, srcmass2):mass1' '*'\", is given "
                 "then 'mass1' and 'distance' will be loaded. Otherwise, "
                 "without the '*', only 'mass1' would be loaded. "
                 "Note that any parameter that is used in a function "
                 "will not automatically be added. Tip: enclose "
                 "arguments in single quotes, or else special characters will "
                 "be interpreted as shell commands. For example, the "
                 "wildcard should be given as either '*' or \\*, otherwise "
                 "bash will expand the * into the names of all the files in "
                 "the current directory."
                 .format(lblhelp))
            "--constraint", type=str, nargs="+", metavar="CONSTRAINT[:FILE]",
            help="Apply a constraint to the samples. If a file is provided "
                 "after the constraint, it will only be applied to the given "
                 "file. Otherwise, the constraint will be applied to all "
                 "files. Only one constraint may be applied to a file. "
                 "Samples that violate the constraint will be removed. Syntax "
                 "is python; any parameter or function of parameter can be "
                 "used, similar to the parameters argument. Multiple "
                 "constraints may be combined by using '&' and '|'.")
        return results_reading_group

def results_from_cli(opts, load_samples=True, **kwargs):
    """Loads an inference result file along with any labels associated with it
    from the command line options.

    opts : ArgumentParser options
        The options from the command line.
    load_samples : bool, optional
        Load the samples from the file.

    Returns
    -------
    fp_all : (list of) BaseInferenceFile type
        The result file as an hdf file. If more than one input file,
        then it returns a list.
        List of the parameters to use, parsed from the parameters option.
    labels : dict
        Dictionary of labels to associate with the parameters.
    samples_all : (list of) FieldArray(s) or None
        If load_samples, the samples as a FieldArray; otherwise, None.
        If more than one input file, then it returns a list.
        Any other keyword arguments that are passed to read samples using
        samples_from_cli
    """

    # lists for files and samples from all input files
    fp_all = []
    samples_all = []

    input_files = opts.input_file
    if isinstance(input_files, str):
        input_files = [input_files]

    # load constraints
    constraints = {}
    if opts.constraint is not None:
        for constraint in opts.constraint:
            if len(constraint.split(':')) == 2:
                constraint, fn = constraint.split(':')
                constraints[fn] = constraint
            # no file provided, make sure there's only one constraint
            elif len(opts.constraint) > 1:
                raise ValueError("must provide a file to apply constraints "
                                 "to if providing more than one constraint")
            else:
                # this means no file, only one constraint, apply to all
                # files
                constraints = {fn: constraint for fn in input_files}

    # loop over all input files
    for input_file in input_files:
        logging.info("Reading input file %s", input_file)

        # read input file
        fp = loadfile(input_file, "r")

        # load the samples
        if load_samples:
            logging.info("Loading samples")

            # read samples from file
            samples = fp.samples_from_cli(opts, parameters=opts.parameters,
                                          **kwargs)

            logging.info("Loaded {} samples".format(samples.size))

            if input_file in constraints:
                logging.info("Applying constraints")
                mask = samples[constraints[input_file]]
                samples = samples[mask]
                if samples.size == 0:
                    raise ValueError("No samples remain after constraint {} "
                                     "applied".format(constraints[input_file]))
                logging.info("{} samples remain".format(samples.size))

        # else do not read samples
        else:
            samples = None

        # add results to lists from all input files
        if len(input_files) > 1:
            fp_all.append(fp)
            samples_all.append(samples)

        # else only one input file then do not return lists
        else:
            fp_all = fp
            samples_all = samples

    return fp_all, opts.parameters, opts.parameters_labels, samples_all

def injections_from_cli(opts):
    """Gets injection parameters from the inference file(s).

    If the opts have a ``injection_samples_map`` option, the injection
    parameters will be remapped accordingly. See
    :py:func:`pycbc.inference.option_utils.add_injsamples_map_opt` for
    details.

    opts : argparser
        Argparser object that has the command-line objects to parse.

    Returns
    -------
    FieldArray
        Array of the injection parameters from all of the input files given
        by ``opts.input_file``.
    """
    # see if a mapping was provided
    if hasattr(opts, 'injection_samples_map') and opts.injection_samples_map:
        param_map = [opt.split(':') for opt in opts.injection_samples_map]
    else:
        param_map = []
    injections = None
    # loop over all input files getting the injection files
        fp = loadfile(input_file, 'r')
        these_injs = fp.read_injections()
        # apply mapping if it was provided
        if param_map:
            mapvals = {sp: these_injs[ip] for ip, sp in param_map}
            # if any of the new parameters are the same as the old, just
            # overwrite the values
            common_params = set(these_injs.fieldnames) & set(mapvals.keys())
            for param in common_params:
                these_injs[param] = mapvals.pop(param)
            # add the rest as new fields
            ps = list(mapvals.keys())
            these_injs = these_injs.add_fields([mapvals[p] for p in ps],
                                               names=ps)
        if injections is None:
            injections = these_injs
        else:
            injections = injections.append(these_injs)
    return injections

# Copyright (C) 2020  Collin Capano
#
#

"""Jump proposals that use cyclic boundaries on [0, 2pi)."""

from epsie import proposals as epsie_proposals

from .normal import (epsie_from_config, epsie_adaptive_from_config,
                     epsie_at_adaptive_from_config)

class EpsieAngular(epsie_proposals.Angular):
    """Adds ``from_config`` method to epsie's angular proposal."""

        """Loads a proposal from a config file.

        This calls :py:func:`epsie_from_config` with ``cls`` set to
        :py:class:`epsie.proposals.Angular` and ``with_boundaries`` set
        to False. See that function for details on options that can be read.

        Example::

            [jump_proposal-ra]
            name = angular
            var-ra = 0.01

        cp : WorkflowConfigParser instance
            Config file to read from.
            The name of the section to look in.
        tag : str
            :py:const:`pycbc.VARARGS_DELIM` separated list of parameter names
            to create proposals for.

        Returns
        -------
        :py:class:`epsie.proposals.Angular`:
            An angular proposal for use with ``epsie`` samplers.
        """
        return epsie_from_config(cls, cp, section, tag, with_boundaries=False)

class EpsieAdaptiveAngular(epsie_proposals.AdaptiveAngular):
    """Adds ``from_config`` method to epsie's adaptive angular proposal."""

        r"""Loads a proposal from a config file.

        This calls :py:func:`epsie_adaptive_from_config` with ``cls`` set to
        :py:class:`epsie.proposals.AdaptiveBoundedNormal` and
        ``with_boundaries`` set to False (since the boundaries for the angular
        proposals are always :math:`[0, 2\pi)`). See that function
        for details on options that can be read.

        Example::

            name = adaptive_angular
            adaptation-duration = 1000

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveAngular`:
            An adaptive angular proposal for use with ``epsie`` samplers.
        """
        return epsie_adaptive_from_config(cls, cp, section, tag,
                                          with_boundaries=False)

class EpsieATAdaptiveAngular(epsie_proposals.ATAdaptiveAngular):

        Example::

            name = adaptive_angular_proposal

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveAngularProposal`:
        """
        return epsie_at_adaptive_from_config(cls, cp, section, tag,
                                             with_boundaries=False)

#
#

"""Jump proposals that use a bounded normal distribution."""

class EpsieBoundedNormal(epsie_proposals.BoundedNormal):
    """Adds ``from_config`` method to epsie's bounded normal proposal."""

        :py:class:`epsie.proposals.BoundedNormal` and ``with_boundaries`` set
        to True. See that function for details on options that can be read.

        Example::

            [jump_proposal-mchrip+q]
            name = bounded_normal
            min-q = 1
            max-q = 8
            min-mchirp = 20
            max-mchirp = 80

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.BoundedNormal`:
            A bounded normal proposal for use with ``epsie`` samplers.
        """
        return epsie_from_config(cls, cp, section, tag, with_boundaries=True)

class EpsieAdaptiveBoundedNormal(epsie_proposals.AdaptiveBoundedNormal):
    """Adds ``from_config`` method to epsie's adaptive normal proposal."""

        :py:class:`epsie.proposals.AdaptiveBoundedNormal`. See that function

        Example::

            [jump_proposal-q]
            name = adaptive_bounded_normal
            min-q = 1
            max-q = 8

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveBoundedNormal`:
            An adaptive normal proposal for use with ``epsie`` samplers.
        """
        return epsie_adaptive_from_config(cls, cp, section, tag)

class EpsieATAdaptiveBoundedNormal(epsie_proposals.ATAdaptiveBoundedNormal):
    """Adds ``from_config`` method to epsie's adaptive bounded proposal."""

        :py:class:`epsie.proposals.AdaptiveBoundedProposal`. See that function

        Example::

            name = adaptive_bounded_proposal
            min-q = 1
            max-q = 8

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveBoundedProposal`:
            An adaptive bounded proposal for use with ``epsie`` samplers.
        """
                                             with_boundaries=True)

#
#

from .normal import (epsie_from_config, epsie_adaptive_from_config)

class EpsieNormalDiscrete(epsie_proposals.NormalDiscrete):
    """Adds ``from_config`` method to epsie's normal discrete proposal."""

        :py:class:`epsie.proposals.NormalDiscrete` and ``with_boundaries`` set

        Example::

            [jump_proposal-index]
            name = discrete

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.BoundedDiscrete`:
            A bounded discrete proposal for use with ``epsie`` samplers.
        """

class EpsieBoundedDiscrete(epsie_proposals.BoundedDiscrete):
    """Adds ``from_config`` method to epsie's bounded discrete proposal."""

        :py:class:`epsie.proposals.BoundedDiscrete` and ``with_boundaries`` set

        Example::

            name = bounded_discrete
            min-index = 0
            max-index = 19

        tag : str

        Returns
        -------
        """

class EpsieAdaptiveNormalDiscrete(epsie_proposals.AdaptiveNormalDiscrete):
    """Adds ``from_config`` method to epsie's adaptive bounded discrete
    proposal."""

        :py:class:`epsie.proposals.AdaptiveNormalDiscrete`. See that function

        Example::

            name = adaptive_normal_discrete
            max-index = 42

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveBoundedDiscrete`:
        """
                                          boundary_arg_name='prior_widths')

class EpsieAdaptiveBoundedDiscrete(epsie_proposals.AdaptiveBoundedDiscrete):

        :py:class:`epsie.proposals.AdaptiveBoundedDiscrete`. See that function

        Example::

            name = adaptive_bounded_discrete

        tag : str

        Returns
        -------
        """

#
#

"""Jump proposals that use a normal distribution."""

from epsie.proposals import Boundaries

class EpsieNormal(epsie_proposals.Normal):
    """Adds ``from_config`` method to epsie's normal proposal."""

        :py:class:`epsie.proposals.Normal` and ``with_boundaries`` set to
        False. See that function for details on options that can be read.

        Example::

            name = normal
            var-q = 0.1

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.Normal`:
            A normal proposal for use with ``epsie`` samplers.
        """

class EpsieAdaptiveNormal(epsie_proposals.AdaptiveNormal):

        :py:class:`epsie.proposals.AdaptiveNormal`. See that function for
        details on options that can be read.

        Example::

            [jump_proposal-mchirp+q]
            name = adaptive_normal
            min-q = 1
            max-q = 8

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveNormal`:
        """

class EpsieATAdaptiveNormal(epsie_proposals.ATAdaptiveNormal):
    """Adds ``from_config`` method to epsie's ATAdaptiveProposal."""

        :py:class:`epsie.proposals.AdaptiveProposal` and ``with_boundaries``
        set to False. See that function for details on options that can be
        read.

        Example::

            name = adaptive_proposal
            diagonal =

        tag : str

        Returns
        -------
        :py:class:`epsie.proposals.AdaptiveProposal`:
            An adaptive proposal for use with ``epsie`` samplers.
        """

def epsie_from_config(cls, cp, section, tag, with_boundaries=False):
    r"""Generic function for loading epsie proposals from a config file.

    This should be used for proposals that are not adaptive.

    The section that is read should have the format ``[{section}-{tag}]``,
    where ``{tag}`` is a :py:const:`pycbc.VARARGS_DELIM` separated list
    of the parameters to create the jump proposal for.

    Options that are read:

    * name : str
        Required. Must match the name of the proposal.
    * var-{param} : float
        Optional. Variance to use for parameter {param}. If ``with_boundaries``
        is True, then any parameter not specified will use a default variance
        of :math:`(\Delta p/10)^2`, where :math:`\Delta p` is the boundary
        width for that parameter. If ``with_boundaries`` is False, will use
        a default value of 1.
    * min-{param} : float
    * max-{param} : float
        The bounds on each parameter. Required if ``with_boundaries`` is set to
        True, in which case bounds must be provided for every parameter.

    cls : epsie.proposals.BaseProposal
        The epsie proposal class to initialize.
    cp : WorkflowConfigParser instance
        Config file to read from.
        The name of the section to look in.
    tag : str
        :py:const:`pycbc.VARARGS_DELIM` separated list of parameter names
        to create proposals for.
    with_boundaries : bool, optional
        Try to load boundaries from the section and pass a ``boundaries``
        argument to the class's initialization. This should be set to true
        for bounded proposals. Default is False.

    Returns
    -------
    cls :
        The class initialized with the options read from the config file.
    """
    # check that the name matches
    assert cp.get_opt_tag(section, "name", tag) == cls.name, (
        "name in specified section must match mine")
    params, opts = load_opts(cp, section, tag, skip=['name'])
    args = {'parameters': params}
    if with_boundaries:
        boundaries = get_param_boundaries(params, opts)
        args['boundaries'] = boundaries
    if 'discrete' in cls.name.split('_'):
        args.update({'successive':
                     get_epsie_discrete_successive_settings(params, opts)})
    # if there are any options left, assume they are for setting the variance
    if opts:
        cov = get_variance(params, opts)
    elif with_boundaries:
        cov = numpy.array([abs(boundaries[p])/10. for p in params])**2.
    else:
        cov = None
    args['cov'] = cov
    # no other options should remain
    if opts:
        raise ValueError("unrecognized options {}"
                         .format(', '.join(opts.keys())))
    return cls(**args)

def epsie_adaptive_from_config(cls, cp, section, tag, with_boundaries=True,
                               boundary_arg_name='boundaries'):
    """Generic function for loading adaptive epsie proposals from a config
    file.

    * name : str
    * adaptation-duration : int
        Required. Sets the ``adaptation_duration``.
        Optional. Initial variance to use. If not provided, will use a
        default based on the bounds (see
        :py:class:`epsie.proposals.AdaptiveSupport` for details).
    * adaptation-decay : int
        Optional. Sets the ``adaptation_decay``. If not provided, will use
        the class's default.
    * start-iteration : int
        Optional. Sets the ``start_iteration``.If not provided, will use
    * target-rate : float
        Optional. Sets the ``target_rate``. If not provided, will use

        The epsie proposal class to initialize. The class should have
        :py:class:`epsie.proposals.normal.AdaptiveSupport`.
    tag : str
        argument to the class's initialization. Default is True.
    boundary_arg_name : str, optional
        The name of the argument for the boundaries (only used if
        ``with_boundaries`` is True). Provided because some adaptive proposals
        that only need the boundary widths call this ``prior_widths``. Default
        is ``'boundaries'``.

    Returns
    -------
    cls :
    """
    # get the bounds
        args[boundary_arg_name] = get_param_boundaries(params, opts)
    # get the adaptation parameters
    args.update(get_epsie_adaptation_settings(opts))
    # if there are any other options, assume they are for setting the
    # initial standard deviation
    if opts:
        var = get_variance(params, opts)
        args['initial_std'] = var**0.5
        # at this point, there should be no options left
        if opts:
            raise ValueError('unrecognized options {} in section {}'
                             .format(', '.join(opts.keys()),
                                     '-'.join([section, tag])))

def epsie_at_adaptive_from_config(cls, cp, section, tag,
                                  with_boundaries=False):
    """Generic function for loading AT Adaptive Normal proposals from a config
    file.

    * name : str
        Sets the ``adaptation_duration``. If not provided will use the class's
        default.
    * diagonal : bool, optional
        Determines whether only to adapt the variance. If True will only train
        the diagonal elements.
    * componentwise : bool, optional
        Whether to include a componentwise scaling of the parameters.
        By default set to False. Componentwise scaling `ndim` times more
        expensive than global scaling.
        Optional. Sets the ``start_iteration``. If not provided, will use

    tag : str

    Returns
    -------
    cls :
    """
        args['boundaries'] = get_param_boundaries(params, opts)
    args.update(get_epsie_adaptation_settings(opts, cls.name))
    # bounded and angular adaptive proposals support diagonal-only
    diagonal = opts.pop('diagonal', None)
    if not any(p in cls.name.split('_') for p in ['bounded', 'angular']):
        args.update({'diagonal': diagonal is not None})
    componentwise = opts.pop('componentwise', None)
    if componentwise is not None:
        args.update({'componentwise': True})
    if opts:

def load_opts(cp, section, tag, skip=None):
    """Loads config options for jump proposals.

    All `-` in option names are converted to `_` before returning.

    tag : str
    skip : list, optional
        List of option names to skip loading.

    Returns
    -------
        List of parameter names the jump proposal is for.
    opts : dict
        Dictionary of option names -> values, where all values are strings.
    """
    if skip is None:
        skip = []
    params = tag.split(VARARGS_DELIM)
    # get options
    readsection = '-'.join([section, tag])
    opts = {opt.replace('-', '_'): cp.get(readsection, opt)
            for opt in cp.options(readsection) if opt not in skip}
    return params, opts

def get_variance(params, opts, default=1.):
    """Gets variance for jump proposals from the dictionary of options.

    This looks for ``var_{param}`` for every parameter listed in ``params``.
    If found, the argument is popped from the given ``opts`` dictionary. If not
    found, ``default`` will be used.

    params : list of str
        List of parameter names to look for.
    opts : dict
        Dictionary of option -> value that was loaded from a config file
        section.
    default : float, optional
        Default value to use for parameters that do not have variances
        provided. Default is 1.

    Returns
    -------
    numpy.array
        Array of variances to use. Order is the same as the parameter names
        given in ``params``.
    """
    varfmt = 'var_{}'
    cov = numpy.array([float(opts.pop(varfmt.format(param), default))
                       for param in params])
    return cov

def get_param_boundaries(params, opts):
    """Gets parameter boundaries for jump proposals.

    The syntax for the options should be ``(min|max)_{param} = value``. Both
    a minimum and maximum should be provided for every parameter in ``params``.
    If the opts are created using ``load_opts``, then the options can be
    formatted as ``(min|max)-{param}``, since that function will turn all ``-``
    to ``_`` in option names.

    Arguments will be popped from the given ``opts`` dictionary.

        List of parameter names to get boundaries for.
    opts : dict
        section.

    Returns
    -------
    dict :
        Dictionary of parameter names -> :py:class:`epsie.proposals.Boundaries`
    """
    boundaries = {}
    for param in params:
        minbound = opts.pop('min_{}'.format(param), None)
        if minbound is None:
            raise ValueError("Must provide a minimum bound for {p}."
                             "Syntax is min_{p} = val".format(p=param))
        maxbound = opts.pop('max_{}'.format(param), None)
        if maxbound is None:
            raise ValueError("Must provide a maximum bound for {p}."
                             "Syntax is max_{p} = val".format(p=param))
        boundaries[param] = Boundaries((float(minbound), float(maxbound)))
    return boundaries

def get_epsie_adaptation_settings(opts, name=None):
    """Get settings for Epsie adaptive proposals from a config file.

    This requires that ``adaptation_duration`` is in the given dictionary.
    It will also look for ``adaptation_decay``, ``start_iteration``, and
    ``target_rate``, but these are optional. Arguments will be popped from the
    given dictionary.

    opts : dict
        section.
    name : str (optional)
        Proposal name

    Returns
    -------
    dict :
        Dictionary of argument name -> values.
    """
    args = {}
    adaptation_duration = opts.pop('adaptation_duration', None)
    if adaptation_duration is None:
        if name is not None:
            if all(p in name.split('_') for p in ['at', 'adaptive']):
                args.update({'adaptation_duration': None})
        else:
            raise ValueError("No adaptation_duration specified")
    else:
        args.update({'adaptation_duration': int(adaptation_duration)})
    # optional args
    adaptation_decay = opts.pop('adaptation_decay', None)
    if adaptation_decay is not None:
        args.update({'adaptation_decay': int(adaptation_decay)})
    start_iteration = opts.pop('start_iteration', None)
    if start_iteration is not None:
        args.update({'start_iteration': int(start_iteration)})
    target_rate = opts.pop('target_rate', None)
    if target_rate is not None:
        args.update({'target_rate': float(target_rate)})

def get_epsie_discrete_successive_settings(params, opts):
    """Get settings for Epsie successive discrete proposal successive jumps
    from a config file.

    If ``successive`` is not defined for a parameter then assumes successive
    jumps are not allowed (i.e. jumps from an integer to the same integer).
    Arguments will be popped from the given dictionary.

    Example::
        [jump_proposal-k+n]
        name = discrete
        successive-k =

    This example sets successive jumps for ``k`` but does not do so for ``n``.

        List of parameter names to get the successive option for.
    opts : dict
        section.

    Returns
    -------
    dict :
        Dictionary of parameter names -> bools
    """
    successive = {}
        successive.update(
            {param: opts.pop('successive_{}'.format(param), None) is not None})
    return successive

#
#
"""Provides custom jump proposals for samplers."""

from .normal import (EpsieNormal, EpsieAdaptiveNormal, EpsieATAdaptiveNormal)
from .bounded_normal import (EpsieBoundedNormal, EpsieAdaptiveBoundedNormal,
                             EpsieATAdaptiveBoundedNormal)
from .angular import (EpsieAngular, EpsieAdaptiveAngular,
                      EpsieATAdaptiveAngular)
from .discrete import (EpsieNormalDiscrete, EpsieBoundedDiscrete,
                       EpsieAdaptiveNormalDiscrete,
                       EpsieAdaptiveBoundedDiscrete)

epsie_proposals = {
    EpsieNormal.name: EpsieNormal,
    EpsieAdaptiveNormal.name: EpsieAdaptiveNormal,
    EpsieATAdaptiveNormal.name: EpsieATAdaptiveNormal,
    EpsieBoundedNormal.name: EpsieBoundedNormal,
    EpsieAdaptiveBoundedNormal.name: EpsieAdaptiveBoundedNormal,
    EpsieATAdaptiveBoundedNormal.name: EpsieATAdaptiveBoundedNormal,
    EpsieAngular.name: EpsieAngular,
    EpsieAdaptiveAngular.name: EpsieAdaptiveAngular,
    EpsieATAdaptiveAngular.name: EpsieATAdaptiveAngular,
    EpsieNormalDiscrete.name: EpsieNormalDiscrete,
    EpsieAdaptiveNormalDiscrete.name: EpsieAdaptiveNormalDiscrete,
    EpsieBoundedDiscrete.name: EpsieBoundedDiscrete,
    EpsieAdaptiveBoundedDiscrete.name: EpsieAdaptiveBoundedDiscrete,
}

def epsie_proposals_from_config(cp, section='jump_proposal'):
    """Loads epsie jump proposals from the given config file.

    This loads jump proposals from sub-sections starting with ``section``
    (default is 'jump_proposal'). The tag part of the sub-sections' headers
    should list the parameters the proposal is to be used for.

    Example::

        [jump_proposal-mtotal+q]
        name = adaptive_normal
        adaptation-duration = 1000
        min-q = 1
        max-q = 8
        min-mtotal = 20
        max-mtotal = 160

        [jump_proposal-spin1_a]
        name = normal

    section : str, optional
        The section name to read jump proposals from. Default is
        ``'jump_proposal'``.

    Returns
    -------
    list :
        List of the proposal instances.
    """
    tags = cp.get_subsections(section)
    proposals = []
    for tag in tags:
        # get the name of the proposal
        name = cp.get_opt_tag(section, "name", tag)
        prop = epsie_proposals[name].from_config(cp, section, tag)
        proposals.append(prop)
    return proposals

#
#

"""
This modules provides models that have analytic solutions for the
log likelihood.
"""

from .base import BaseModel

class TestNormal(BaseModel):
    r"""The test distribution is an multi-variate normal distribution.

    The number of dimensions is set by the number of ``variable_params`` that
    are passed. For details on the distribution used, see
    ``scipy.stats.multivariate_normal``.

    variable_params : (tuple of) string(s)
        A tuple of parameter names that will be varied.
    mean : array-like, optional
        The mean values of the parameters. If None provide, will use 0 for all
    cov : array-like, optional
        The covariance matrix of the parameters. If None provided, will use
        unit variance for all parameters, with cross-terms set to 0.
    **kwargs :
        All other keyword arguments are passed to ``BaseModel``.

    Examples
    --------
    Create a 2D model with zero mean and unit variance:

    >>> m = TestNormal(['x', 'y'])

    Set the current parameters and evaluate the log posterior:

    >>> m.update(x=-0.2, y=0.1)
    >>> m.logposterior
    -1.8628770664093453

    See the current stats that were evaluated:

    >>> m.current_stats
    {'logjacobian': 0.0, 'loglikelihood': -1.8628770664093453, 'logprior': 0.0}

    """
    name = "test_normal"

    def __init__(self, variable_params, mean=None, cov=None, **kwargs):
        # set up base likelihood parameters
        super(TestNormal, self).__init__(variable_params, **kwargs)
        # store the pdf
        if mean is None:
            mean = [0.]*len(variable_params)
        if cov is None:
            cov = [1.]*len(variable_params)
        self._dist = stats.multivariate_normal(mean=mean, cov=cov)
        # check that the dimension is correct
        if self._dist.dim != len(variable_params):
            raise ValueError("dimension mis-match between variable_params and "
                             "mean and/or cov")

    def _loglikelihood(self):
        """Returns the log pdf of the multivariate normal.
        """
        return self._dist.logpdf([self.current_params[p]
                                  for p in self.variable_params])

class TestEggbox(BaseModel):
    r"""The test distribution is an 'eggbox' function:

    .. math::

        \log \mathcal{L}(\Theta) = \left[
            2+\prod_{i=1}^{n}\cos\left(\frac{\theta_{i}}{2}\right)\right]^{5}

    are passed.

    **kwargs :

    """
    name = "test_eggbox"

    def __init__(self, variable_params, **kwargs):
        super(TestEggbox, self).__init__(variable_params, **kwargs)

        """Returns the log pdf of the eggbox function.
        """
        return (2 + numpy.prod(numpy.cos([
            self.current_params[p]/2. for p in self.variable_params]))) ** 5

class TestRosenbrock(BaseModel):
    r"""The test distribution is the Rosenbrock function:

    .. math::

        \log \mathcal{L}(\Theta) = -\sum_{i=1}^{n-1}[
            (1-\theta_{i})^{2}+100(\theta_{i+1} - \theta_{i}^{2})^{2}]

    **kwargs :

    """
    name = "test_rosenbrock"

        super(TestRosenbrock, self).__init__(variable_params, **kwargs)

        """Returns the log pdf of the Rosenbrock function.
        """
        logl = 0
        p = [self.current_params[p] for p in self.variable_params]
        for i in range(len(p) - 1):
            logl -= ((1 - p[i])**2 + 100 * (p[i+1] - p[i]**2)**2)
        return logl

class TestVolcano(BaseModel):
    r"""The test distribution is a two-dimensional 'volcano' function:

    .. math::
        \Theta =
            \sqrt{\theta_{1}^{2} + \theta_{2}^{2}} \log \mathcal{L}(\Theta) =
            25\left(e^{\frac{-\Theta}{35}} +
                    \frac{1}{2\sqrt{2\pi}} e^{-\frac{(\Theta-5)^{2}}{8}}\right)

        A tuple of parameter names that will be varied. Must have length 2.
    **kwargs :

    """
    name = "test_volcano"

        super(TestVolcano, self).__init__(variable_params, **kwargs)

        # make sure there are exactly two variable args
        if len(self.variable_params) != 2:
            raise ValueError("TestVolcano distribution requires exactly "
                             "two variable args")

        """Returns the log pdf of the 2D volcano function.
        """
        r = numpy.sqrt(p[0]**2 + p[1]**2)
        mu, sigma = 5.0, 2.0
        return 25 * (
            numpy.exp(-r/35) + 1 / (sigma * numpy.sqrt(2 * numpy.pi)) *
            numpy.exp(-0.5 * ((r - mu) / sigma) ** 2))

class TestPrior(BaseModel):
    r"""Uses the prior as the test distribution.

    **kwargs :

    """
    name = "test_prior"

        super(TestPrior, self).__init__(variable_params, **kwargs)

        """Returns zero.
        """
        return 0.

class TestPosterior(BaseModel):
    r"""Build a test posterior from a set of samples using a kde

    posterior_file : hdf file
        A compatible pycbc inference output file which posterior samples can
        be read from.
        Number of samples to draw from posterior file to build KDE.
    **kwargs :

    """
    name = "test_posterior"

    def __init__(self, variable_params, posterior_file, nsamples, **kwargs):
        super(TestPosterior, self).__init__(variable_params, **kwargs)

        from pycbc.inference.io import loadfile  # avoid cyclic import
        logging.info('loading test posterior model')
        inf_file = loadfile(posterior_file)
        logging.info('reading samples')
        samples = inf_file.read_samples(variable_params)
        samples = numpy.array([samples[v] for v in variable_params])

        # choose only the requested amount of samples
        idx = numpy.arange(0, samples.shape[-1])
        idx = numpy.random.choice(idx, size=int(nsamples), replace=False)
        samples = samples[:, idx]

        logging.info('making kde with %s samples', samples.shape[-1])
        self.kde = stats.gaussian_kde(samples)
        logging.info('done initializing test posterior model')

        """Returns the log pdf of the test posterior kde
        """
        p = numpy.array([self.current_params[p] for p in self.variable_params])
        logpost = self.kde.logpdf(p)
        return float(logpost[0])

#
#

#
#
#                                   Preamble
#
#

"""Base class for models.
"""

from six.moves.configparser import NoSectionError
from pycbc import (transforms, distributions)

#
#
#                               Support classes
#
#

class _NoPrior(object):
    """Dummy class to just return 0 if no prior is given to a model.
    """
    def apply_boundary_conditions(**params):

        return 0.

class ModelStats(object):
    """Class to hold model's current stat values."""

    @property
    def statnames(self):
        """Returns the names of the stats that have been stored."""
        return list(self.__dict__.keys())

    def getstats(self, names, default=numpy.nan):
        """Get the requested stats as a tuple.

        If a requested stat is not an attribute (implying it hasn't been
        stored), then the default value is returned for that stat.

        names : list of str
            The names of the stats to get.
        default : float, optional
            What to return if a requested stat is not an attribute of self.
            Default is ``numpy.nan``.

        Returns
        -------
        tuple
            A tuple of the requested stats.
        """
        return tuple(getattr(self, n, default) for n in names)

    def getstatsdict(self, names, default=numpy.nan):
        """Get the requested stats as a dictionary.

        Returns
        -------
        dict
            A dictionary of the requested stats.
        """
        return dict(zip(names, self.getstats(names, default=default)))

class SamplingTransforms(object):
    """Provides methods for transforming between sampling parameter space and
    model parameter space.
    """

    def __init__(self, variable_params, sampling_params,
                 replace_parameters, sampling_transforms):
        assert len(replace_parameters) == len(sampling_params), (
            "number of sampling parameters must be the "
            "same as the number of replace parameters")
        # pull out the replaced parameters
        self.sampling_params = [arg for arg in variable_params
                                if arg not in replace_parameters]
        # add the sampling parameters
        self.sampling_params += sampling_params
        # sort to make sure we have a consistent order
        self.sampling_params.sort()
        self.sampling_transforms = sampling_transforms

    def logjacobian(self, **params):
        r"""Returns the log of the jacobian needed to transform pdfs in the
        ``variable_params`` parameter space to the ``sampling_params``
        parameter space.

        Let :math:`\mathbf{x}` be the set of variable parameters,
        :math:`\mathbf{y} = f(\mathbf{x})` the set of sampling parameters, and
        :math:`p_x(\mathbf{x})` a probability density function defined over
        :math:`\mathbf{x}`.
        The corresponding pdf in :math:`\mathbf{y}` is then:

        .. math::

            p_y(\mathbf{y}) =
                p_x(\mathbf{x})\left|\mathrm{det}\,\mathbf{J}_{ij}\right|,

        where :math:`\mathbf{J}_{ij}` is the Jacobian of the inverse transform
        :math:`\mathbf{x} = g(\mathbf{y})`. This has elements:

        .. math::

            \mathbf{J}_{ij} = \frac{\partial g_i}{\partial{y_j}}

        This function returns
        :math:`\log \left|\mathrm{det}\,\mathbf{J}_{ij}\right|`.

        \**params :
            The keyword arguments should specify values for all of the variable
            args and all of the sampling args.

        Returns
        -------
        float :
            The value of the jacobian.
        """
        return numpy.log(abs(transforms.compute_jacobian(
            params, self.sampling_transforms, inverse=True)))

    def apply(self, samples, inverse=False):
        """Applies the sampling transforms to the given samples.

        samples : dict or FieldArray
            The samples to apply the transforms to.
        inverse : bool, optional
            Whether to apply the inverse transforms (i.e., go from the sampling
            args to the ``variable_params``). Default is False.

        Returns
        -------
            The transformed samples, along with the original samples.
        """
        return transforms.apply_transforms(samples, self.sampling_transforms,
                                           inverse=inverse)

    def from_config(cls, cp, variable_params):
        """Gets sampling transforms specified in a config file.

        Sampling parameters and the parameters they replace are read from the
        ``sampling_params`` section, if it exists. Sampling transforms are
        read from the ``sampling_transforms`` section(s), using
        ``transforms.read_transforms_from_config``.

        An ``AssertionError`` is raised if no ``sampling_params`` section
        exists in the config file.

        cp : WorkflowConfigParser
            Config file parser to read.
        variable_params : list
            List of parameter names of the original variable params.

        Returns
        -------
        SamplingTransforms
            A sampling transforms class.
        """
        # Check if a sampling_params section is provided
        try:
            sampling_params, replace_parameters = \
                read_sampling_params_from_config(cp)
        except NoSectionError as e:
            logging.warning("No sampling_params section read from config file")
            raise e
        # get sampling transformations
        sampling_transforms = transforms.read_transforms_from_config(
            cp, 'sampling_transforms')
        logging.info("Sampling in {} in place of {}".format(
            ', '.join(sampling_params), ', '.join(replace_parameters)))
        return cls(variable_params, sampling_params,
                   replace_parameters, sampling_transforms)

def read_sampling_params_from_config(cp, section_group=None,
                                     section='sampling_params'):
    """Reads sampling parameters from the given config file.

    Parameters are read from the `[({section_group}_){section}]` section.
    The options should list the variable args to transform; the parameters they
    point to should list the parameters they are to be transformed to for
    sampling. If a multiple parameters are transformed together, they should
    be comma separated. Example:

        [sampling_params]
        mass1, mass2 = mchirp, logitq
        spin1_a = logitspin1_a

    Note that only the final sampling parameters should be listed, even if
    multiple intermediate transforms are needed. (In the above example, a
    transform is needed to go from mass1, mass2 to mchirp, q, then another one
    needed to go from q to logitq.) These transforms should be specified
    in separate sections; see ``transforms.read_transforms_from_config`` for
    details.

    section_group : str, optional
        Append `{section_group}_` to the section name. Default is None.
        The name of the section. Default is 'sampling_params'.

    Returns
    -------
    sampling_params : list
        The list of sampling parameters to use instead.
    replaced_params : list
        The list of variable args to replace in the sampler.
    """
    if section_group is not None:
        section_prefix = '{}_'.format(section_group)
    else:
        section_prefix = ''
    section = section_prefix + section
    replaced_params = set()
    sampling_params = set()
    for args in cp.options(section):
        map_args = cp.get(section, args)
        sampling_params.update(set(map(str.strip, map_args.split(','))))
        replaced_params.update(set(map(str.strip, args.split(','))))
    return sorted(sampling_params), sorted(replaced_params)

#
#
#                               Base model definition
#
#

class BaseModel(object):
    r"""Base class for all models.

    Given some model :math:`h` with parameters :math:`\Theta`, Bayes Theorem
    states that the probability of observing parameter values :math:`\vartheta`
    is:

    .. math::

        p(\vartheta|h) = \frac{p(h|\vartheta) p(\vartheta)}{p(h)}.

    Here:

     * :math:`p(\vartheta|h)` is the **posterior** probability;

     * :math:`p(h|\vartheta)` is the **likelihood**;

     * :math:`p(\vartheta)` is the **prior**;

     * :math:`p(h)` is the **evidence**.

    This class defines properties and methods for evaluating the log
    likelihood, log prior, and log posteror. A set of parameter values is set
    using the ``update`` method. Calling the class's
    ``log(likelihood|prior|posterior)`` properties will then evaluate the model
    at those parameter values.

    Classes that inherit from this class must implement a ``_loglikelihood``
    function that can be called by ``loglikelihood``.

    static_params : dict, optional
        A dictionary of parameter names -> values to keep fixed.
    prior : callable, optional
        A callable class or function that computes the log of the prior. If
        None provided, will use ``_noprior``, which returns 0 for all parameter
        values.
    sampling_params : list, optional
        Replace one or more of the ``variable_params`` with the given
        parameters for sampling.
    replace_parameters : list, optional
        The ``variable_params`` to replace with sampling parameters. Must be
        the same length as ``sampling_params``.
    sampling_transforms : list, optional
        List of transforms to use to go between the ``variable_params`` and the
        sampling parameters. Required if ``sampling_params`` is not None.
    waveform_transforms : list, optional
        A list of transforms to convert the ``variable_params`` into something
        understood by the likelihood model. This is useful if the prior is
        more easily parameterized in parameters that are different than what
        the likelihood is most easily defined in. Since these are used solely
        for converting parameters, and not for rescaling the parameter space,
        a Jacobian is not required for these transforms.

    Properties
    logjacobian :
        Returns the log of the jacobian needed to go from the parameter space
        of the ``variable_params`` to the sampling params.
    logprior :
        Returns the log of the prior.
    loglikelihood :
        A function that returns the log of the likelihood function.
    logposterior :
        A function that returns the log of the posterior.
    loglr :
        A function that returns the log of the likelihood ratio.
    logplr :
        A function that returns the log of the prior-weighted likelihood ratio.
    """
    name = None

    def __init__(self, variable_params, static_params=None, prior=None,
                 sampling_transforms=None, waveform_transforms=None):
        # store variable and static args
        if isinstance(variable_params, string_types):
            variable_params = (variable_params,)
        if not isinstance(variable_params, tuple):
            variable_params = tuple(variable_params)
        self._variable_params = variable_params
        if static_params is None:
            static_params = {}
        self._static_params = static_params
        # store prior
        if prior is None:
            self.prior_distribution = _NoPrior()
        else:
            assert prior.variable_args == variable_params, (
                "variable params of prior and model must be the same")
            self.prior_distribution = prior
        # store transforms
        self.waveform_transforms = waveform_transforms
        # initialize current params to None
        self._current_params = None
        # initialize a model stats
        self._current_stats = ModelStats()

    @property
    def variable_params(self):
        """Returns the model parameters."""
        return self._variable_params

    @property
        """Returns the model's static arguments."""
        return self._static_params

    @property
    def sampling_params(self):
        """Returns the sampling parameters.

        If ``sampling_transforms`` is None, this is the same as the
        ``variable_params``.
        """
        if self.sampling_transforms is None:
            sampling_params = self.variable_params
        else:
            sampling_params = self.sampling_transforms.sampling_params
        return sampling_params

    def update(self, **params):
        """Updates the current parameter positions and resets stats.

        If any sampling transforms are specified, they are applied to the
        params before being stored.
        """
        # add the static params
        params.update(self.static_params)
        self._current_params = self._transform_params(**params)

    @property
    def current_params(self):
        if self._current_params is None:
            raise ValueError("no parameters values currently stored; "
                             "run update to add some")
        return self._current_params

    @property
    def default_stats(self):
        """The stats that ``get_current_stats`` returns by default."""
        return ['logjacobian', 'logprior', 'loglikelihood'] + self._extra_stats

    @property
    def _extra_stats(self):
        """Allows child classes to add more stats to the default stats.

        This returns an empty list; classes that inherit should override this
        property if they want to add extra stats.
        """
        return []

    def get_current_stats(self, names=None):
        """Return one or more of the current stats as a tuple.

        This function does no computation. It only returns what has already
        been calculated. If a stat hasn't been calculated, it will be returned
        as ``numpy.nan``.

        names : list of str, optional
            Specify the names of the stats to retrieve. If ``None`` (the
            default), will return ``default_stats``.

        Returns
        -------
        tuple :
            The current values of the requested stats, as a tuple. The order
            of the stats is the same as the names.
        """
        if names is None:
            names = self.default_stats
        return self._current_stats.getstats(names)

    @property
    def current_stats(self):
        """Return the ``default_stats`` as a dict.

        This does no computation. It only returns what has already been
        calculated. If a stat hasn't been calculated, it will be returned

        Returns
        -------
        dict :
            Dictionary of stat names -> current stat values.
        """
        return self._current_stats.getstatsdict(self.default_stats)

    def _trytoget(self, statname, fallback, apply_transforms=False, **kwargs):
        r"""Helper function to get a stat from ``_current_stats``.

        If the statistic hasn't been calculated, ``_current_stats`` will raise
        an ``AttributeError``. In that case, the ``fallback`` function will
        be called. If that call is successful, the ``statname`` will be added
        to ``_current_stats`` with the returned value.

        statname : str
            The stat to get from ``current_stats``.
        fallback : method of self
            The function to call if the property call fails.
        apply_transforms : bool, optional
            Apply waveform transforms to the current parameters before calling
            the fallback function. Default is False.
            Any other keyword arguments are passed through to the function.

        Returns
        -------
        float :
            The value of the property.
        """
        try:
            return getattr(self._current_stats, statname)
            # apply waveform transforms if requested
            if apply_transforms and self.waveform_transforms is not None:
                self._current_params = transforms.apply_transforms(
                    self._current_params, self.waveform_transforms,
                    inverse=False)
            val = fallback(**kwargs)
            setattr(self._current_stats, statname, val)
            return val

    @property
    def loglikelihood(self):
        """The log likelihood at the current parameters.

        This will initially try to return the ``current_stats.loglikelihood``.
        If that raises an ``AttributeError``, will call `_loglikelihood`` to
        calculate it and store it to ``current_stats``.
        """
        return self._trytoget('loglikelihood', self._loglikelihood,
                              apply_transforms=True)

        """Low-level function that calculates the log likelihood of the current
        params."""
        pass

    @property
    def logjacobian(self):
        """The log jacobian of the sampling transforms at the current postion.

        If no sampling transforms were provided, will just return 0.

        Returns
        -------
        float :
        """
        return self._trytoget('logjacobian', self._logjacobian)

    def _logjacobian(self):
        """Calculates the logjacobian of the current parameters."""
            logj = 0.
        else:
            logj = self.sampling_transforms.logjacobian(
                **self.current_params)
        return logj

    @property
    def logprior(self):
        """Returns the log prior at the current parameters."""
        return self._trytoget('logprior', self._logprior)

    def _logprior(self):
        """Calculates the log prior at the current parameters."""
        logj = self.logjacobian
        logp = self.prior_distribution(**self.current_params) + logj
        if numpy.isnan(logp):
            logp = -numpy.inf
        return logp

    @property
    def logposterior(self):
        """Returns the log of the posterior of the current parameter values.

        The logprior is calculated first. If the logprior returns ``-inf``
        (possibly indicating a non-physical point), then the ``loglikelihood``
        is not called.
        """
        logp = self.logprior
        if logp == -numpy.inf:
            return logp
        else:
            return logp + self.loglikelihood

    def prior_rvs(self, size=1, prior=None):
        """Returns random variates drawn from the prior.

        If the ``sampling_params`` are different from the ``variable_params``,
        the variates are transformed to the `sampling_params` parameter space
        before being returned.

            Number of random values to return for each parameter. Default is 1.
        prior : JointDistribution, optional
            Use the given prior to draw values rather than the saved prior.

        Returns
        -------
            A field array of the random values.
        """
        # draw values from the prior
            prior = self.prior_distribution
        p0 = prior.rvs(size=size)
        # transform if necessary
        if self.sampling_transforms is not None:
            ptrans = self.sampling_transforms.apply(p0)
            # pull out the sampling args
            p0 = FieldArray.from_arrays([ptrans[arg]
                                         for arg in self.sampling_params],
                                        names=self.sampling_params)
        return p0

    def _transform_params(self, **params):
        """Applies sampling transforms and boundary conditions to parameters.

            Key, value pairs of parameters to apply the transforms to.

        Returns
        -------
        dict
            A dictionary of the transformed parameters.
        """
        # apply inverse transforms to go from sampling parameters to
        # variable args
            params = self.sampling_transforms.apply(params, inverse=True)
        # apply boundary conditions
        params = self.prior_distribution.apply_boundary_conditions(**params)

    #
    # Methods for initiating from a config file.
    #
    def extra_args_from_config(cp, section, skip_args=None, dtypes=None):
        """Gets any additional keyword in the given config file.

        skip_args : list of str, optional
            Names of arguments to skip.
        dtypes : dict, optional
            A dictionary of arguments -> data types. If an argument is found
            in the dict, it will be cast to the given datatype. Otherwise, the
            argument's value will just be read from the config file (and thus
            be a string).

        Returns
        -------
        dict
            Dictionary of keyword arguments read from the config file.
        """
        kwargs = {}
        if dtypes is None:
            dtypes = {}
        read_args = [opt for opt in cp.options(section)
                     if opt not in skip_args]
        for opt in read_args:
            val = cp.get(section, opt)
            # try to cast the value if a datatype was specified for this opt
            try:
                val = dtypes[opt](val)
                pass
            kwargs[opt] = val

    def prior_from_config(cp, variable_params, prior_section,
                          constraint_section):
        """Gets arguments and keyword arguments from a config file.

            List of of model parameter names.
        prior_section : str
            Section to read prior(s) from.
        constraint_section : str
            Section to read constraint(s) from.

        Returns
        -------
        pycbc.distributions.JointDistribution
            The prior.
        """
        # get prior distribution for each variable parameter
        logging.info("Setting up priors for each parameter")
        dists = distributions.read_distributions_from_config(cp, prior_section)
        constraints = distributions.read_constraints_from_config(
            cp, constraint_section)
        return distributions.JointDistribution(variable_params, *dists,
                                               constraints=constraints)

    def _init_args_from_config(cls, cp):
        """Helper function for loading parameters.

        This retrieves the prior, variable parameters, static parameterss,
        constraints, sampling transforms, and waveform transforms
        (if provided).

            Config parser to read.

        Returns
        -------
        dict :
            Dictionary of the arguments. Has keys ``variable_params``,
            ``static_params``, ``prior``, and ``sampling_transforms``. If
            waveform transforms are in the config file, will also have
            ``waveform_transforms``.
        """
        section = "model"
        prior_section = "prior"
        vparams_section = 'variable_params'
        sparams_section = 'static_params'
        constraint_section = 'constraint'
        # check that the name exists and matches
        name = cp.get(section, 'name')
        if name != cls.name:
            raise ValueError("section's {} name does not match mine {}".format(
                             name, cls.name))
        # get model parameters
        variable_params, static_params = distributions.read_params_from_config(
            cp, prior_section=prior_section, vargs_section=vparams_section,
            sargs_section=sparams_section)
        # get prior
        prior = cls.prior_from_config(cp, variable_params, prior_section,
                                      constraint_section)
        args = {'variable_params': variable_params,
                'static_params': static_params,
                'prior': prior}
        # try to load sampling transforms
        try:
            sampling_transforms = SamplingTransforms.from_config(
                cp, variable_params)
        except NoSectionError:
            sampling_transforms = None
        args['sampling_transforms'] = sampling_transforms
        # get any waveform transforms
        if any(cp.get_subsections('waveform_transforms')):
            logging.info("Loading waveform transforms")
            args['waveform_transforms'] = \
                transforms.read_transforms_from_config(
                    cp, 'waveform_transforms')
        return args

    def from_config(cls, cp, **kwargs):
        """Initializes an instance of this class from the given config file.

            All additional keyword arguments are passed to the class. Any
            provided keyword will over ride what is in the config file.
        """
        args = cls._init_args_from_config(cp)
        # get any other keyword arguments provided in the model section
        args.update(cls.extra_args_from_config(cp, "model",
                                               skip_args=['name']))
        args.update(kwargs)
        return cls(**args)

    def write_metadata(self, fp):
        """Writes metadata to the given file handler.

        fp : pycbc.inference.io.BaseInferenceFile instance
            The inference file to write to.
        """
        fp.attrs['model'] = self.name
        fp.attrs['variable_params'] = list(self.variable_params)
        fp.attrs['sampling_params'] = list(self.sampling_params)
        fp.write_kwargs_to_attrs(fp.attrs, static_params=self.static_params)

#
#

#
#
#                                   Preamble
#
#

"""Base classes for models with data.
"""

class BaseDataModel(BaseModel):
    r"""Base class for models that require data and a waveform generator.

    This adds propeties for the log of the likelihood that the data contain
    noise, ``lognl``, and the log likelihood ratio ``loglr``.

    Classes that inherit from this class must define ``_loglr`` and ``_lognl``
    functions, in addition to the ``_loglikelihood`` requirement inherited from
    ``BaseModel``.

    data : dict
        A dictionary of data, in which the keys are the detector names and the
        values are the data.
    recalibration : dict of pycbc.calibration.Recalibrate, optional
        Dictionary of detectors -> recalibration class instances for
        recalibrating data.
    gates : dict of tuples, optional
        Dictionary of detectors -> tuples of specifying gate times. The
        sort of thing returned by `pycbc.gate.gates_from_cli`.
    injection_file : str, optional
        If an injection was added to the data, the name of the injection file
        used. If provided, the injection parameters will be written to
        file when ``write_metadata`` is called.

    data : dict
        The data that the class was initialized with.
    detectors : list
        List of detector names used.
    lognl :
        Returns the log likelihood of the noise.
    loglr :
        Returns the log of the likelihood ratio.
    logplr :
        Returns the log of the prior-weighted likelihood ratio.

    See ``BaseModel`` for additional attributes and properties.
    """
    def __init__(self, variable_params, data, recalibration=None, gates=None,
                 injection_file=None, no_save_data=False, **kwargs):
        self._data = None
        self.recalibration = recalibration
        self.no_save_data = no_save_data
        self.gates = gates
        self.injection_file = injection_file
        super(BaseDataModel, self).__init__(variable_params, **kwargs)

    @property
        """Dictionary mapping detector names to data."""

    @data.setter
    def data(self, data):
        """Store a copy of the data."""
        self._data = {det: d.copy() for (det, d) in data.items()}

    @property
        """Adds ``loglr`` and ``lognl`` to the ``default_stats``."""
        return ['loglr', 'lognl']

    @property
    def lognl(self):
        """The log likelihood of the model assuming the data is noise.

        This will initially try to return the ``current_stats.lognl``.
        If that raises an ``AttributeError``, will call `_lognl`` to
        """
        return self._trytoget('lognl', self._lognl)

    def _lognl(self):
        """Low-level function that calculates the lognl."""
        pass

    @property
    def loglr(self):
        """The log likelihood ratio at the current parameters.

        This will initially try to return the ``current_stats.loglr``.
        If that raises an ``AttributeError``, will call `_loglr`` to
        """
        return self._trytoget('loglr', self._loglr, apply_transforms=True)

    def _loglr(self):
        """Low-level function that calculates the loglr."""
        pass

    @property
    def logplr(self):
        """Returns the log of the prior-weighted likelihood ratio at the
        current parameter values.

        (possibly indicating a non-physical point), then ``loglr`` is not
        called.
        """
        else:
            return logp + self.loglr

    @property
    def detectors(self):
        """Returns the detectors used."""
        return list(self._data.keys())

        """Adds data to the metadata that's written.

        """
        super(BaseDataModel, self).write_metadata(fp)
        if not self.no_save_data:
            fp.write_stilde(self.data)
        # save injection parameters
        if self.injection_file is not None:
            fp.write_injections(self.injection_file)

# Copyright (C) 2020 Alex Nitz
#
#

"""This module provides model classes that do brute force marginalization
using at the likelihood level.
"""

from multiprocessing import Pool
from .gaussian_noise import BaseGaussianNoise
from scipy.special import logsumexp

_model = None
class likelihood_wrapper(object):
    def __init__(self, model):
        global _model
        _model = model

        _model.update(**params)
        loglr = _model.loglr
        return loglr, _model.current_stats

class BruteParallelGaussianMarginalize(BaseGaussianNoise):
    name = "brute_parallel_gaussian_marginalize"

    def __init__(self, variable_params,
                 cores=10,
                 base_model=None,
                 marginalize_phase=None,
                 **kwds):
        super(BruteParallelGaussianMarginalize, self).__init__(variable_params,
                                     **kwds)

        from pycbc.inference.models import models
        self.model = models[base_model](variable_params, **kwds)

        self.call = likelihood_wrapper(self.model)

        # size of pool for each likelihood call
        self.pool = Pool(int(cores))

        # Only one for now, but can be easily extended
        self.phase = None
        if marginalize_phase:
            samples = int(marginalize_phase)
            self.phase = numpy.linspace(0, 2.0 * numpy.pi, samples)

    @property
        stats = self.model._extra_stats
        stats.append('maxl_phase')
        if 'maxl_loglr' not in stats:
            stats.append('maxl_loglr')
        return stats

        if self.phase is not None:
            params = []
            for p in self.phase:
                pref = self.current_params.copy()
                pref['coa_phase'] = p
                params.append(pref)
            vals = list(self.pool.map(self.call, params))
            loglr = numpy.array([v[0] for v in vals])
            # get the maxl values
            if 'maxl_loglr' not in self.model._extra_stats:
                maxl_loglrs = loglr
            else:
                maxl_loglrs = numpy.array([v[1]['maxl_loglr'] for v in vals])
            maxidx = maxl_loglrs.argmax()
            maxstats = vals[maxidx][1]
            maxphase = self.phase[maxidx]
            # set the stats
            for stat in maxstats:
                setattr(self._current_stats, stat, maxstats[stat])
            self._current_stats.maxl_phase = maxphase
            self._current_stats.maxl_loglr = maxl_loglrs[maxidx]
            # calculate the marginal loglr and return
            return logsumexp(loglr) - numpy.log(len(self.phase))

#
#

"""Utilities for loading data for models.
"""

from argparse import ArgumentParser
from time import sleep
try:
    from mpi4py import MPI
    MPI = None

from pycbc.psd import (insert_psd_option_group_multi_ifo,
                       from_cli_multi_ifos as psd_from_cli_multi_ifos,
                       verify_psd_options_multi_ifo)
from pycbc import strain
from pycbc.strain import (gates_from_cli, psd_gates_from_cli,
                          apply_gates_to_td, apply_gates_to_fd,
                          verify_strain_options_multi_ifo)
from pycbc import dq

def strain_from_cli_multi_ifos(*args, **kwargs):
    """Wrapper around strain.from_cli_multi_ifos that tries a few times before
    quiting.

    When running in a parallel environment, multiple concurrent queries to the
    segment data base can cause time out errors. If that happens, this will
    sleep for a few seconds, then try again a few times before giving up.
    """
    count = 0
    while count < 3:
        try:
            return strain.from_cli_multi_ifos(*args, **kwargs)
        except RuntimeError as e:
            exception = e
            count += 1
            sleep(10)
    # if get to here, we've tries 3 times and still got an error, so exit
    raise exception

#
#
#                   Utilities for gravitational-wave data
#
#
class NoValidDataError(Exception):
    """This should be raised if a continous segment of valid data could not be
    found.
    """
    pass

def create_data_parser():
    """Creates an argument parser for loading GW data."""
    parser = ArgumentParser()
    # add data options
    parser.add_argument("--instruments", type=str, nargs="+", required=True,
                        help="Instruments to analyze, eg. H1 L1.")
    parser.add_argument("--trigger-time", type=float, default=0.,
                        help="Reference GPS time (at geocenter) from which "
                             "the (anlaysis|psd)-(start|end)-time options are "
                             "measured. The integer seconds will be used. "
                             "Default is 0; i.e., if not provided, "
                             "the analysis and psd times should be in GPS "
                             "seconds.")
    parser.add_argument("--analysis-start-time", type=int, required=True,
                        nargs='+', action=MultiDetOptionAction,
                        metavar='IFO:TIME',
                        help="The start time to use for the analysis, "
                             "measured with respect to the trigger-time. "
                             "If psd-inverse-length is provided, the given "
                             "start time will be padded by half that length "
                             "to account for wrap-around effects.")
    parser.add_argument("--analysis-end-time", type=int, required=True,
                        help="The end time to use for the analysis, "
                             "end time will be padded by half that length "
    parser.add_argument("--psd-start-time", type=int, default=None,
                        help="Start time to use for PSD estimation, measured "
                             "with respect to the trigger-time.")
    parser.add_argument("--psd-end-time", type=int, default=None,
                        help="End time to use for PSD estimation, measured "
    parser.add_argument("--data-conditioning-low-freq", type=float,
                        nargs="+", action=MultiDetOptionAction,
                        metavar='IFO:FLOW', dest="low_frequency_cutoff",
                        help="Low frequency cutoff of the data. Needed for "
                             "PSD estimation and when creating fake strain. "
                             "If not provided, will use the model's "
                             "low-frequency-cutoff.")
    insert_psd_option_group_multi_ifo(parser)
    strain.insert_strain_option_group_multi_ifo(parser, gps_times=False)
    strain.add_gate_option_group(parser)
    # add arguments for dq
    dqgroup = parser.add_argument_group("Options for quering data quality "
                                        "(DQ)")
    dqgroup.add_argument('--dq-segment-name', default='DATA',
                         help='The status flag to query for data quality. '
                              'Default is "DATA".')
    dqgroup.add_argument('--dq-source', choices=['any', 'GWOSC', 'dqsegdb'],
                         default='any',
                         help='Where to look for DQ information. If "any" '
                              '(the default) will first try GWOSC, then '
                              'dqsegdb.')
    dqgroup.add_argument('--dq-server', default='https://segments.ligo.org',
                         help='The server to use for dqsegdb.')
    dqgroup.add_argument('--veto-definer', default=None,
                         help='Path to a veto definer file that defines '
                              'groups of flags, which themselves define a set '
                              'of DQ segments.')
    return parser

def check_validtimes(detector, gps_start, gps_end, shift_to_valid=False,
                     max_shift=None, segment_name='DATA',
                     **kwargs):
    r"""Checks DQ server to see if the given times are in a valid segment.

    If the ``shift_to_valid`` flag is provided, the times will be shifted left
    or right to try to find a continous valid block nearby. The shifting starts
    by shifting the times left by 1 second. If that does not work, it shifts
    the times right by one second. This continues, increasing the shift time by
    1 second, until a valid block could be found, or until the shift size
    exceeds ``max_shift``.

    If the given times are not in a continuous valid segment, or a valid block
    cannot be found nearby, a ``NoValidDataError`` is raised.

    detector : str
        The name of the detector to query; e.g., 'H1'.
    gps_start : int
        The GPS start time of the segment to query.
    gps_end : int
        The GPS end time of the segment to query.
    shift_to_valid : bool, optional
        If True, will try to shift the gps start and end times to the nearest
        continous valid segment of data. Default is False.
    max_shift : int, optional
        The maximum number of seconds to try to shift left or right to find
        a valid segment. Default is ``gps_end - gps_start``.
    segment_name : str, optional
        The status flag to query; passed to :py:func:`pycbc.dq.query_flag`.
        Default is "DATA".
        :py:func:`pycbc.dq.query_flag`.

    Returns
    -------
    use_start : int
        The start time to use. If ``shift_to_valid`` is True, this may differ
        from the given GPS start time.
    use_end : int
        The end time to use. If ``shift_to_valid`` is True, this may differ
        from the given GPS end time.
    """
    # expand the times checked encase we need to shift
    if max_shift is None:
        max_shift = int(gps_end - gps_start)
    check_start = gps_start - max_shift
    check_end = gps_end + max_shift
    # if we're running in an mpi enviornment and we're not the parent process,
    # we'll wait before quering the segment database. This will result in
    # getting the segments from the cache, so as not to overload the database
    if MPI is not None and (MPI.COMM_WORLD.Get_size() > 1 and
                            MPI.COMM_WORLD.Get_rank() != 0):
        # we'll wait for 2 minutes
        sleep(120)
    validsegs = dq.query_flag(detector, segment_name, check_start,
                              check_end, cache=True,
                              **kwargs)
    use_start = gps_start
    use_end = gps_end
    # shift if necessary
    if shift_to_valid:
        shiftsize = 1
        while (use_start, use_end) not in validsegs and shiftsize < max_shift:
            # try shifting left
            use_start = gps_start - shiftsize
            use_end = gps_end - shiftsize
            if (use_start, use_end) not in validsegs:
                # try shifting right
                use_start = gps_start + shiftsize
                use_end = gps_end + shiftsize
            shiftsize += 1
    # check that we have a valid range
    if (use_start, use_end) not in validsegs:
        raise NoValidDataError("Could not find a continous valid segment in "
                               "in detector {}".format(detector))
    return use_start, use_end

def detectors_with_valid_data(detectors, gps_start_times, gps_end_times,
                              pad_data=None, err_on_missing_detectors=False,
                              **kwargs):
    r"""Determines which detectors have valid data.

    detectors : list of str
        Names of the detector names to check.
    gps_start_times : dict
        Dictionary of detector name -> start time listing the GPS start times
        of the segment to check for each detector.
    gps_end_times : dict
        Dictionary of detector name -> end time listing the GPS end times of
        the segment to check for each detector.
    pad_data : dict, optional
        Dictionary of detector name -> pad time to add to the beginning/end of
        the GPS start/end times before checking. A pad time for every detector
        in ``detectors`` must be given. Default (None) is to apply no pad to
        the times.
    err_on_missing_detectors : bool, optional
        If True, a ``NoValidDataError`` will be raised if any detector does not
        have continous valid data in its requested segment. Otherwise, the
        detector will not be included in the returned list of detectors with
        valid data. Default is False.
        All other keyword arguments are passed to ``check_validtimes``.

    Returns
    -------
    dict :
        A dictionary of detector name -> valid times giving the detectors with
        valid data and their segments. If ``shift_to_valid`` was passed to
        ``check_validtimes`` this may not be the same as the input segments. If
        no valid times could be found for a detector (and
        ``err_on_missing_detectors`` is False), it will not be included in the
        returned dictionary.
    """
    if pad_data is None:
        pad_data = {det: 0 for det in detectors}
    dets_with_data = {}
    for det in detectors:
        logging.info("Checking that %s has valid data in the requested "
                     "segment", det)
        try:
            pad = pad_data[det]
            start, end = check_validtimes(det, gps_start_times[det]-pad,
                                          gps_end_times[det]+pad,
                                          **kwargs)
            dets_with_data[det] = (start+pad, end-pad)
        except NoValidDataError as e:
            if err_on_missing_detectors:
                raise e
            logging.warning("WARNING: Detector %s will not be used in "
                            "the analysis, as it does not have "
                            "continuous valid data that spans the "
                            "segment [%d, %d).", det, gps_start_times[det]-pad,
                            gps_end_times[det]+pad)
    return dets_with_data

def check_for_nans(strain_dict):
    """Checks if any data in a dictionary of strains has NaNs.

    If any NaNs are found, a ``ValueError`` is raised.

    strain_dict : dict
        Dictionary of detectors ->
        :py:class:`pycbc.types.timeseries.TimeSeries`.
    """
    for det, ts in strain_dict.items():
        if numpy.isnan(ts.numpy()).any():
            raise ValueError("NaN found in strain from {}".format(det))

def data_opts_from_config(cp, section, filter_flow):
    """Loads data options from a section in a config file.

        Config file to read.
        The section to read. All options in the section will be loaded as-if
        they wre command-line arguments.
    filter_flow : dict
        Dictionary of detectors -> inner product low frequency cutoffs.
        If a `data-conditioning-low-freq` cutoff wasn't provided for any
        of the detectors, these values will be used. Otherwise, the
        data-conditioning-low-freq must be less than the inner product cutoffs.
        If any are not, a ``ValueError`` is raised.

    Returns
    -------
    opts : parsed argparse.ArgumentParser
        An argument parser namespace that was constructed as if the options
        were specified on the command line.
    """
    # convert the section options into a command-line options
    optstr = cp.section_to_cli(section)
    # create a fake parser to parse them
    parser = create_data_parser()
    # parse the options
    opts = parser.parse_args(optstr.split())
    # figure out the times to use
    logging.info("Determining analysis times to use")
    opts.trigger_time = int(opts.trigger_time)
    gps_start = opts.analysis_start_time.copy()
    gps_end = opts.analysis_end_time.copy()
    for det in opts.instruments:
        gps_start[det] += opts.trigger_time
        gps_end[det] += opts.trigger_time
        if opts.psd_inverse_length[det] is not None:
            pad = int(numpy.ceil(opts.psd_inverse_length[det] / 2))
            logging.info("Padding %s analysis start and end times by %d "
                         "(= psd-inverse-length/2) seconds to "
                         "account for PSD wrap around effects.",
                         det, pad)
        else:
            pad = 0
        gps_start[det] -= pad
        gps_end[det] += pad
        if opts.psd_start_time[det] is not None:
            opts.psd_start_time[det] += opts.trigger_time
        if opts.psd_end_time[det] is not None:
            opts.psd_end_time[det] += opts.trigger_time
    opts.gps_start_time = gps_start
    opts.gps_end_time = gps_end
    # check for the frequencies
    low_freq_cutoff = filter_flow.copy()
    if opts.low_frequency_cutoff:
        # add in any missing detectors
        low_freq_cutoff.update({det: opts.low_frequency_cutoff[det]
                                for det in opts.instruments
                                if opts.low_frequency_cutoff[det] is not None})
        # make sure the data conditioning low frequency cutoff is < than
        # the matched filter cutoff
        if any(low_freq_cutoff[det] > filter_flow[det] for det in filter_flow):
            raise ValueError("data conditioning low frequency cutoff must "
                             "be less than the filter low frequency "
                             "cutoff")
    opts.low_frequency_cutoff = low_freq_cutoff

    # verify options are sane
    verify_psd_options_multi_ifo(opts, parser, opts.instruments)
    verify_strain_options_multi_ifo(opts, parser, opts.instruments)
    return opts

def data_from_cli(opts, check_for_valid_times=False,
                  shift_psd_times_to_valid=False,
                  err_on_missing_detectors=False):
    """Loads the data needed for a model from the given command-line options.

    Gates specifed on the command line are also applied.

    opts : ArgumentParser parsed args
        Argument options parsed from a command line string (the sort of thing
        returned by `parser.parse_args`).
    check_for_valid_times : bool, optional
        Check that valid data exists in the requested gps times. Default is
        False.
    shift_psd_times_to_valid : bool, optional
        If estimating the PSD from data, shift the PSD times to a valid
        segment if needed. Default is False.
        Raise a NoValidDataError if any detector does not have valid data.
        Otherwise, a warning is printed, and that detector is skipped.

    Returns
    -------
        Dictionary of detectors -> time series strain.
    psd_strain_dict : dict or None
        If ``opts.psd_(start|end)_time`` were set, a dctionary of
        detectors -> time series data to use for PSD estimation. Otherwise,
        ``None``.
    """
    # get gates to apply
    gates = gates_from_cli(opts)
    psd_gates = psd_gates_from_cli(opts)

    # get strain time series
    instruments = opts.instruments

    # validate times
    if check_for_valid_times:
        dets_with_data = detectors_with_valid_data(
            instruments, opts.gps_start_time, opts.gps_end_time,
            pad_data=opts.pad_data,
            err_on_missing_detectors=err_on_missing_detectors,
            shift_to_valid=False,
            segment_name=opts.dq_segment_name, source=opts.dq_source,
            server=opts.dq_server, veto_definer=opts.veto_definer)
        # reset instruments to only be those with valid data
        instruments = list(dets_with_data.keys())

    strain_dict = strain_from_cli_multi_ifos(opts, instruments,
                                             precision="double")
    # apply gates if not waiting to overwhiten
    if not opts.gate_overwhitened:
        strain_dict = apply_gates_to_td(strain_dict, gates)

    # check that there aren't nans in the data
    check_for_nans(strain_dict)

    # get strain time series to use for PSD estimation
    # if user has not given the PSD time options then use same data as analysis
    if opts.psd_start_time and opts.psd_end_time:
        logging.info("Will generate a different time series for PSD "
                     "estimation")
        if check_for_valid_times:
            psd_times = detectors_with_valid_data(
                instruments, opts.psd_start_time, opts.psd_end_time,
                pad_data=opts.pad_data,
                err_on_missing_detectors=err_on_missing_detectors,
                shift_to_valid=shift_psd_times_to_valid,
                segment_name=opts.dq_segment_name, source=opts.dq_source,
                server=opts.dq_server, veto_definer=opts.veto_definer)
            # remove detectors from the strain dict that did not have valid
            # times for PSD estimation
            for det in set(strain_dict.keys())-set(psd_times.keys()):
                _ = strain_dict.pop(det)
            # reset instruments to only be those with valid data
            instruments = list(psd_times.keys())
        else:
            psd_times = {det: (opts.psd_start_time[det],
                               opts.psd_end_time[det])
                         for det in instruments}
        psd_strain_dict = {}
        for det, (psd_start, psd_end) in psd_times.items():
            opts.gps_start_time = psd_start
            opts.gps_end_time = psd_end
            psd_strain_dict.update(
                strain_from_cli_multi_ifos(opts, [det], precision="double"))
        # apply any gates
        logging.info("Applying gates to PSD data")
        psd_strain_dict = apply_gates_to_td(psd_strain_dict, psd_gates)
        # check that there aren't nans in the psd data
        check_for_nans(psd_strain_dict)
    elif opts.psd_start_time or opts.psd_end_time:
        raise ValueError("Must give psd-start-time and psd-end-time")
    else:
        psd_strain_dict = None

    # check that we have data left to analyze
    if instruments == []:
        raise NoValidDataError("No valid data could be found in any of the "
                               "requested instruments.")

    return strain_dict, psd_strain_dict

def fd_data_from_strain_dict(opts, strain_dict, psd_strain_dict=None):
    """Converts a dictionary of time series to the frequency domain, and gets
    the PSDs.

        Dictionary of detectors -> time series data.
    psd_strain_dict : dict, optional
        Dictionary of detectors -> time series data to use for PSD estimation.
        If not provided, will use the ``strain_dict``. This is
        ignored if ``opts.psd_estimation`` is not set. See
        :py:func:`pycbc.psd.psd_from_cli_multi_ifos` for details.

    Returns
    -------
    stilde_dict : dict
        Dictionary of detectors -> frequency series data.
    psd_dict : dict
        Dictionary of detectors -> frequency-domain PSDs.
    """
    # FFT strain and save each of the length of the FFT, delta_f, and
    # low frequency cutoff to a dict
    stilde_dict = {}
    length_dict = {}
    delta_f_dict = {}
    for det, tsdata in strain_dict.items():
        stilde_dict[det] = tsdata.to_frequencyseries()
        length_dict[det] = len(stilde_dict[det])
        delta_f_dict[det] = stilde_dict[det].delta_f

    if psd_strain_dict is None:
        psd_strain_dict = strain_dict

    # get PSD as frequency series
    psd_dict = psd_from_cli_multi_ifos(
        opts, length_dict, delta_f_dict, opts.low_frequency_cutoff,
        list(psd_strain_dict.keys()), strain_dict=psd_strain_dict,
        precision="double")

    return stilde_dict, psd_dict

def gate_overwhitened_data(stilde_dict, psd_dict, gates):
    """Applies gates to overwhitened data.

        Dictionary of detectors -> frequency series data to apply the gates to.
        Dictionary of detectors -> PSD to use for overwhitening.
    gates : dict
        Dictionary of detectors -> gates.

    Returns
    -------
    dict :
        Dictionary of detectors -> frequency series data with the gates
        applied after overwhitening. The returned data is not overwhitened.
    """
    logging.info("Applying gates to overwhitened data")
    # overwhiten the data
    out = {}
    for det in gates:
        out[det] = stilde_dict[det] / psd_dict[det]
    # now apply the gate
    out = apply_gates_to_fd(out, gates)
    # now unwhiten
        out[det] *= psd_dict[det]
    return out

#
#

"""This module provides model classes that assume the noise is Gaussian.
"""

import shlex
from abc import ABCMeta

from pycbc import filter as pyfilter
from pycbc.waveform import (NoWaveformError, FailedWaveformError)
from pycbc.waveform import generator
from pycbc.types import Array, FrequencySeries
from pycbc.strain import gates_from_cli
from pycbc.strain.calibration import Recalibrate
from pycbc.types.optparse import MultiDetOptionAction

from .base import ModelStats
from .base_data import BaseDataModel
from .data_utils import (data_opts_from_config, data_from_cli,
                         fd_data_from_strain_dict, gate_overwhitened_data)

class BaseGaussianNoise(BaseDataModel):
    r"""Model for analyzing GW data with assuming a wide-sense stationary
    Gaussian noise model.

    This model will load gravitational wave data and calculate the log noise
    likelihood ``_lognl`` and normalization. It also implements the
    ``_loglikelihood`` function as the sum of the log likelihood ratio and the
    ``lognl``. It does not implement a log likelihood ratio function
    ``_loglr``, however, since that can differ depending on the signal model.
    Models that analyze GW data assuming it is stationary Gaussian should
    therefore inherit from this class and implement their own ``_loglr``
    function.

    For more details on the inner product used, the log likelihood of the
    noise, and the normalization factor, see :py:class:`GaussianNoise`.

    data : dict
        values are the data (assumed to be unwhitened). All data must have the
        same frequency resolution.
    low_frequency_cutoff : dict
        A dictionary of starting frequencies, in which the keys are the
        detector names and the values are the starting frequencies for the
        respective detectors to be used for computing inner products.
    psds : dict, optional
        A dictionary of FrequencySeries keyed by the detector names. The
        dictionary must have a psd for each detector specified in the data
        dictionary. If provided, the inner products in each detector will be
        weighted by 1/psd of that detector.
    high_frequency_cutoff : dict, optional
        A dictionary of ending frequencies, in which the keys are the
        detector names and the values are the ending frequencies for the
        respective detectors to be used for computing inner products. If not
        provided, the minimum of the largest frequency stored in the data
        and a given waveform will be used.
    normalize : bool, optional
        If True, the normalization factor :math:`alpha` will be included in the
        log likelihood. See :py:class:`GaussianNoise` for details. Default is
        to not include it.
    ignore_failed_waveforms : bool, optional
        If the waveform generator raises an error when it tries to generate,
        treat the point as having zero likelihood. This allows the parameter
        estimation to continue. Otherwise, an error will be raised, stopping
        the run. Default is False.
        All other keyword arguments are passed to ``BaseDataModel``.

    data : dict
        Dictionary of detectors -> frequency-domain data.
    ignore_failed_waveforms : bool
        If True, points in parameter space that cause waveform generation to
        fail (i.e., they raise a ``FailedWaveformError``) will be treated as
        points with zero likelihood. Otherwise, such points will cause the
        model to raise a ``FailedWaveformError``.
    low_frequency_cutoff
    high_frequency_cutoff
    kmin
    kmax
    psds
    psd_segments
    weight
    whitened_data
    normalize
    lognorm
    """
    def __init__(self, variable_params, data, low_frequency_cutoff, psds=None,
                 high_frequency_cutoff=None, normalize=False,
                 static_params=None, ignore_failed_waveforms=False,
                 no_save_data=False,
        # set up the boiler-plate attributes
        super(BaseGaussianNoise, self).__init__(variable_params, data,
                                                static_params=static_params,
                                                no_save_data=no_save_data,
                                                **kwargs)
        self.ignore_failed_waveforms = ignore_failed_waveforms

        # check if low frequency cutoff has been provided for every IFO with
        # data
        for ifo in self.data:
            if low_frequency_cutoff[ifo] is None:
                    "A low-frequency-cutoff must be provided for every "
                    "detector for which data has been provided. If "
                    "loading the model settings from "
                    "a config file, please provide "
                    "`{DETECTOR}:low-frequency-cutoff` options for "
                    "every detector in the `[model]` section, where "
                    "`{DETECTOR} is the name of the detector,"
                    "or provide a single low-frequency-cutoff option"
                    "which will be used for all detectors")

        # check that the data sets all have the same delta fs and delta ts
        dts = numpy.array([d.delta_t for d in self.data.values()])
        dfs = numpy.array([d.delta_f for d in self.data.values()])
        if not all(dts == dts[0]):
            raise ValueError("all data must have the same sample rate")
        if not all(dfs == dfs[0]):
            raise ValueError("all data must have the same segment length")

        # store the number of samples in the time domain
        self._N = int(1./(dts[0]*dfs[0]))

        # set lower/upper frequency cutoff
        if high_frequency_cutoff is None:
            high_frequency_cutoff = {ifo: None for ifo in self.data}
        self._f_upper = high_frequency_cutoff
        self._f_lower = low_frequency_cutoff

        # Set the cutoff indices
        self._kmin = {}
        self._kmax = {}

        for (det, d) in self._data.items():
            kmin, kmax = pyfilter.get_cutoff_indices(self._f_lower[det],
                                                     self._f_upper[det],
                                                     d.delta_f, self._N)
            self._kmin[det] = kmin
            self._kmax[det] = kmax

        # store the psd segments
        self._psd_segments = {}
        if psds is not None:
            self.set_psd_segments(psds)

        # store the psds and calculate the inner product weight
        self._psds = {}
        self._weight = {}
        self._det_lognls = {}
        self._whitened_data = {}

        # set the normalization state
        self._normalize = False
        self.normalize = normalize
        # store the psds and whiten the data
        self.psds = psds

    @property
    def high_frequency_cutoff(self):
        """The high frequency cutoff of the inner product."""
        return self._f_upper

    @property
    def low_frequency_cutoff(self):
        """The low frequency cutoff of the inner product."""
        return self._f_lower

    @property
    def kmin(self):
        """Dictionary of starting indices for the inner product.

        This is determined from the lower frequency cutoff and the ``delta_f``
        of the data using
        :py:func:`pycbc.filter.matchedfilter.get_cutoff_indices`.
        """
        return self._kmin

    @property
    def kmax(self):
        """Dictionary of ending indices for the inner product.

        This is determined from the high frequency cutoff and the ``delta_f``
        :py:func:`pycbc.filter.matchedfilter.get_cutoff_indices`. If no high
        frequency cutoff was provided, this will be the indice corresponding to
        the Nyquist frequency.
        """
        return self._kmax

    @property
    def psds(self):
        """Dictionary of detectors -> PSD frequency series.

        If no PSD was provided for a detector, this will just be a frequency
        series of ones.
        """
        return self._psds

    @psds.setter
    def psds(self, psds):
        """Sets the psds, and calculates the weight and norm from them.

        The data and the low and high frequency cutoffs must be set first.
        """
        # check that the data has been set
        if self._data is None:
            raise ValueError("No data set")
        if self._f_lower is None:
            raise ValueError("low frequency cutoff not set")
        if self._f_upper is None:
            raise ValueError("high frequency cutoff not set")
        # make sure the relevant caches are cleared
        self._psds.clear()
        self._weight.clear()
        self._lognorm.clear()
        self._det_lognls.clear()
        self._whitened_data.clear()
        for det, d in self._data.items():
            if psds is None:
                # No psd means assume white PSD
                p = FrequencySeries(numpy.ones(int(self._N/2+1)),
                                    delta_f=d.delta_f)
            else:
                # copy for storage
                p = psds[det].copy()
            self._psds[det] = p
            # we'll store the weight to apply to the inner product
            w = Array(numpy.zeros(len(p)))
            # only set weight in band we will analyze
            kmin = self._kmin[det]
            kmax = self._kmax[det]
            w[kmin:kmax] = numpy.sqrt(4.*p.delta_f/p[kmin:kmax])
            self._weight[det] = w
            self._whitened_data[det] = d.copy()
            self._whitened_data[det][kmin:kmax] *= w[kmin:kmax]
        # set the lognl and lognorm; we'll get this by just calling lognl
        _ = self.lognl

    @property
    def psd_segments(self):
        """Dictionary giving times used for PSD estimation for each detector.

        If a detector's PSD was not estimated from data, or the segment wasn't
        provided, that detector will not be in the dictionary.
        """
        return self._psd_segments

    def set_psd_segments(self, psds):
        """Sets the PSD segments from a dictionary of PSDs.

        This attempts to get the PSD segment from a ``psd_segment`` attribute
        of each detector's PSD frequency series. If that attribute isn't set,
        then that detector is not added to the dictionary of PSD segments.

        psds : dict
            Dictionary of detector name -> PSD frequency series. The segment
            used for each PSD will try to be retrieved from the PSD's
            ``.psd_segment`` attribute.
        """
        for det, p in psds.items():
            try:
                self._psd_segments[det] = p.psd_segment
                continue

    @property
    def weight(self):
        r"""Dictionary of detectors -> frequency series of inner-product
        weights.

        The weights are :math:`\sqrt{4 \Delta f / S_n(f)}`. This is set when
        the PSDs are set.
        """
        return self._weight

    @property
    def whitened_data(self):
        r"""Dictionary of detectors -> whitened data frequency series.

        The whitened data is the data multiplied by the inner-product weight.
        Note that this includes the :math:`\sqrt{4 \Delta f}` factor. This
        is set when the PSDs are set.
        """
        return self._whitened_data

    def det_lognorm(self, det):
        """The log of the likelihood normalization in the given detector.

        If ``self.normalize`` is False, will just return 0.
        """
        if not self.normalize:
            return 0.
        try:
            return self._lognorm[det]
            # hasn't been calculated yet
            p = self._psds[det]
            dt = self._whitened_data[det].delta_t
            lognorm = -float(self._N*numpy.log(numpy.pi*self._N*dt)/2.
                             + numpy.log(p[kmin:kmax]).sum())
            self._lognorm[det] = lognorm

    @property
    def normalize(self):
        """Determines if the loglikelihood includes the normalization term.
        """
        return self._normalize

    @normalize.setter
    def normalize(self, normalize):
        """Clears the current stats if the normalization state is changed.
        """
        if normalize != self._normalize:
            self._current_stats = ModelStats()
            self._lognorm.clear()
            self._det_lognls.clear()
        self._normalize = normalize

    @property
        """The log of the normalization of the log likelihood."""
        return sum(self.det_lognorm(det) for det in self._data)

    def det_lognl(self, det):
        r"""Returns the log likelihood of the noise in the given detector:

        .. math::

            \log p(d_i|n_i) = \log \alpha_i -
                \frac{1}{2} \left<d_i | d_i\right>.

        det : str
            The name of the detector.

        Returns
        -------
        float :
            The log likelihood of the noise in the requested detector.
        """
        try:
            return self._det_lognls[det]
            # hasn't been calculated yet; calculate & store
            d = self._whitened_data[det]
            lognorm = self.det_lognorm(det)
            lognl = lognorm - 0.5 * d[kmin:kmax].inner(d[kmin:kmax]).real
            self._det_lognls[det] = lognl

        """Computes the log likelihood assuming the data is noise.

        Since this is a constant for Gaussian noise, this is only computed once
        then stored.
        """
        return sum(self.det_lognl(det) for det in self._data)

        r"""Computes the log likelihood of the paramaters,

        .. math::

            \log p(d|\Theta, h) = \log \alpha -\frac{1}{2}\sum_i
                \left<d_i - h_i(\Theta) | d_i - h_i(\Theta)\right>,

        at the current parameter values :math:`\Theta`.

        Returns
        -------
        float
            The value of the log likelihood evaluated at the given point.
        """
        # since the loglr has fewer terms, we'll call that, then just add
        # back the noise term that canceled in the log likelihood ratio
        return self.loglr + self.lognl

        """Adds writing the psds and lognl, since it's a constant.

        The lognl is written to the sample group's ``attrs``.

        The analyzed detectors, their analysis segments, and the segments
        used for psd estimation are written to the file's ``attrs``, as
        ``analyzed_detectors``, ``{{detector}}_analysis_segment``, and
        ``{{detector}}_psd_segment``, respectively.

        """
        super(BaseGaussianNoise, self).write_metadata(fp)
        # write the analyzed detectors and times
        fp.attrs['analyzed_detectors'] = self.detectors
        for det, data in self.data.items():
            key = '{}_analysis_segment'.format(det)
            fp.attrs[key] = [float(data.start_time), float(data.end_time)]
        if self._psds is not None and not self.no_save_data:
            fp.write_psd(self._psds)
        # write the times used for psd estimation (if they were provided)
        for det in self.psd_segments:
            key = '{}_psd_segment'.format(det)
            fp.attrs[key] = list(map(float, self.psd_segments[det]))
        try:
            attrs = fp[fp.samples_group].attrs
            # group doesn't exist, create it
            fp.create_group(fp.samples_group)
        attrs['lognl'] = self.lognl
        for det in self.detectors:
            # Save lognl for each IFO as attributes in the samples group
            attrs['{}_lognl'.format(det)] = self.det_lognl(det)
            # Save each IFO's low frequency cutoff used in the likelihood
            # computation as an attribute
            fp.attrs['{}_likelihood_low_freq'.format(det)] = self._f_lower[det]
            # Save the IFO's high frequency cutoff used in the likelihood
            # computation as an attribute if one was provided the user
            if self._f_upper[det] is not None:
                fp.attrs['{}_likelihood_high_freq'.format(det)] = \
                                                        self._f_upper[det]

    def from_config(cls, cp, data_section='data', data=None, psds=None,
                    **kwargs):
        r"""Initializes an instance of this class from the given config file.

        In addition to ``[model]``, a ``data_section`` (default ``[data]``)
        must be in the configuration file. The data section specifies settings
        for loading data and estimating PSDs. See the `online documentation
        <http://pycbc.org/pycbc/latest/html/inference.html#setting-data>`_ for
        more details.

        The following options are read from the ``[model]`` section, in
        addition to ``name`` (which must be set):

        * ``{{DET}}-low-frequency-cutoff = FLOAT`` :
          The low frequency cutoff to use for each detector {{DET}}. A cutoff
          must be provided for every detector that may be analyzed (any
          additional detectors are ignored).
        * ``{{DET}}-high-frequency-cutoff = FLOAT`` :
          (Optional) A high frequency cutoff for each detector. If not
          provided, the Nyquist frequency is used.
        * ``check-for-valid-times =`` :
          (Optional) If provided, will check that there are no data quality
          flags on during the analysis segment and the segment used for PSD
          estimation in each detector. To check for flags,
          :py:func:`pycbc.dq.query_flag` is used, with settings pulled from the
          ``dq-*`` options in the ``[data]`` section. If a detector has bad
          data quality during either the analysis segment or PSD segment, it
          will be removed from the analysis.
        * ``shift-psd-times-to-valid =`` :
          (Optional) If provided, the segment used for PSD estimation will
          automatically be shifted left or right until a continous block of
          data with no data quality issues can be found. If no block can be
          found with a maximum shift of +/- the requested psd segment length,
          the detector will not be analyzed.
        * ``err-on-missing-detectors =`` :
          Raises an error if any detector is removed from the analysis because
          a valid time could not be found. Otherwise, a warning is printed
          to screen and the detector is removed from the analysis.
        * ``normalize =`` :
          (Optional) Turn on the normalization factor.
        * ``ignore-failed-waveforms =`` :
          Sets the ``ignore_failed_waveforms`` attribute.

        data_section : str, optional
            The name of the section to load data options from.
        """
        # get the injection file, to replace any FROM_INJECTION settings
        if 'injection-file' in cp.options('data'):
            injection_file = cp.get('data', 'injection-file')
        else:
            injection_file = None
        # update any values that are to be retrieved from the injection
        # Note: this does nothing if there are FROM_INJECTION values
        get_values_from_injection(cp, injection_file, update_cp=True)
        # add the injection file
        args['injection_file'] = injection_file
        # check if normalize is set
        if cp.has_option('model', 'normalize'):
            args['normalize'] = True
        if cp.has_option('model', 'ignore-failed-waveforms'):
            args['ignore_failed_waveforms'] = True
        if cp.has_option('model', 'no-save-data'):
            args['no_save_data'] = True
        ignore_args = ['name', 'normalize',
                       'ignore-failed-waveforms', 'no-save-data']
        for option in cp.options("model"):
            if option in ("low-frequency-cutoff", "high-frequency-cutoff"):
                ignore_args.append(option)
                name = option.replace('-', '_')
                args[name] = cp.get_cli_option('model', name,
                                               nargs='+', type=float,
                                               action=MultiDetOptionAction)

        if 'low_frequency_cutoff' not in args:
            raise ValueError("low-frequency-cutoff must be provided in the"
                             " model section, but is not found!")

        # data args
        bool_args = ['check-for-valid-times', 'shift-psd-times-to-valid',
                     'err-on-missing-detectors']
        data_args = {arg.replace('-', '_'): True for arg in bool_args
                     if cp.has_option('model', arg)}
        ignore_args += bool_args
        # load the data
        opts = data_opts_from_config(cp, data_section,
                                     args['low_frequency_cutoff'])
        if data is None or psds is None:
            strain_dict, psd_strain_dict = data_from_cli(opts, **data_args)
            # convert to frequency domain and get psds
            stilde_dict, psds = fd_data_from_strain_dict(opts, strain_dict,
                                                         psd_strain_dict)
            # save the psd data segments if the psd was estimated from data
            if opts.psd_estimation is not None:
                _tdict = psd_strain_dict or strain_dict
                for det in psds:
                    psds[det].psd_segment = (_tdict[det].start_time,
                                             _tdict[det].end_time)
            # gate overwhitened if desired
            if opts.gate_overwhitened and opts.gate is not None:
                stilde_dict = gate_overwhitened_data(stilde_dict, psds, opts.gate)
            data = stilde_dict
        args.update({'data': data, 'psds': psds})
        # any extra args
                                               skip_args=ignore_args))
        # get ifo-specific instances of calibration model
        if cp.has_section('calibration'):
            logging.info("Initializing calibration model")
            recalib = {
                ifo: Recalibrate.from_config(cp, ifo, section='calibration')
                for ifo in opts.instruments}
            args['recalibration'] = recalib
        # get gates for templates
        gates = gates_from_cli(opts)
        if gates:
            args['gates'] = gates

class GaussianNoise(BaseGaussianNoise):
    r"""Model that assumes data is stationary Gaussian noise.

    With Gaussian noise the log likelihood functions for signal
    :math:`\log p(d|\Theta, h)` and for noise :math:`\log p(d|n)` are given by:

    .. math::

        \log p(d|\Theta, h) &=  \log\alpha -\frac{1}{2} \sum_i
            \left< d_i - h_i(\Theta) | d_i - h_i(\Theta) \right> \\
        \log p(d|n) &= \log\alpha -\frac{1}{2} \sum_i \left<d_i | d_i\right>

    where the sum is over the number of detectors, :math:`d_i` is the data in
    each detector, and :math:`h_i(\Theta)` is the model signal in each
    detector. The (discrete) inner product is given by:

    .. math::

        \left<a_i | b_i\right> = 4\Re \Delta f
            \sum_{k=k_{\mathrm{min}}}^{k_{\mathrm{max}}}
            \frac{\tilde{a}_i^{*}[k] \tilde{b}_i[k]}{S^{(i)}_n[k]},

    where :math:`\Delta f` is the frequency resolution (given by 1 / the
    observation time :math:`T`), :math:`k` is an index over the discretely
    sampled frequencies :math:`f = k \Delta_f`, and :math:`S^{(i)}_n[k]` is the
    PSD in the given detector. The upper cutoff on the inner product
    :math:`k_{\max}` is by default the Nyquist frequency
    :math:`k_{\max} = N/2+1`, where :math:`N = \lfloor T/\Delta t \rfloor`
    is the number of samples in the time domain, but this can be set manually
    to a smaller value.

    The normalization factor :math:`\alpha` is:

    .. math::

        \alpha = \prod_{i} \frac{1}{\left(\pi T\right)^{N/2}
            \prod_{k=k_\mathrm{min}}^{k_{\mathrm{max}}} S^{(i)}_n[k]},

    where the product is over the number of detectors. By default, the
    normalization constant is not included in the log likelihood, but it can
    be turned on using the ``normalize`` keyword argument.

    Note that the log likelihood ratio has fewer terms than the log likelihood,
    since the normalization and :math:`\left<d_i|d_i\right>` terms cancel:

    .. math::

        \log \mathcal{L}(\Theta) = \sum_i \left[
            \left<h_i(\Theta)|d_i\right> -
            \frac{1}{2} \left<h_i(\Theta)|h_i(\Theta)\right> \right]

    Upon initialization, the data is whitened using the given PSDs. If no PSDs
    are given the data and waveforms returned by the waveform generator are
    assumed to be whitened.

    For more details on initialization parameters and definition of terms, see
    :py:class:`models.BaseDataModel`.

    data : dict
        values are the data (assumed to be unwhitened). The list of keys must
        match the waveform generator's detectors keys, and the epoch of every
        data set must be the same as the waveform generator's epoch.
        log likelihood. Default is to not include it.

    Examples
    --------
    Create a signal, and set up the model using that signal:

    >>> from pycbc import psd as pypsd
    >>> from pycbc.inference.models import GaussianNoise
    >>> from pycbc.waveform.generator import (FDomainDetFrameGenerator,
    ...                                       FDomainCBCGenerator)
    >>> seglen = 4
    >>> sample_rate = 2048
    >>> N = seglen*sample_rate/2+1
    >>> fmin = 30.
    >>> static_params = {'approximant': 'IMRPhenomD', 'f_lower': fmin,
    ...                  'mass1': 38.6, 'mass2': 29.3,
    ...                  'spin1z': 0., 'spin2z': 0., 'ra': 1.37, 'dec': -1.26,
    ...                  'polarization': 2.76, 'distance': 3*500.}
    >>> variable_params = ['tc']
    >>> tsig = 3.1
    >>> generator = FDomainDetFrameGenerator(
    ...     FDomainCBCGenerator, 0., detectors=['H1', 'L1'],
    ...     variable_args=variable_params,
    ...     delta_f=1./seglen, **static_params)
    >>> signal = generator.generate(tc=tsig)
    >>> psd = pypsd.aLIGOZeroDetHighPower(N, 1./seglen, 20.)
    >>> psds = {'H1': psd, 'L1': psd}
    >>> low_frequency_cutoff = {'H1': fmin, 'L1': fmin}
    >>> model = GaussianNoise(variable_params, signal, low_frequency_cutoff,
                              psds=psds, static_params=static_params)

    Set the current position to the coalescence time of the signal:

    >>> model.update(tc=tsig)

    Now compute the log likelihood ratio and prior-weighted likelihood ratio;
    since we have not provided a prior, these should be equal to each other:

    >>> print('{:.2f}'.format(model.loglr))
    282.43
    >>> print('{:.2f}'.format(model.logplr))
    282.43

    Print all of the default_stats:

    >>> print(',\n'.join(['{}: {:.2f}'.format(s, v)
    ...                   for (s, v) in sorted(model.current_stats.items())]))
    H1_cplx_loglr: 177.76+0.00j,
    H1_optimal_snrsq: 355.52,
    L1_cplx_loglr: 104.67+0.00j,
    L1_optimal_snrsq: 209.35,
    logjacobian: 0.00,
    loglikelihood: 0.00,
    loglr: 282.43,
    logprior: 0.00

    Compute the SNR; for this system and PSD, this should be approximately 24:

    >>> from pycbc.conversions import snr_from_loglr
    >>> x = snr_from_loglr(model.loglr)
    >>> print('{:.2f}'.format(x))
    23.77

    Since there is no noise, the SNR should be the same as the quadrature sum
    of the optimal SNRs in each detector:

    >>> x = (model.det_optimal_snrsq('H1') +
    ...      model.det_optimal_snrsq('L1'))**0.5
    23.77

    Toggle on the normalization constant:

    >>> model.normalize = True
    >>> model.loglikelihood
    835397.8757405131

    Using the same model, evaluate the log likelihood ratio at several points
    in time and check that the max is at tsig:

    >>> times = numpy.linspace(tsig-1, tsig+1, num=101)
    >>> loglrs = numpy.zeros(len(times))
    >>> for (ii, t) in enumerate(times):
    ...     model.update(tc=t)
    ...     loglrs[ii] = model.loglr
    >>> print('tsig: {:.2f}, time of max loglr: {:.2f}'.format(
    ...     tsig, times[loglrs.argmax()]))
    tsig: 3.10, time of max loglr: 3.10

    Create a prior and use it (see distributions module for more details):

    >>> uniform_prior = distributions.Uniform(tc=(tsig-0.2,tsig+0.2))
    >>> prior = distributions.JointDistribution(variable_params, uniform_prior)
    >>> model = GaussianNoise(variable_params,
    ...     signal, low_frequency_cutoff, psds=psds, prior=prior,
    ...     static_params=static_params)
    283.35
    logprior: 0.92

    """
    name = 'gaussian_noise'

                 static_params=None, **kwargs):
        super(GaussianNoise, self).__init__(
            variable_params, data, low_frequency_cutoff, psds=psds,
            high_frequency_cutoff=high_frequency_cutoff, normalize=normalize,
            static_params=static_params, **kwargs)
        # create the waveform generator
        self.waveform_generator = create_waveform_generator(
            self.variable_params, self.data,
            waveform_transforms=self.waveform_transforms,
            recalibration=self.recalibration,
            gates=self.gates, **self.static_params)

    @property
        """Adds ``loglr``, plus ``cplx_loglr`` and ``optimal_snrsq`` in each
        detector."""
        return ['loglr'] + \
               ['{}_cplx_loglr'.format(det) for det in self._data] + \
               ['{}_optimal_snrsq'.format(det) for det in self._data]

    def _nowaveform_loglr(self):
        """Convenience function to set loglr values if no waveform generated.
        """
        for det in self._data:
            setattr(self._current_stats, 'loglikelihood', -numpy.inf)
            setattr(self._current_stats, '{}_cplx_loglr'.format(det),
                    -numpy.inf)
            # snr can't be < 0 by definition, so return 0
            setattr(self._current_stats, '{}_optimal_snrsq'.format(det), 0.)
        return -numpy.inf

        r"""Computes the log likelihood ratio,

        .. math::

            \log \mathcal{L}(\Theta) = \sum_i
                \left<h_i(\Theta)|d_i\right> -
                \frac{1}{2}\left<h_i(\Theta)|h_i(\Theta)\right>,

        Returns
        -------
        float
            The value of the log likelihood ratio.
        """
        params = self.current_params
        try:
            wfs = self.waveform_generator.generate(**params)
        except NoWaveformError:
            return self._nowaveform_loglr()
        except FailedWaveformError as e:
            if self.ignore_failed_waveforms:
                return self._nowaveform_loglr()
            else:
                raise e
        lr = 0.
        for det, h in wfs.items():
            # the kmax of the waveforms may be different than internal kmax
            kmax = min(len(h), self._kmax[det])
            if self._kmin[det] >= kmax:
                # if the waveform terminates before the filtering low frequency
                # cutoff, then the loglr is just 0 for this detector
                cplx_hd = 0j
                hh = 0.
            else:
                slc = slice(self._kmin[det], kmax)
                # whiten the waveform
                h[self._kmin[det]:kmax] *= self._weight[det][slc]
                # the inner products
                cplx_hd = self._whitened_data[det][slc].inner(h[slc])  # <h, d>
                hh = h[slc].inner(h[slc]).real  # < h, h>
            cplx_loglr = cplx_hd - 0.5*hh
            # store
            setattr(self._current_stats, '{}_optimal_snrsq'.format(det), hh)
                    cplx_loglr)
            lr += cplx_loglr.real
        # also store the loglikelihood, to ensure it is populated in the
        # current stats even if loglikelihood is never called
        self._current_stats.loglikelihood = lr + self.lognl
        return float(lr)

    def det_cplx_loglr(self, det):
        """Returns the complex log likelihood ratio in the given detector.

        det : str

        Returns
        -------
        complex float :
            The complex log likelihood ratio.
        """
        # try to get it from current stats
        try:
            return getattr(self._current_stats, '{}_cplx_loglr'.format(det))
            # hasn't been calculated yet; call loglr to do so
            self._loglr()
            # now try returning again

    def det_optimal_snrsq(self, det):
        """Returns the opitmal SNR squared in the given detector.

        det : str

        Returns
        -------
        float :
            The opimtal SNR squared.
        """
        try:
            return getattr(self._current_stats, '{}_optimal_snrsq'.format(det))

#
#
#                               Support functions
#
#

def get_values_from_injection(cp, injection_file, update_cp=True):
    """Replaces all FROM_INJECTION values in a config file with the
    corresponding value from the injection.

    This looks for any options that start with ``FROM_INJECTION[:ARG]`` in
    a config file. It then replaces that value with the corresponding value
    from the injection file. An argument may be optionally provided, in which
    case the argument will be retrieved from the injection file. Functions of
    parameters in the injection file may be used; the syntax and functions
    available is the same as the ``--parameters`` argument in executables
    such as ``pycbc_inference_extract_samples``. If no ``ARG`` is provided,
    then the option name will try to be retrieved from the injection.

    For example,

       mass1 = FROM_INJECTION

    will cause ``mass1`` to be retrieved from the injection file, while:

    .. code-black:: ini

       mass1 = FROM_INJECTION:'primary_mass(mass1, mass2)'

    will cause the larger of mass1 and mass2 to be retrieved from the injection
    file. Note that if spaces are in the argument, it must be encased in
    single quotes.

    The injection file may contain only one injection. Otherwise, a ValueError
    will be raised.

    cp : ConfigParser
        The config file within which to replace values.
    injection_file : str or None
        The injection file to get values from. A ValueError will be raised
        if there are any ``FROM_INJECTION`` values in the config file, and
        injection file is None, or if there is more than one injection.
    update_cp : bool, optional
        Update the config parser with the replaced parameters. If False,
        will just retrieve the parameter values to update, without updating
        the config file. Default is True.

    Returns
    -------
    list
        The parameters that were replaced, as a tuple of section name, option,
        value.
    """
    lookfor = 'FROM_INJECTION'
    # figure out what parameters need to be set
    replace_params = []
    for sec in cp.sections():
        for opt in cp.options(sec):
            val = cp.get(sec, opt)
            splitvals = shlex.split(val)
            replace_this = []
            for ii, subval in enumerate(splitvals):
                if subval.startswith(lookfor):
                    # determine what we should retrieve from the injection
                    subval = subval.split(':', 1)
                    if len(subval) == 1:
                        subval = opt
                    else:
                        subval = subval[1]
                    replace_this.append((ii, subval))
            if replace_this:
                replace_params.append((sec, opt, splitvals, replace_this))
    if replace_params:
        # check that we have an injection file
        if injection_file is None:
            raise ValueError("One or values are set to {}, but no injection "
                             "file provided".format(lookfor))
        # load the injection file
        inj = InjectionSet(injection_file).table.view(type=FieldArray)
        # make sure there's only one injection provided
        if inj.size > 1:
            raise ValueError("One or more values are set to {}, but more than "
                             "one injection exists in the injection file."
                             .format(lookfor))
    # get the injection values to replace
    for ii, (sec, opt, splitvals, replace_this) in enumerate(replace_params):
        # replace the value in the shlex-splitted string with the value
        # from the injection
        for jj, arg in replace_this:
            splitvals[jj] = str(inj[arg][0])
        # now rejoin the string...
        # shlex will strip quotes around arguments; this can be problematic
        # when rejoining if the the argument had a space in it. In python 3.8
        # there is a shlex.join function which properly rejoins things taking
        # that into account. Since we need to continue to support earlier
        # versions of python, the following kludge tries to account for that.
        # If/when we drop support for all earlier versions of python, then the
        # following can just be replaced by:
        # replace_val = shlex.join(splitvals)
        for jj, arg in enumerate(splitvals):
            if ' ' in arg:
                arg = "'" + arg + "'"
                splitvals[jj] = arg
        replace_val = ' '.join(splitvals)
        replace_params[ii] = (sec, opt, replace_val)
    # replace in the config file
    if update_cp:
        for (sec, opt, replace_val) in replace_params:
            cp.set(sec, opt, replace_val)
    return replace_params

def create_waveform_generator(
                variable_params, data, waveform_transforms=None,
                recalibration=None, gates=None,
                generator_class=generator.FDomainDetFrameGenerator,
                **static_params):
    r"""Creates a waveform generator for use with a model.

    variable_params : list of str
        The names of the parameters varied.
    data : dict
        Dictionary mapping detector names to either a
        :py:class:`<pycbc.types.TimeSeries TimeSeries>` or
        :py:class:`<pycbc.types.FrequencySeries FrequencySeries>`.
        The list of transforms applied to convert variable parameters into
        parameters that will be understood by the waveform generator.
    recalibration : dict, optional
        Dictionary mapping detector names to
        :py:class:`<pycbc.calibration.Recalibrate>` instances for
        sort of thing returned by :py:func:`pycbc.gate.gates_from_cli`.
    generator_class : detector-frame fdomain generator, optional
        Class to use for generating waveforms. Default is
        :py:class:`waveform.generator.FDomainDetFrameGenerator`.
    \**static_params :
        All other keyword arguments are passed as static parameters to the
        waveform generator.

    Returns
    -------
    pycbc.waveform.FDomainDetFrameGenerator
        A waveform generator for frequency domain generation.
    """
    # the waveform generator will get the variable_params + the output
    # of the waveform transforms, so we'll add them to the list of
    # parameters
    if waveform_transforms is not None:
        wfoutputs = set.union(*[t.outputs
                                for t in waveform_transforms])
    else:
        wfoutputs = set()
    variable_params = list(variable_params) + list(wfoutputs)
    # figure out what generator to use based on the approximant
    try:
        approximant = static_params['approximant']
        raise ValueError("no approximant provided in the static args")

    generator_function = generator_class.select_rframe_generator(approximant)
    # get data parameters; we'll just use one of the data to get the
    # values, then check that all the others are the same
    delta_f = None
    for d in data.values():
        if delta_f is None:
            delta_f = d.delta_f
            delta_t = d.delta_t
            start_time = d.start_time
        else:
            if not all([d.delta_f == delta_f, d.delta_t == delta_t,
                        d.start_time == start_time]):
                raise ValueError("data must all have the same delta_t, "
                                 "delta_f, and start_time")
    waveform_generator = generator_class(
        generator_function, epoch=start_time,
        variable_args=variable_params, detectors=list(data.keys()),
        delta_f=delta_f, delta_t=delta_t,
        recalib=recalibration, gates=gates,
        **static_params)
    return waveform_generator

# Copyright (C) 2018  Charlie Hoy, Collin Capano
#
#

"""This module provides model classes that assume the noise is Gaussian and
allows for the likelihood to be marginalized over phase and/or time and/or
distance.
"""

from scipy import special

from .gaussian_noise import (BaseGaussianNoise, create_waveform_generator)

class MarginalizedPhaseGaussianNoise(BaseGaussianNoise):
    r"""The likelihood is analytically marginalized over phase.

    This class can be used with signal models that can be written as:

    .. math::

        \tilde{h}(f; \Theta, \phi) = A(f; \Theta)e^{i\Psi(f; \Theta) + i \phi},

    where :math:`\phi` is an arbitrary phase constant. This phase constant
    can be analytically marginalized over with a uniform prior as follows:
    assuming the noise is stationary and Gaussian (see `GaussianNoise`
    for details), the posterior is:

    .. math::

        p(\Theta,\phi|d)
            &\propto p(\Theta)p(\phi)p(d|\Theta,\phi) \\
            &\propto p(\Theta)\frac{1}{2\pi}\exp\left[
                -\frac{1}{2}\sum_{i}^{N_D} \left<
                    h_i(\Theta,\phi) - d_i, h_i(\Theta,\phi) - d_i
                \right>\right].

    Here, the sum is over the number of detectors :math:`N_D`, :math:`d_i`
    and :math:`h_i` are the data and signal in detector :math:`i`,
    respectively, and we have assumed a uniform prior on :math:`phi \in [0,
    2\pi)`. With the form of the signal model given above, the inner product
    in the exponent can be written as:

    .. math::

        -\frac{1}{2}\left<h_i - d_i, h_i- d_i\right>
            &= \left<h_i, d_i\right> -
               \frac{1}{2}\left<h_i, h_i\right> -
               \frac{1}{2}\left<d_i, d_i\right> \\
            &= \Re\left\{O(h^0_i, d_i)e^{-i\phi}\right\} -
               \frac{1}{2}\left<h^0_i, h^0_i\right> -
               \frac{1}{2}\left<d_i, d_i\right>,

    where:

    .. math::

        h_i^0 &\equiv \tilde{h}_i(f; \Theta, \phi=0); \\
        O(h^0_i, d_i) &\equiv 4 \int_0^\infty
            \frac{\tilde{h}_i^*(f; \Theta,0)\tilde{d}_i(f)}{S_n(f)}\mathrm{d}f.

    Gathering all of the terms that are not dependent on :math:`\phi` together:

    .. math::

        \alpha(\Theta, d) \equiv \exp\left[-\frac{1}{2}\sum_i
            \left<h^0_i, h^0_i\right> + \left<d_i, d_i\right>\right],

    we can marginalize the posterior over :math:`\phi`:

    .. math::

        p(\Theta|d)
            &\propto p(\Theta)\alpha(\Theta,d)\frac{1}{2\pi}
                     \int_{0}^{2\pi}\exp\left[\Re \left\{
                         e^{-i\phi} \sum_i O(h^0_i, d_i)
                     \right\}\right]\mathrm{d}\phi \\
            &\propto p(\Theta)\alpha(\Theta, d)\frac{1}{2\pi}
                     \int_{0}^{2\pi}\exp\left[
                         x(\Theta,d)\cos(\phi) + y(\Theta, d)\sin(\phi)
                     \right]\mathrm{d}\phi.

    The integral in the last line is equal to :math:`2\pi I_0(\sqrt{x^2+y^2})`,
    where :math:`I_0` is the modified Bessel function of the first kind. Thus
    the marginalized log posterior is:

    .. math::

        \log p(\Theta|d) \propto \log p(\Theta) +
            I_0\left(\left|\sum_i O(h^0_i, d_i)\right|\right) -
            \frac{1}{2}\sum_i\left[ \left<h^0_i, h^0_i\right> -
                                    \left<d_i, d_i\right> \right]
    """
    name = 'marginalized_phase'

        super(MarginalizedPhaseGaussianNoise, self).__init__(

    @property
        return ['loglr', 'maxl_phase'] + \

        """
        setattr(self._current_stats, 'loglikelihood', -numpy.inf)
        # maxl phase doesn't exist, so set it to nan
        setattr(self._current_stats, 'maxl_phase', numpy.nan)

        .. math::
            \log \mathcal{L}(\Theta) =
                I_0 \left(\left|\sum_i O(h^0_i, d_i)\right|\right) -
                \frac{1}{2}\left<h^0_i, h^0_i\right>,
        at the current point in parameter space :math:`\Theta`.
        Returns
        -------
        float
            The value of the log likelihood ratio evaluated at the given point.
        """
        try:
            else:
                raise e
        hh = 0.
        hd = 0j
                hh_i = 0.
                hd_i = 0j
            else:
                h[self._kmin[det]:kmax] *= \
                    self._weight[det][self._kmin[det]:kmax]
                # calculate inner products
                hh_i = h[self._kmin[det]:kmax].inner(
                    h[self._kmin[det]:kmax]).real
                hd_i = self._whitened_data[det][self._kmin[det]:kmax].inner(
                    h[self._kmin[det]:kmax])
            # store
            setattr(self._current_stats, '{}_optimal_snrsq'.format(det), hh_i)
            hh += hh_i
            hd += hd_i
        hd = abs(hd)
        self._current_stats.maxl_phase = numpy.angle(hd)
        return numpy.log(special.i0e(hd)) + hd - 0.5*hh

class MarginalizedPolarization(BaseGaussianNoise):
    r""" This likelihood numerically marginalizes over polarization angle

    This class implements the Gaussian likelihood with an explicit numerical
    marginalization over polarization angle. This is accomplished using
    a fixed set of integration points distribution uniformation between
    0 and 2pi. By default, 1000 integration points are used.
    The 'polarization_samples' argument can be passed to set an alternate
    number of integration points.
    """
    name = 'marginalized_polarization'

                 polarization_samples=1000,
        super(MarginalizedPolarization, self).__init__(
            generator_class=generator.FDomainDetFrameTwoPolGenerator,

        self.polarization_samples = polarization_samples
        self.pol = numpy.linspace(0, 2*numpy.pi, self.polarization_samples)
        self.dets = {}

    @property
        """Adds ``loglr``, ``maxl_polarization``, and the ``optimal_snrsq`` in
        each detector.
        """
        return ['loglr', 'maxl_polarization', 'maxl_loglr'] + \

        """
        setattr(self._current_stats, 'loglr', -numpy.inf)
        setattr(self._current_stats, 'maxl_polarization', numpy.nan)

        .. math::

        Returns
        -------
        float
        """
        try:
            else:
                raise e

        lr = 0.
        for det, (hp, hc) in wfs.items():
            if det not in self.dets:
                self.dets[det] = Detector(det)
            fp, fc = self.dets[det].antenna_pattern(self.current_params['ra'],
                                                    self.current_params['dec'],
                                                    self.pol,
                                                    self.current_params['tc'])

            kmax = min(max(len(hp), len(hc)), self._kmax[det])
            slc = slice(self._kmin[det], kmax)

            # whiten both polarizations
            hp[self._kmin[det]:kmax] *= self._weight[det][slc]
            hc[self._kmin[det]:kmax] *= self._weight[det][slc]

            # h = fp * hp + hc * hc
            # <h, d> = fp * <hp,d> + fc * <hc,d>
            # the inner products
            cplx_hpd = self._whitened_data[det][slc].inner(hp[slc])  # <hp, d>
            cplx_hcd = self._whitened_data[det][slc].inner(hc[slc])  # <hc, d>

            cplx_hd = fp * cplx_hpd + fc * cplx_hcd

            # <h, h> = <fp * hp + fc * hc, fp * hp + fc * hc>
            # = Real(fpfp * <hp,hp> + fcfc * <hc,hc> + \
            #  fphc * (<hp, hc> + <hc, hp>))
            hphp = hp[slc].inner(hp[slc]).real  # < hp, hp>
            hchc = hc[slc].inner(hc[slc]).real  # <hc, hc>

            # Below could be combined, but too tired to figure out
            # if there should be a sign applied if so
            hphc = hp[slc].inner(hc[slc]).real  # <hp, hc>
            hchp = hc[slc].inner(hp[slc]).real  # <hc, hp>

            hh = fp * fp * hphp + fc * fc * hchc + fp * fc * (hphc + hchp)
            # store
            lr += cplx_hd.real - 0.5 * hh

        lr_total = special.logsumexp(lr) - numpy.log(len(self.pol))

        # store the maxl polarization
        idx = lr.argmax()
        setattr(self._current_stats, 'maxl_polarization', self.pol[idx])
        setattr(self._current_stats, 'maxl_loglr', lr[idx])

        # just store the maxl optimal snrsq
        for det in wfs:
            p = '{}_optimal_snrsq'.format(det)
            setattr(self._current_stats, p,
                    getattr(self._current_stats, p)[idx])

        return float(lr_total)

class MarginalizedHMPolPhase(BaseGaussianNoise):
    r"""Numerically marginalizes waveforms with higher modes over polarization
    `and` phase.

    marginalization over polarization angle and orbital phase. This is
    accomplished using a fixed set of integration points distributed uniformly
    between 0 and 2:math:`\pi` for both the polarization and phase. By default,
    100 integration points are used for each parameter, giving :math:`10^4`
    evaluation points in total. This can be modified using the
    ``polarization_samples`` and ``coa_phase_samples`` arguments.

    This only works with waveforms that return separate spherical harmonic
    modes for each waveform. For a list of currently supported approximants,
    see :py:func:`pycbc.waveform.waveform_modes.fd_waveform_mode_approximants`
    and :py:func:`pycbc.waveform.waveform_modes.td_waveform_mode_approximants`.

    data : dict
    polarization_samples : int, optional
        How many points to use in polarization. Default is 100.
    coa_phase_samples : int, optional
        How many points to use in phase. Defaults is 100.
        :py:class:`gaussian_noise.BaseGaussianNoisei <BaseGaussianNoise>`.

    """
    name = 'marginalized_hmpolphase'

                 polarization_samples=100,
                 coa_phase_samples=100,
        super(MarginalizedHMPolPhase, self).__init__(
            generator_class=generator.FDomainDetFrameModesGenerator,
        pol = numpy.linspace(0, 2*numpy.pi, polarization_samples)
        phase = numpy.linspace(0, 2*numpy.pi, coa_phase_samples)
        # remap to every combination of the parameters
        # this gets every combination by mappin them to an NxM grid
        # one needs to be transposed so that they run allong opposite
        # dimensions
        n = coa_phase_samples * polarization_samples
        self.nsamples = n
        self.pol = numpy.resize(pol, n)
        phase = numpy.resize(phase, n)
        phase = phase.reshape(coa_phase_samples, polarization_samples)
        self.phase = phase.T.flatten()
        self._phase_fac = {}

    def phase_fac(self, m):
        r"""The phase :math:`\exp[i m \phi]`."""
        try:
            return self._phase_fac[m]
            # hasn't been computed yet, calculate it
            self._phase_fac[m] = numpy.exp(1.0j * m * self.phase)

    @property
        """Adds ``maxl_polarization`` and the ``maxl_phase``
        """
        return ['maxl_polarization', 'maxl_phase',]

        """

    def _loglr(self, return_unmarginalized=False):

        .. math::

        Returns
        -------
        float
        """
        try:
            else:
                raise e

        # ---------------------------------------------------------------------
        # Some optimizations not yet taken:
        # * higher m calculations could have a lot of redundancy
        # * fp/fc need not be calculated except where polarization is different
        # * may be possible to simplify this by making smarter use of real/imag
        lr = 0.
        hds = {}
        hhs = {}
        for det, modes in wfs.items():

                                                    self.pol,

            # loop over modes and prepare the waveform modes
            # we will sum up zetalm = glm <ulm, d> + i glm <vlm, d>
            # over all common m so that we can apply the phase once
            zetas = {}
            rlms = {}
            slms = {}
            for mode in modes:
                l, m = mode
                ulm, vlm = modes[mode]

                # whiten the waveforms
                # the kmax of the waveforms may be different than internal kmax
                kmax = min(max(len(ulm), len(vlm)), self._kmax[det])
                ulm[self._kmin[det]:kmax] *= self._weight[det][slc]
                vlm[self._kmin[det]:kmax] *= self._weight[det][slc]

                # <ulm, d>
                ulmd = ulm[slc].inner(self._whitened_data[det][slc]).real
                # <vlm, d>
                vlmd = vlm[slc].inner(self._whitened_data[det][slc]).real

                # add inclination, and pack into a complex number
                import lal
                glm = lal.SpinWeightedSphericalHarmonic(
                    self.current_params['inclination'], 0, -2, l, m).real

                if m not in zetas:
                    zetas[m] = 0j
                zetas[m] += glm * (ulmd + 1j*vlmd)

                # Get condense set of the parts of the waveform that only diff
                # by m, this is used next to help calculate <h, h>
                r = glm * ulm
                s = glm * vlm

                if m not in rlms:
                    rlms[m] = r
                    slms[m] = s
                else:
                    rlms[m] += r
                    slms[m] += s

            # now compute all possible <hlm, hlm>
            rr_m = {}
            ss_m = {}
            rs_m = {}
            sr_m = {}
            combos = itertools.combinations_with_replacement(rlms.keys(), 2)
            for m, mprime in combos:
                r = rlms[m]
                s = slms[m]
                rprime = rlms[mprime]
                sprime = slms[mprime]
                rr_m[mprime, m] = r[slc].inner(rprime[slc]).real
                ss_m[mprime, m] = s[slc].inner(sprime[slc]).real
                rs_m[mprime, m] = s[slc].inner(rprime[slc]).real
                sr_m[mprime, m] = r[slc].inner(sprime[slc]).real
                # store the conjugate for easy retrieval later
                rr_m[m, mprime] = rr_m[mprime, m]
                ss_m[m, mprime] = ss_m[mprime, m]
                rs_m[m, mprime] = sr_m[mprime, m]
                sr_m[m, mprime] = rs_m[mprime, m]
            # now apply the phase to all the common ms
            hpd = 0.
            hcd = 0.
            hphp = 0.
            hchc = 0.
            hphc = 0.
            for m, zeta in zetas.items():
                phase_coeff = self.phase_fac(m)

                # <h+, d> = (exp[i m phi] * zeta).real()
                # <hx, d> = -(exp[i m phi] * zeta).imag()
                z = phase_coeff * zeta
                hpd += z.real
                hcd -= z.imag

                # now calculate the contribution to <h, h>
                cosm = phase_coeff.real
                sinm = phase_coeff.imag

                for mprime in zetas:
                    pcprime = self.phase_fac(mprime)

                    cosmprime = pcprime.real
                    sinmprime = pcprime.imag
                    # needed components
                    rr = rr_m[m, mprime]
                    ss = ss_m[m, mprime]
                    rs = rs_m[m, mprime]
                    sr = sr_m[m, mprime]
                    # <hp, hp>
                    hphp += rr * cosm * cosmprime \
                        + ss * sinm * sinmprime \
                        - rs * cosm * sinmprime \
                        - sr * sinm * cosmprime
                    # <hc, hc>
                    hchc += rr * sinm * sinmprime \
                        + ss * cosm * cosmprime \
                        + rs * sinm * cosmprime \
                        + sr * cosm * sinmprime
                    # <hp, hc>
                    hphc += -rr * cosm * sinmprime \
                        + ss * sinm * cosmprime \
                        + sr * sinm * sinmprime \
                        - rs * cosm * cosmprime

            # Now apply the polarizations and calculate the loglr
            # We have h = Fp * hp + Fc * hc
            # loglr = <h, d> - <h, h>/2
            #       = Fp*<hp, d> + Fc*<hc, d>
            #          - (1/2)*(Fp*Fp*<hp, hp> + Fc*Fc*<hc, hc>
            #                   + 2*Fp*Fc<hp, hc>)
            # (in the last line we have made use of the time series being
            #  real, so that <a, b> = <b, a>).
            hd = fp * hpd + fc * hcd
            hh = fp * fp * hphp + fc * fc * hchc + 2 * fp * fc * hphc
            hds[det] = hd
            hhs[det] = hh
            lr += hd - 0.5 * hh

        if return_unmarginalized:
            return self.pol, self.phase, lr, hds, hhs

        lr_total = special.logsumexp(lr) - numpy.log(self.nsamples)

        # store the maxl values
        setattr(self._current_stats, 'maxl_phase', self.phase[idx])

# Copyright (C) 2020  Daniel Finstad
#
#

#
#
#                                   Preamble
#
#
"""This module provides model classes and functions for implementing
a relative binning likelihood for parameter estimation.
"""

from pycbc.waveform import get_fd_waveform_sequence

from .relbin_cpu import likelihood_parts, likelihood_parts_v

def setup_bins(f_full, f_lo, f_hi, chi=1.0, eps=0.5, gammas=None):
    """Construct frequency bins for use in a relative likelihood
    model. For details, see [Barak, Dai & Venumadhav 2018].

    f_full : array
        The full resolution array of frequencies being used in the analysis.
    f_lo : float
        The starting frequency used in matched filtering. This will be the
        left edge of the first frequency bin.
    f_hi : float
        The ending frequency used in matched filtering. This will be the right
        edge of the last frequency bin.
    chi : float, optional
        Tunable parameter, see [Barak, Dai & Venumadhav 2018]
    eps : float, optional
        Tunable parameter, see [Barak, Dai & Venumadhav 2018]. Lower values
        result in larger number of bins.
    gammas : array, optional
        Frequency powerlaw indices to be used in computing bins.

    Returns
    -------
    nbin : int
        Number of bins.
    fbin : numpy.array of floats
        Bin edge frequencies.
    fbin_ind : numpy.array of ints
        Indices of bin edges in full frequency array.
    """
    f = numpy.linspace(f_lo, f_hi, 10000)
    # f^ga power law index
    ga = (
        gammas
        if gammas is not None
        else numpy.array([-5.0 / 3, -2.0 / 3, 1.0, 5.0 / 3, 7.0 / 3])
    )
    logging.info("Using powerlaw indices: %s", ga)
    dalp = chi * 2.0 * numpy.pi / numpy.absolute((f_lo ** ga) - (f_hi ** ga))
    dphi = numpy.sum(
        numpy.array([numpy.sign(g) * d * (f ** g) for g, d in zip(ga, dalp)]),
        axis=0,
    )
    dphi_diff = dphi - dphi[0]
    # now construct frequency bins
    nbin = int(dphi_diff[-1] / eps)
    dphi2f = interp1d(
        dphi_diff, f, kind="slinear", bounds_error=False, fill_value=0.0
    )
    dphi_grid = numpy.linspace(dphi_diff[0], dphi_diff[-1], nbin + 1)
    # frequency grid points
    fbin = dphi2f(dphi_grid)
    # indices of frequency grid points in the FFT array
    fbin_ind = numpy.unique(
        [numpy.argmin(numpy.absolute(f_full - ff)) for ff in fbin]
    )
    # make sure grid points are precise
    fbin = numpy.array([f_full[i] for i in fbin_ind])
    nbin = len(fbin)
    return nbin, fbin, fbin_ind

class Relative(BaseGaussianNoise):
    r"""Model that assumes the likelihood in a region around the peak
    is slowly varying such that a linear approximation can be made, and
    likelihoods can be calculated at a coarser frequency resolution. For
    more details on the implementation, see https://arxiv.org/abs/1806.08792.

    This model requires the use of a fiducial waveform whose parameters are
    near the peak of the likelihood. The fiducial waveform and all template
    waveforms used in likelihood calculation are currently generated using
    the SPAtmplt approximant.

    :py:class:`BaseGaussianNoise`.

    data : dict
    figucial_params : dict
        A dictionary of waveform parameters to be used for generating the
        fiducial waveform. Keys must be parameter names in the form
        'PARAM_ref' where PARAM is a recognized extrinsic parameter or
        an intrinsic parameter compatible with the chosen approximant.
    gammas : array of floats, optional
        Frequency powerlaw indices to be used in computing frequency bins.
    epsilon : float, optional
        Tuning parameter used in calculating the frequency bins. Lower values
        will result in higher resolution and more bins.
    earth_rotation: boolean, optional
        Default is False. If True, then vary the fp/fc polarization values
        as a function of frequency bin, using a predetermined PN approximation
        for the time offsets.
        :py:class:`BaseGaussianNoise`.
    """
    name = "relative"

    def __init__(
        self,
        variable_params,
        data,
        low_frequency_cutoff,
        fiducial_params=None,
        gammas=None,
        epsilon=0.5,
        earth_rotation=False,
        **kwargs
    ):
        super(Relative, self).__init__(
            variable_params, data, low_frequency_cutoff, **kwargs
        )

        self.epsilon = float(epsilon)

        # reference waveform and bin edges
        self.h00, self.h00_sparse = {}, {}
        self.f, self.df, self.end_time, self.det = {}, {}, {}, {}
        self.edges, self.fedges, self.bins, self.fbins = {}, {}, {}, {}
        self.ta = {}
        self.antenna_time = {}

        # filtered summary data for linear approximation
        self.sdat = {}

        # store data and psds as arrays for faster computation
        self.comp_data = {ifo: d.numpy() for ifo, d in self.data.items()}
        self.comp_psds = {ifo: p.numpy() for ifo, p in self.psds.items()}

        # store fiducial waveform params
        self.fid_params = self.static_params.copy()
        self.fid_params.update(fiducial_params)

        for ifo in data:
            # store data and frequencies
            d0 = self.data[ifo]
            self.f[ifo] = numpy.array(d0.sample_frequencies)
            self.df[ifo] = d0.delta_f
            self.end_time[ifo] = float(d0.end_time)
            self.det[ifo] = Detector(ifo)

            # get detector-specific arrival times relative to end of data
            dt = self.det[ifo].time_delay_from_earth_center(
                self.fid_params["ra"],
                self.fid_params["dec"],
                self.fid_params["tc"],
            )

            self.ta[ifo] = self.fid_params["tc"] + dt - self.end_time[ifo]

            # generate fiducial waveform
            f_lo = self.kmin[ifo] * self.df[ifo]
            f_hi = self.kmax[ifo] * self.df[ifo]
            logging.info(
                "%s: Generating fiducial waveform from %s to %s Hz",
                ifo,
                f_lo,
                f_hi,
            )

            # prune low frequency samples to avoid waveform errors
            fpoints = Array(self.f[ifo].astype(numpy.float64))
            fpoints = fpoints[self.kmin[ifo]:self.kmax[ifo]+1]
            fid_hp, fid_hc = get_fd_waveform_sequence(sample_points=fpoints,
                                                      **self.fid_params)

            # check for zeros at high frequencies
            # make sure only nonzero samples are included in bins
            numzeros = list(fid_hp[::-1] != 0j).index(True)
            if numzeros > 0:
                new_kmax = self.kmax[ifo] - numzeros
                f_hi = new_kmax * self.df[ifo]
                logging.info(
                    "WARNING! Fiducial waveform terminates below "
                    "high-frequency-cutoff, final bin frequency "
                    "will be %s Hz", f_hi)

            # make copy of fiducial wfs, adding back in low frequencies
            fid_hp.resize(len(self.f[ifo]))
            fid_hc.resize(len(self.f[ifo]))
            hp0 = numpy.roll(fid_hp, self.kmin[ifo])
            hc0 = numpy.roll(fid_hc, self.kmin[ifo])

            fp, fc = self.det[ifo].antenna_pattern(
                self.fid_params["ra"], self.fid_params["dec"],
                self.fid_params["polarization"], self.fid_params["tc"])

            tshift = numpy.exp(-2.0j * numpy.pi * self.f[ifo] * self.ta[ifo])
            self.h00[ifo] = numpy.array(hp0 * fp + hc0 * fc) * tshift

            # compute frequency bins
            logging.info("Computing frequency bins")
            nbin, fbin, fbin_ind = setup_bins(
                f_full=self.f[ifo],
                f_lo=f_lo,
                f_hi=f_hi,
                gammas=gammas,
                eps=self.epsilon,
            )
            logging.info("Using %s bins for this model", nbin)

            # store bins and edges in sample and frequency space
            self.edges[ifo] = fbin_ind
            self.fedges[ifo] = numpy.array(fbin).astype(numpy.float64)
            self.bins[ifo] = numpy.array(
                [
                    (self.edges[ifo][i], self.edges[ifo][i + 1])
                    for i in range(len(self.edges[ifo]) - 1)
                ]
            )
            self.fbins[ifo] = numpy.array(
                [(fbin[i], fbin[i + 1]) for i in range(len(fbin) - 1)]
            )

            # store low res copy of fiducial waveform
            self.h00_sparse[ifo] = self.h00[ifo].copy().take(self.edges[ifo])

            # compute summary data
                "Calculating summary data at frequency resolution %s Hz",
                self.df[ifo],
            )
            self.sdat[ifo] = self.summary_data(ifo)

            # Calculate the times to evaluate fp/fc
            if earth_rotation is not False:
                logging.info("Enabling frequency-dependent earth rotation")
                from pycbc.waveform.spa_tmplt import spa_length_in_time

                times = spa_length_in_time(
                    phase_order=-1,
                    mass1=self.fid_params["mass1"],
                    mass2=self.fid_params["mass2"],
                    f_lower=self.fedges[ifo],
                )
                self.antenna_time[ifo] = self.fid_params["tc"] - times
                self.lik = likelihood_parts_v
            else:
                self.antenna_time[ifo] = self.fid_params["tc"]
                self.lik = likelihood_parts

        # determine the unique ifo layouts
        self.edge_unique = []
        self.ifo_map = {}
        for ifo in self.fedges:
            if len(self.edge_unique) == 0:
                self.ifo_map[ifo] = 0
                self.edge_unique.append(Array(self.fedges[ifo]))
            else:
                for i, edge in enumerate(self.edge_unique):
                    if numpy.array_equal(edge, self.fedges[ifo]):
                        self.ifo_map[ifo] = i
                        break
                else:
                    self.ifo_map[ifo] = len(self.edge_unique)
                    self.edge_unique.append(Array(self.fedges[ifo]))
        logging.info("%s unique ifo layouts", len(self.edge_unique))

    def summary_data(self, ifo):
        """Compute summary data bin coefficients encoding linear
        approximation to full resolution likelihood.

        Returns
        -------
        dict
            Dictionary containing bin coefficients a0, b0, a1, b1,
            for each frequency bin.
        """
        # calculate coefficients
        hd = numpy.conjugate(self.comp_data[ifo]) * self.h00[ifo]
        hd /= self.comp_psds[ifo]
        hh = (numpy.absolute(self.h00[ifo]) ** 2.0) / self.comp_psds[ifo]

        # constant terms
        a0 = numpy.array([
                4.0 * self.df[ifo] * numpy.sum(hd[l:h])
                for l, h in self.bins[ifo]
            ])
        b0 = numpy.array([
                4.0 * self.df[ifo] * numpy.sum(hh[l:h])
            ])

        # linear terms
        bin_lefts = [fl for fl, fh in self.fbins[ifo]]
        a1 = numpy.array([
                4.0 * self.df[ifo]
                * numpy.sum(hd[l:h] * (self.f[ifo][l:h] - bl))
                for (l, h), bl in zip(self.bins[ifo], bin_lefts)
            ])
        b1 = numpy.array([
                * numpy.sum(hh[l:h] * (self.f[ifo][l:h] - bl))
            ])

        freqs = self.fedges[ifo]
        df = (freqs[1:] - freqs[:-1])
        a1 /= df
        b1 /= df
        return {"a0": a0, "a1": a1, "b0": b0, "b1": b1}

    def get_waveforms(self, params):
        """ Get the waveform polarizations for each ifo
        """
        wfs = []
        for edge in self.edge_unique:
            hp, hc = get_fd_waveform_sequence(sample_points=edge, **params)
            hp = hp.numpy()
            hc = hc.numpy()
            wfs.append((hp, hc))
        return {ifo: wfs[self.ifo_map[ifo]] for ifo in self.data}

        .. math::

        Returns
        -------
        float
        """
        # get model params
        p = self.current_params.copy()
        p.update(self.static_params)
        wfs = self.get_waveforms(p)

        hh = 0.0
        hd = 0j

            det = self.det[ifo]
            freqs = self.fedges[ifo]
            sdat = self.sdat[ifo]
            hp, hc = wfs[ifo]
            h00 = self.h00_sparse[ifo]
            end_time = self.end_time[ifo]
            times = self.antenna_time[ifo]

            # project waveform to detector frame
            fp, fc = det.antenna_pattern(p["ra"], p["dec"],
                                         p["polarization"], times)
            dt = det.time_delay_from_earth_center(p["ra"], p["dec"], times)
            dtc = p["tc"] + dt - end_time

            hdp, hhp = self.lik(freqs, fp, fc, dtc,
                                hp, hc, h00,
                                sdat['a0'], sdat['a1'],
                                sdat['b0'], sdat['b1'])

            hd += hdp
            hh += hhp

        llr = numpy.log(special.i0e(hd)) + hd - 0.5 * hh
        return float(llr)

        """Adds writing the fiducial parameters and epsilon to file's attrs.

        """
        super(Relative, self).write_metadata(fp)
        fp.attrs["epsilon"] = self.epsilon
        for p, v in self.fid_params.items():
            fp.attrs["{}_ref".format(p)] = v

        """Adds reading fiducial waveform parameters from config file."""
        # add fiducial params to skip list
        skip_args += [
            option for option in cp.options(section) if option.endswith("_ref")
        ]

        # get frequency power-law indices if specified
        # NOTE these should be supplied in units of 1/3
        gammas = None
        if cp.has_option(section, "gammas"):
            skip_args.append("gammas")
            gammas = numpy.array(
                [float(g) / 3.0 for g in cp.get(section, "gammas").split()]
            )
        args = super(Relative, Relative).extra_args_from_config(
            cp, section, skip_args=skip_args, dtypes=dtypes
        )

        # get fiducial params from config
        fid_params = {
            p.replace("_ref", ""): float(cp.get("model", p))
            for p in cp.options("model")
            if p.endswith("_ref")
        }

        # add optional params with default values if not specified
        opt_params = {
            "ra": numpy.pi,
            "dec": 0.0,
            "inclination": 0.0,
            "polarization": numpy.pi,
        }
        fid_params.update(
            {p: opt_params[p] for p in opt_params if p not in fid_params}
        )
        args.update({"fiducial_params": fid_params, "gammas": gammas})

#
#

"""

import scipy.special

from pycbc.waveform import get_fd_waveform

# In this model we only calculate terms up to a constant.
# We are primarily interested in the posterior result

class SingleTemplate(BaseGaussianNoise):
    r"""Model that assumes we know all the intrinsic parameters.

    This model assumes we know all the intrinsic parameters, and are only
    maximizing over the extrinsic ones. We also assume a dominant mode waveform
    approximant only and non-precessing.

    data : dict
    sample_rate : int, optional
        The sample rate to use. Default is 32768.
    polarization_samples: int, optional
        Parameter to specify how finely to marginalize over polarization angle.
        If None, then polarization must be a parameter.
        :py:class:`BaseGaussianNoise`; see that class for details.
    """
    name = 'single_template'

    def __init__(self, variable_params, data, low_frequency_cutoff,
                 sample_rate=32768, polarization_samples=None, **kwargs):
        super(SingleTemplate, self).__init__(
            variable_params, data, low_frequency_cutoff, **kwargs)

        # Generate template waveforms
        df = data[self.detectors[0]].delta_f
        p = self.static_params.copy()
        if 'distance' in p:
            _ = p.pop('distance')
        if 'inclination' in p:
            _ = p.pop('inclination')
        hp, _ = get_fd_waveform(delta_f=df, distance=1, inclination=0, **p)

        # Extend template to high sample rate
        flen = int(int(sample_rate) / df) / 2 + 1
        hp.resize(flen)
        #polarization array to marginalize over if num_samples given
        self.pflag = 0
        if polarization_samples is not None:
            self.polarization = numpy.linspace(0, 2*numpy.pi,
                                               int(polarization_samples))
            self.pflag = 1

        # Calculate high sample rate SNR time series
        self.sh = {}
        self.hh = {}
        self.det = {}
            flow = self.kmin[ifo] * df
            fhigh = self.kmax[ifo] * df
            # Extend data to high sample rate
            self.data[ifo].resize(flen)
            snr, _, _ = pyfilter.matched_filter_core(
                hp, self.data[ifo],
                psd=self.psds[ifo],
                low_frequency_cutoff=flow,
                high_frequency_cutoff=fhigh)

            self.sh[ifo] = 4 * df * snr
            self.hh[ifo] = -0.5 * pyfilter.sigmasq(
                hp, psd=self.psds[ifo],
        self.time = None

        r"""Computes the log likelihood ratio

        Returns
        -------
        float
        """
        # calculate <d-h|d-h> = <h|h> - 2<h|d> + <d|d> up to a constant
        if self.pflag == 0:
            polarization = p['polarization']
        elif self.pflag == 1:
            polarization = self.polarization

        if self.time is None:
            self.time = p['tc']

        shloglr = hhloglr = 0
        for ifo in self.sh:
            fp, fc = self.det[ifo].antenna_pattern(p['ra'], p['dec'],
                                                   polarization,
                                                   self.time)
            dt = self.det[ifo].time_delay_from_earth_center(p['ra'],
                                                            p['dec'],
                                                            self.time)
            ic = numpy.cos(p['inclination'])
            ip = 0.5 * (1.0 + ic * ic)
            htf = (fp * ip + 1.0j * fc * ic) / p['distance']

            sh = self.sh[ifo].at_time(p['tc'] + dt) * htf
            shloglr += sh
            hhloglr += self.hh[ifo] * abs(htf) ** 2.0
        vloglr = numpy.log(scipy.special.i0e(abs(shloglr)))
        vloglr += abs(shloglr) + hhloglr
            return float(vloglr)
        else:
            return float(logsumexp(vloglr)) - numpy.log(len(vloglr))

#
#

"""
This package provides classes and functions for evaluating Bayesian statistics
assuming various noise models.
"""

from .analytic import (TestEggbox, TestNormal, TestRosenbrock, TestVolcano,
                       TestPrior, TestPosterior)
from .gaussian_noise import GaussianNoise
from .marginalized_gaussian_noise import MarginalizedPhaseGaussianNoise
from .marginalized_gaussian_noise import MarginalizedPolarization
from .marginalized_gaussian_noise import MarginalizedHMPolPhase
from .brute_marg import BruteParallelGaussianMarginalize
from .single_template import SingleTemplate
from .relbin import Relative

# Used to manage a model instance across multiple cores or MPI
_global_instance = None

def _call_global_model(*args, **kwds):
    """Private function for global model (needed for parallelization)."""
    return _global_instance(*args, **kwds)  # pylint:disable=not-callable

def _call_global_model_logprior(*args, **kwds):
    """Private function for a calling global's logprior.

    This is needed for samplers that use a separate function for the logprior,
    like ``emcee_pt``.
    """
    # pylint:disable=not-callable
    return _global_instance(*args, callstat='logprior', **kwds)

class CallModel(object):
    """Wrapper class for calling models from a sampler.

    This class can be called like a function, with the parameter values to
    evaluate provided as a list in the same order as the model's
    ``variable_params``. In that case, the model is updated with the provided
    parameters and then the ``callstat`` retrieved. If ``return_all_stats`` is
    set to ``True``, then all of the stats specified by the model's
    ``default_stats`` will be returned as a tuple, in addition to the stat
    value.

    The model's attributes are promoted to this class's namespace, so that any
    attribute and method of ``model`` may be called directly from this class.

    This class must be initalized prior to the creation of a ``Pool`` object.

    model : Model instance
        The model to call.
    callstat : str
        The statistic to call.
    return_all_stats : bool, optional
        Whether or not to return all of the other statistics along with the
        ``callstat`` value.

    Examples
    --------
    Create a wrapper around an instance of the ``TestNormal`` model, with the
    ``callstat`` set to ``logposterior``:

    >>> from pycbc.inference.models import TestNormal, CallModel
    >>> model = TestNormal(['x', 'y'])
    >>> call_model = CallModel(model, 'logposterior')

    Now call on a set of parameter values:

    >>> call_model([0.1, -0.2])
    (-1.8628770664093453, (0.0, 0.0, -1.8628770664093453))

    Note that a tuple of all of the model's ``default_stats`` were returned in
    addition to the ``logposterior`` value. We can shut this off by toggling
    ``return_all_stats``:

    >>> call_model.return_all_stats = False

    Attributes of the model can be called from the call model. For example:

    >>> call_model.variable_params
    ('x', 'y')

    """

    def __init__(self, model, callstat, return_all_stats=True):
        self.model = model
        self.callstat = callstat
        self.return_all_stats = return_all_stats

        """Adds the models attributes to self."""
        return getattr(self.model, attr)

    def __call__(self, param_values, callstat=None, return_all_stats=None):
        """Updates the model with the given parameter values, then calls the
        call function.

        param_values : list of float
            The parameter values to test. Assumed to be in the same order as
            ``model.sampling_params``.
        callstat : str, optional
            Specify which statistic to call. Default is to call whatever self's
            ``callstat`` is set to.
        return_all_stats : bool, optional
            Whether or not to return all stats in addition to the ``callstat``
            value. Default is to use self's ``return_all_stats``.

        Returns
        -------
        stat : float
            The statistic returned by the ``callfunction``.
        all_stats : tuple, optional
            The values of all of the model's ``default_stats`` at the given
            param values. Any stat that has not be calculated is set to
            ``numpy.nan``. This is only returned if ``return_all_stats`` is
            set to ``True``.
        """
        if callstat is None:
            callstat = self.callstat
        if return_all_stats is None:
            return_all_stats = self.return_all_stats
        params = dict(zip(self.model.sampling_params, param_values))
        self.model.update(**params)
        val = getattr(self.model, callstat)
        if return_all_stats:
            return val, self.model.get_current_stats()
        else:
            return val

def read_from_config(cp, **kwargs):
    """Initializes a model from the given config file.

    The section must have a ``name`` argument. The name argument corresponds to
    the name of the class to initialize.

        Config file parser to read.
        All other keyword arguments are passed to the ``from_config`` method
        of the class specified by the name argument.

    Returns
    -------
    cls
        The initialized model.
    """
    # use the name to get the distribution
    name = cp.get("model", "name")
    return models[name].from_config(cp, **kwargs)

models = {_cls.name: _cls for _cls in (
    TestEggbox,
    TestNormal,
    TestRosenbrock,
    TestVolcano,
    TestPosterior,
    TestPrior,
    GaussianNoise,
    MarginalizedPhaseGaussianNoise,
    MarginalizedPolarization,
    MarginalizedHMPolPhase,
    BruteParallelGaussianMarginalize,
    SingleTemplate,
    Relative
)}

#
#

#
#
#                                   Preamble
#
#
"""
Defines the base sampler class to be inherited by all samplers.
"""

from abc import ABCMeta, abstractmethod, abstractproperty

from pycbc.inference.io import validate_checkpoint_files

#
#
#                           Base Sampler definition
#
#

class BaseSampler(object):
    """Abstract base class for all inference samplers.

    All sampler classes must inherit from this class and implement its abstract
    methods.

    model : Model
        An instance of a model from ``pycbc.inference.models``.
    """
    name = None

        self.checkpoint_file = None
        self.backup_file = None
        self.checkpoint_valid = None
        self.new_checkpoint = None

    # @classmethod <--uncomment when we move to python 3.3
    def from_config(cls, cp, model, output_file=None, nprocesses=1,
                    use_mpi=False):
        """This should initialize the sampler given a config file.
        """
        pass

    @property
        """Returns the parameters varied in the model.
        """
        return self.model.variable_params

    @property
        """Returns the sampling params used by the model.
        """
        return self.model.sampling_params

    @property
        """Returns the model's fixed parameters.
        """
        return self.model.static_params

    @abstractproperty
    def samples(self):
        """A dict mapping variable_params to arrays of samples currently
        in memory. The dictionary may also contain sampling_params.

        The sample arrays may have any shape, and may or may not be thinned.
        """
        pass

    def model_stats(self):
        """A dict mapping model's metadata fields to arrays of values for
        each sample in ``raw_samples``.

        The arrays may have any shape, and may or may not be thinned.
        """
        pass

    def run(self):
        """This function should run the sampler.

        Any checkpointing should be done internally in this function.
        """
        pass

    def io(self):
        """A class that inherits from ``BaseInferenceFile`` to handle IO with
        an hdf file.

        This should be a class, not an instance of class, so that the sampler
        can initialize it when needed.
        """
        pass

    def checkpoint(self):
        """The sampler must have a checkpoint method for dumping raw samples
        and stats to the file type defined by ``io``.
        """
        pass

    def finalize(self):
        """Do any finalization to the samples file before exiting."""
        pass

    def resume_from_checkpoint(self):
        """Resume the sampler from the output file.
        """
        pass

#
#
#                           Convenience functions
#
#

def setup_output(sampler, output_file, check_nsamples=True):
    r"""Sets up the sampler's checkpoint and output files.

    The checkpoint file has the same name as the output file, but with
    ``.checkpoint`` appended to the name. A backup file will also be
    created.

    sampler : sampler instance
        Sampler
    output_file : str
        Name of the output file.
    """
    # check for backup file(s)
    checkpoint_file = output_file + '.checkpoint'
    backup_file = output_file + '.bkup'
    # check if we have a good checkpoint and/or backup file
    logging.info("Looking for checkpoint file")
    checkpoint_valid = validate_checkpoint_files(checkpoint_file,
                                                 backup_file,
                                                 check_nsamples)

    # Create a new file if the checkpoint doesn't exist, or if it is
    # corrupted
    sampler.new_checkpoint = False  # keeps track if this is a new file or not
    if not checkpoint_valid:
        logging.info("Checkpoint not found or not valid")
        create_new_output_file(sampler, checkpoint_file)
        # now the checkpoint is valid
        sampler.new_checkpoint = True
        # copy to backup
    # write the command line, startup
    for fn in [checkpoint_file, backup_file]:
        with sampler.io(fn, "a") as fp:
            fp.write_command_line()
            fp.write_resume_point()
            fp.write_run_start_time()
    # store
    sampler.checkpoint_file = checkpoint_file
    sampler.backup_file = backup_file

def create_new_output_file(sampler, filename, **kwargs):
    r"""Creates a new output file.

        Sampler
        Name of the file to create.
        All other keyword arguments are passed through to the file's
        ``write_metadata`` function.
    """
    logging.info("Creating file {}".format(filename))
    with sampler.io(filename, "w") as fp:
        # create the samples group and sampler info group
        fp.create_group(fp.samples_group)
        fp.create_group(fp.sampler_group)
        # save the sampler's metadata
        fp.write_sampler_metadata(sampler)

def initial_dist_from_config(cp, variable_params):
    r"""Loads a distribution for the sampler start from the given config file.

    A distribution will only be loaded if the config file has a [initial-\*]
    section(s).

    cp : Config parser
        The config parser to try to load from.
        The variable parameters for the distribution.

    Returns
    -------
    JointDistribution or None :
        The initial distribution. If no [initial-\*] section found in the
        config file, will just return None.
    """
    if len(cp.get_subsections("initial")):
        logging.info("Using a different distribution for the starting points "
                     "than the prior.")
        initial_dists = distributions.read_distributions_from_config(
            cp, section="initial")
            cp, constraint_section="initial_constraint")
        init_dist = distributions.JointDistribution(
            variable_params, *initial_dists,
            **{"constraints": constraints})
    else:
        init_dist = None
    return init_dist

# Copyright 2007 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.

"""Abstract Base Classes (ABCs) according to PEP 3119."""

def abstractmethod(funcobj):
    """A decorator indicating abstract methods.

    Requires that the metaclass is ABCMeta or derived from it.  A
    class that has a metaclass derived from ABCMeta cannot be
    instantiated unless all of its abstract methods are overridden.
    The abstract methods can be called using any of the normal
    'super' call mechanisms.

    Usage:

        class C(metaclass=ABCMeta):
            @abstractmethod
            def my_abstract_method(self, ...):
                ...
    """
    funcobj.__isabstractmethod__ = True
    return funcobj

class abstractclassmethod(classmethod):
    """A decorator indicating abstract classmethods.

    Similar to abstractmethod.

    Usage:

            @abstractclassmethod
            def my_abstract_classmethod(cls, ...):
                ...

    'abstractclassmethod' is deprecated. Use 'classmethod' with
    'abstractmethod' instead.
    """

    __isabstractmethod__ = True

    def __init__(self, callable):
        callable.__isabstractmethod__ = True
        super().__init__(callable)

class abstractstaticmethod(staticmethod):
    """A decorator indicating abstract staticmethods.

    Usage:

            @abstractstaticmethod
            def my_abstract_staticmethod(...):
                ...

    'abstractstaticmethod' is deprecated. Use 'staticmethod' with
    """

class abstractproperty(property):
    """A decorator indicating abstract properties.

    instantiated unless all of its abstract properties are overridden.
    The abstract properties can be called using any of the normal

    Usage:

            @abstractproperty
            def my_abstract_property(self):
                ...

    This defines a read-only property; you can also define a read-write
    abstract property using the 'long' form of property declaration:

            def getx(self): ...
            def setx(self, value): ...
            x = abstractproperty(getx, setx)

    'abstractproperty' is deprecated. Use 'property' with 'abstractmethod'
    instead.
    """

try:
    from _abc import (get_cache_token, _abc_init, _abc_register,
                      _abc_instancecheck, _abc_subclasscheck, _get_dump,
                      _reset_registry, _reset_caches)
    from _py_abc import ABCMeta, get_cache_token
    ABCMeta.__module__ = 'abc'
else:
    class ABCMeta(type):
        """Metaclass for defining Abstract Base Classes (ABCs).

        Use this metaclass to create an ABC.  An ABC can be subclassed
        directly, and then acts as a mix-in class.  You can also register
        unrelated concrete classes (even built-in classes) and unrelated
        ABCs as 'virtual subclasses' -- these and their descendants will
        be considered subclasses of the registering ABC by the built-in
        issubclass() function, but the registering ABC won't show up in
        their MRO (Method Resolution Order) nor will method
        implementations defined by the registering ABC be callable (not
        even via super()).
        """
        def __new__(mcls, name, bases, namespace, **kwargs):
            cls = super().__new__(mcls, name, bases, namespace, **kwargs)
            _abc_init(cls)
            return cls

        def register(cls, subclass):
            """Register a virtual subclass of an ABC.

            Returns the subclass, to allow usage as a class decorator.
            """
            return _abc_register(cls, subclass)

        def __instancecheck__(cls, instance):
            """Override for isinstance(instance, cls)."""
            return _abc_instancecheck(cls, instance)

        def __subclasscheck__(cls, subclass):
            """Override for issubclass(subclass, cls)."""
            return _abc_subclasscheck(cls, subclass)

        def _dump_registry(cls, file=None):
            """Debug helper to print the ABC registry."""
            print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
            print(f"Inv. counter: {get_cache_token()}", file=file)
            (_abc_registry, _abc_cache, _abc_negative_cache,
             _abc_negative_cache_version) = _get_dump(cls)
            print(f"_abc_registry: {_abc_registry!r}", file=file)
            print(f"_abc_cache: {_abc_cache!r}", file=file)
            print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file)
            print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}",
                  file=file)

        def _abc_registry_clear(cls):
            """Clear the registry (for debugging or testing)."""
            _reset_registry(cls)

        def _abc_caches_clear(cls):
            """Clear the caches (for debugging or testing)."""
            _reset_caches(cls)

class ABC(metaclass=ABCMeta):
    """Helper class that provides a standard way to create an ABC using
    inheritance.
    """
    __slots__ = ()

# Copyright (C) 2020 Sumit Kumar, Alex Nitz
#
#

#
#
#                                   Preamble
#
#
"""
Common utilities for samplers that rely on transforming between a unit cube
and the prior space. This is typical of many nested sampling algorithms.
"""

from .. import models

def call_global_loglikelihood(cube):
    return models._global_instance.log_likelihood(cube)

def call_global_logprior(cube):
    return models._global_instance.prior_transform(cube)

def setup_calls(model, loglikelihood_function=None, copy_prior=False):
    """ Configure calls for MPI support
    """
    model_call = CubeModel(model, loglikelihood_function,
                           copy_prior=copy_prior)

    # these are used to help paralleize over multiple cores / MPI
    models._global_instance = model_call
    log_likelihood_call = call_global_loglikelihood
    prior_call = call_global_logprior
    return log_likelihood_call, prior_call

class CubeModel(object):
    """ Class for making PyCBC Inference 'model class'

    model : inference.BaseModel instance
             A model instance from pycbc.
    """

    def __init__(self, model, loglikelihood_function=None, copy_prior=False):
        if model.sampling_transforms is not None:
            raise ValueError("Ultranest or dynesty do not support sampling transforms")
        if loglikelihood_function is None:
            loglikelihood_function = 'loglikelihood'
        self.loglikelihood_function = loglikelihood_function
        self.copy_prior = copy_prior

    def log_likelihood(self, cube):
        """
        returns log likelihood function
        """
        params = dict(zip(self.model.sampling_params, cube))
        if self.model.logprior == -numpy.inf:
        return getattr(self.model, self.loglikelihood_function)

    def prior_transform(self, cube):
        """
        prior transform function for ultranest sampler
        It takes unit cube as input parameter and apply
        prior transforms
        """
        if self.copy_prior:
            cube = cube.copy()

        # we preserve the type of cube to whatever we were given
        dict_cube = dict(zip(self.model.variable_params, cube))
        inv = self.model.prior_distribution.cdfinv(**dict_cube)
        for i, param in enumerate(self.model.variable_params):
            cube[i] = inv[param]
        return cube

#
#

#
#
#                                   Preamble
#
#
"""Provides constructor classes and convenience functions for MCMC samplers."""

import os
from abc import (ABCMeta, abstractmethod, abstractproperty)

from six.moves import configparser as ConfigParser

from pycbc.filter import autocorrelation
from pycbc.inference.io import (validate_checkpoint_files, loadfile)
from pycbc.inference.io.base_mcmc import nsamples_in_chain

from .base import setup_output
from .base import initial_dist_from_config

#
#
#
#

def raw_samples_to_dict(sampler, raw_samples):
    """Convenience function for converting ND array to a dict of samples.

    The samples are assumed to have dimension
    ``[sampler.base_shape x] niterations x len(sampler.sampling_params)``.

        An instance of an MCMC sampler.
    raw_samples : array
        The array of samples to convert.

    Returns
    -------
    dict :
        A dictionary mapping the raw samples to the variable params. If the
        sampling params are not the same as the variable params, they will
        also be included. Each array will have shape
        ``[sampler.base_shape x] niterations``.
    """
    sampling_params = sampler.sampling_params
    # convert to dictionary
    samples = {param: raw_samples[..., ii] for
               ii, param in enumerate(sampling_params)}
    # apply boundary conditions
    samples = sampler.model.prior_distribution.apply_boundary_conditions(
        **samples)
    # apply transforms to go to model's variable params space
    if sampler.model.sampling_transforms is not None:
        samples = sampler.model.sampling_transforms.apply(
            samples, inverse=True)

def blob_data_to_dict(stat_names, blobs):
    """Converts list of "blobs" to a dictionary of model stats.

    Samplers like ``emcee`` store the extra tuple returned by ``CallModel`` to
    a list called blobs. This is a list of lists of tuples with shape
    niterations x nwalkers x nstats, where nstats is the number of stats
    returned by the model's ``default_stats``. This converts that list to a
    dictionary of arrays keyed by the stat names.

    stat_names : list of str
        The list of the stat names.
    blobs : list of list of tuples
        The data to convert.

    Returns
    -------
    dict :
        A dictionary mapping the model's ``default_stats`` to arrays of values.
        Each array will have shape ``nwalkers x niterations``.
    """
    # get the dtypes of each of the stats; we'll just take this from the
    # first iteration and walker
    dtypes = [type(val) for val in blobs[0][0]]
    assert len(stat_names) == len(dtypes), (
        "number of stat names must match length of tuples in the blobs")
    # convert to an array; to ensure that we get the dtypes correct, we'll
    # cast to a structured array
    raw_stats = numpy.array(blobs, dtype=list(zip(stat_names, dtypes)))
    # transpose so that it has shape nwalkers x niterations
    raw_stats = raw_stats.transpose()
    # now return as a dictionary
    return {stat: raw_stats[stat] for stat in stat_names}

def get_optional_arg_from_config(cp, section, arg, dtype=str):
    """Convenience function to retrieve an optional argument from a config
    file.

        Open config parser to retrieve the argument from.
        Name of the section to retrieve from.
    arg : str
        Name of the argument to retrieve.
    dtype : datatype, optional
        Cast the retrieved value (if it exists) to the given datatype. Default
        is ``str``.

    Returns
    -------
    val : None or str
        If the argument is present, the value. Otherwise, None.
    """
    if cp.has_option(section, arg):
        val = dtype(cp.get(section, arg))
    else:
        val = None
    return val

#
#
#                              BaseMCMC definition
#
#

class BaseMCMC(object):
    """Abstract base class that provides methods common to MCMCs.

    This is not a sampler class itself. Sampler classes can inherit from this
    along with ``BaseSampler``.

    This class provides ``set_initial_conditions``, ``run``, and ``checkpoint``
    methods, which are some of the abstract methods required by
    ``BaseSampler``.

    This class introduces the following abstract properties and methods:

    * base_shape
        [`property`] Should give the shape of the samples arrays used by the
        sampler, excluding the iteraitons dimension. Needed for writing
        results.
    * run_mcmc(niterations)
        Should run the sampler for the given number of iterations. Called by
        ``run``.
    * clear_samples()
        Should clear samples from memory. Called by ``run``.
    * set_state_from_file(filename)
        Should set the random state of the sampler using the given filename.
        Called by ``set_initial_conditions``.
    * write_results(filename)
        Writes results to the given filename. Called by ``checkpoint``.
    * compute_acf(filename, \**kwargs)
        [`classmethod`] Should compute the autocorrelation function using
        the given filename. Also allows for other keyword arguments.
    * compute_acl(filename, \**kwargs)
        [`classmethod`] Should compute the autocorrelation length using

    p0
    pos
    nchains
    niterations
    checkpoint_interval
    checkpoint_signal
    target_niterations
    target_eff_nsamples
    thin_interval
    max_samples_per_chain
    thin_safety_factor
    burn_in
    effective_nsamples
    acl
    raw_acls
    act
    raw_acts
    """
    _lastclear = None  # the iteration when samples were cleared from memory
    _itercounter = None  # the number of iterations since the last clear
    _pos = None
    _p0 = None
    _nchains = None
    _burn_in = None
    _acls = None
    _checkpoint_interval = None
    _checkpoint_signal = None
    _target_niterations = None
    _target_eff_nsamples = None
    _thin_interval = 1
    _max_samples_per_chain = None

    def base_shape(self):
        """What shape the sampler's samples arrays are in, excluding
        the iterations dimension.

        For example, if a sampler uses 20 chains and 3 temperatures, this
        would be ``(3, 20)``. If a sampler only uses a single walker and no
        temperatures this would be ``()``.
        """
        pass

    @property
        """The number of chains used."""
        if self._nchains is None:
            raise ValueError("number of chains not set")
        return self._nchains

    @nchains.setter
    def nchains(self, value):
        """Sets the number of chains."""
        # we'll actually store it to the nchains attribute
        self._nchains = int(value)

    @property
        """The current number of iterations."""
        itercounter = self._itercounter
        if itercounter is None:
            itercounter = 0
        lastclear = self._lastclear
        if lastclear is None:
            lastclear = 0
        return itercounter + lastclear

    @property
    def checkpoint_interval(self):
        """The number of iterations to do between checkpoints."""
        return self._checkpoint_interval

    @property
    def checkpoint_signal(self):
        """The signal to use when checkpointing."""
        return self._checkpoint_signal

    @property
    def target_niterations(self):
        """The number of iterations the sampler should run for."""
        return self._target_niterations

    @property
    def target_eff_nsamples(self):
        """The target number of effective samples the sampler should get."""
        return self._target_eff_nsamples

    @property
        """Returns the thin interval being used."""
        return self._thin_interval

    @thin_interval.setter
    def thin_interval(self, interval):
        """Sets the thin interval to use.

        If ``None`` provided, will default to 1.
        """
        if interval is None:
            interval = 1
        if interval < 1:
            raise ValueError("thin interval must be >= 1")
        self._thin_interval = interval

    @property
    def thin_safety_factor(self):
        """The minimum value that ``max_samples_per_chain`` may be set to."""
        return 100

    @property
    def max_samples_per_chain(self):
        """The maximum number of samplers per chain that is written to disk."""
        return self._max_samples_per_chain

    @max_samples_per_chain.setter
    def max_samples_per_chain(self, n):
        if n is not None:
            n = int(n)
            if n < self.thin_safety_factor:
                raise ValueError("max samples per chain must be >= {}"
                                 .format(self.thin_safety_factor))
            # also check that this is consistent with the target number of
            # effective samples
            if self.target_eff_nsamples is not None:
                target_samps_per_chain = int(numpy.ceil(
                    self.target_eff_nsamples / self.nchains))
                if n <= target_samps_per_chain:
                    raise ValueError("max samples per chain must be > target "
                                     "effective number of samples per walker "
                                     "({})".format(target_samps_per_chain))
        self._max_samples_per_chain = n

    def get_thin_interval(self):
        """Gets the thin interval to use.

        If ``max_samples_per_chain`` is set, this will figure out what thin
        interval is needed to satisfy that criteria. In that case, the thin
        interval used must be a multiple of the currently used thin interval.
        """
        if self.max_samples_per_chain is not None:
            # the extra factor of 2 is to account for the fact that the thin
            # interval will need to be at least twice as large as a previously
            # used interval
            thinfactor = 2*(self.niterations // self.max_samples_per_chain)
            # make sure it's at least 1
            thinfactor = max(thinfactor, 1)
            # make the new interval is a multiple of the previous, to ensure
            # that any samples currently on disk can be thinned accordingly
            if thinfactor < self.thin_interval:
                thin_interval = self.thin_interval
            else:
                thin_interval = (thinfactor // self.thin_interval) * \
                    self.thin_interval
        else:
            thin_interval = self.thin_interval
        return thin_interval

    def set_target(self, niterations=None, eff_nsamples=None):
        """Sets the target niterations/nsamples for the sampler.

        One or the other must be provided, not both.
        """
        if niterations is None and eff_nsamples is None:
            raise ValueError("Must provide a target niterations or "
                             "eff_nsamples")
        if niterations is not None and eff_nsamples is not None:
                             "eff_nsamples, not both")
        self._target_niterations = int(niterations) \
            if niterations is not None else None
        self._target_eff_nsamples = int(eff_nsamples) \
            if eff_nsamples is not None else None

    def clear_samples(self):
        """A method to clear samples from memory."""
        pass

    @property
    def pos(self):
        """A dictionary of the current walker positions.

        If the sampler hasn't been run yet, returns p0.
        """
        pos = self._pos
        if pos is None:
            return self.p0
        # convert to dict
        pos = {param: self._pos[..., k]
               for (k, param) in enumerate(self.sampling_params)}
        return pos

    @property
    def p0(self):
        """A dictionary of the initial position of the chains.

        This is set by using ``set_p0``. If not set yet, a ``ValueError`` is
        raised when the attribute is accessed.
        """
        if self._p0 is None:
            raise ValueError("initial positions not set; run set_p0")
        p0 = {param: self._p0[..., k]
              for (k, param) in enumerate(self.sampling_params)}
        return p0

    def set_p0(self, samples_file=None, prior=None):
        """Sets the initial position of the chains.

        samples_file : InferenceFile, optional
            If provided, use the last iteration in the given file for the
            starting positions.
            Use the given prior to set the initial positions rather than
            ``model``'s prior.

        Returns
        -------
        p0 : dict
            A dictionary maping sampling params to the starting positions.
        """
        # if samples are given then use those as initial positions
        if samples_file is not None:
            with self.io(samples_file, 'r') as fp:
                samples = fp.read_samples(self.variable_params,
                                          iteration=-1, flatten=False)
                # remove the (length 1) niterations dimension
                samples = samples[..., 0]
                # make sure we have the same shape
                assert samples.shape == self.base_shape, (
                       "samples in file {} have shape {}, but I have shape {}".
                       format(samples_file, samples.shape, self.base_shape))
            # transform to sampling parameter space
            if self.model.sampling_transforms is not None:
                samples = self.model.sampling_transforms.apply(samples)
        # draw random samples if samples are not provided
        else:
            nsamples = numpy.prod(self.base_shape)
            samples = self.model.prior_rvs(size=nsamples, prior=prior).reshape(
                self.base_shape)
        # store as ND array with shape [base_shape] x nparams
        ndim = len(self.variable_params)
        p0 = numpy.ones(list(self.base_shape)+[ndim])
        for i, param in enumerate(self.sampling_params):
            p0[..., i] = samples[param]
        self._p0 = p0
        return self.p0

    def set_state_from_file(self, filename):
        """Sets the state of the sampler to the instance saved in a file.
        """
        pass

    def set_start_from_config(self, cp):
        """Sets the initial state of the sampler from config file
        """
        if cp.has_option('sampler', 'start-file'):
            start_file = cp.get('sampler', 'start-file')
            logging.info("Using file %s for initial positions", start_file)
            init_prior = None
        else:
            start_file = None
            init_prior = initial_dist_from_config(cp, self.variable_params)
        self.set_p0(samples_file=start_file, prior=init_prior)

        """Resume the sampler from the checkpoint file
        """
        with self.io(self.checkpoint_file, "r") as fp:
            self._lastclear = fp.niterations
        self.set_p0(samples_file=self.checkpoint_file)
        self.set_state_from_file(self.checkpoint_file)

        """Runs the sampler."""
        if self.target_eff_nsamples and self.checkpoint_interval is None:
            raise ValueError("A checkpoint interval must be set if "
                             "targetting an effective number of samples")
        # get the starting number of samples:
        # "nsamples" keeps track of the number of samples we've obtained (if
        # target_eff_nsamples is not None, this is the effective number of
        # samples; otherwise, this is the total number of samples).
        # contains (either due to sampler burn-in, or a previous checkpoint)
        if self.new_checkpoint:
            self._lastclear = 0
        else:
            with self.io(self.checkpoint_file, "r") as fp:
                self._lastclear = fp.niterations
                self.thin_interval = fp.thinned_by
        if self.target_eff_nsamples is not None:
            target_nsamples = self.target_eff_nsamples
                nsamples = fp.effective_nsamples
        elif self.target_niterations is not None:
            # the number of samples is the number of iterations times the
            # number of chains
            target_nsamples = self.nchains * self.target_niterations
            nsamples = self._lastclear * self.nchains
        else:
            raise ValueError("must set either target_eff_nsamples or "
                             "target_niterations; see set_target")
        self._itercounter = 0
        # figure out the interval to use
        iterinterval = self.checkpoint_interval
        if iterinterval is None:
            iterinterval = self.target_niterations
        # run sampler until we have the desired number of samples
        while nsamples < target_nsamples:
            # adjust the interval if we would go past the number of iterations
            if self.target_niterations is not None and (
                    self.niterations + iterinterval > self.target_niterations):
                iterinterval = self.target_niterations - self.niterations
            # run sampler and set initial values to None so that sampler
            # picks up from where it left off next call
            logging.info("Running sampler for {} to {} iterations".format(
                self.niterations, self.niterations + iterinterval))
            # run the underlying sampler for the desired interval
            self.run_mcmc(iterinterval)
            # update the itercounter
            self._itercounter = self._itercounter + iterinterval
            # dump the current results
            self.checkpoint()
            # update nsamples for next loop
                nsamples = self.effective_nsamples
                logging.info("Have {} effective samples post burn in".format(
                    nsamples))
            else:
                nsamples += iterinterval * self.nchains

    @property
    def burn_in(self):
        """The class for doing burn-in tests (if specified)."""
        return self._burn_in

    def set_burn_in(self, burn_in):
        """Sets the object to use for doing burn-in tests."""
        self._burn_in = burn_in

        """The effective number of samples post burn-in that the sampler has
        acquired so far.
        """
        pass

    def run_mcmc(self, niterations):
        """Run the MCMC for the given number of iterations."""
        pass

    def write_results(self, filename):
        """Should write all samples currently in memory to the given file."""
        pass

        """Dumps current samples to the checkpoint file."""
        # thin and write new samples
        # get the updated thin interval to use
        thin_interval = self.get_thin_interval()
        for fn in [self.checkpoint_file, self.backup_file]:
            with self.io(fn, "a") as fp:
                # write the current number of iterations
                fp.write_niterations(self.niterations)
                # thin samples on disk if it changed
                if thin_interval > 1:
                    # if this is the first time writing, set the file's
                    # thinned_by
                    if fp.last_iteration() == 0:
                        fp.thinned_by = thin_interval
                    elif thin_interval < fp.thinned_by:
                        # whatever was done previously resulted in a larger
                        # thin interval, so we'll set it to the file's
                        thin_interval = fp.thinned_by
                    elif thin_interval > fp.thinned_by:
                        # we need to thin the samples on disk
                        logging.info("Thinning samples in %s by a factor "
                                     "of %i", fn, int(thin_interval))
                        fp.thin(thin_interval)
                fp_lastiter = fp.last_iteration()
            logging.info("Writing samples to %s with thin interval %i", fn,
                         thin_interval)
            self.write_results(fn)
        # update the running thin interval
        self.thin_interval = thin_interval
        # see if we had anything to write after thinning; if not, don't try
        # to compute anything
            nsamples_written = fp.last_iteration() - fp_lastiter
        if nsamples_written == 0:
            logging.info("No samples written due to thinning")
        else:
            # check for burn in, compute the acls
            self.raw_acls = None
            if self.burn_in is not None:
                logging.info("Updating burn in")
                self.burn_in.evaluate(self.checkpoint_file)
                # write
                for fn in [self.checkpoint_file, self.backup_file]:
                    with self.io(fn, "a") as fp:
                        self.burn_in.write(fp)
            logging.info("Computing autocorrelation time")
            self.raw_acls = self.compute_acl(self.checkpoint_file)
            # write acts, effective number of samples
            for fn in [self.checkpoint_file, self.backup_file]:
                with self.io(fn, "a") as fp:
                    if self.raw_acls is not None:
                        fp.raw_acls = self.raw_acls
                        fp.acl = self.acl
                    # write effective number of samples
                    fp.write_effective_nsamples(self.effective_nsamples)
        # write history
                fp.update_checkpoint_history()
        # check validity
        logging.info("Validating checkpoint and backup files")
        checkpoint_valid = validate_checkpoint_files(
            self.checkpoint_file, self.backup_file)
        if not checkpoint_valid:
            raise IOError("error writing to checkpoint file")
        elif self.checkpoint_signal:
            # kill myself with the specified signal
            logging.info("Exiting with SIG{}".format(self.checkpoint_signal))
            kill_cmd="os.kill(os.getpid(), signal.SIG{})".format(
                self.checkpoint_signal)
            exec(kill_cmd)
        # clear the in-memory chain to save memory
        logging.info("Clearing samples from memory")
        self.clear_samples()

    def checkpoint_from_config(cp, section):
        """Gets the checkpoint interval from the given config file.

        This looks for 'checkpoint-interval' in the section.

            Open config parser to retrieve the argument from.
            Name of the section to retrieve from.

        Return
        ------
        int or None :
            The checkpoint interval, if it is in the section. Otherw
        """
        return get_optional_arg_from_config(cp, section, 'checkpoint-interval',
                                            dtype=int)

    def ckpt_signal_from_config(cp, section):
        """Gets the checkpoint signal from the given config file.

        This looks for 'checkpoint-signal' in the section.

        Return
        ------
        """
        return get_optional_arg_from_config(cp, section, 'checkpoint-signal',
                                            dtype=str)

    def set_target_from_config(self, cp, section):
        """Sets the target using the given config file.

        This looks for ``niterations`` to set the ``target_niterations``, and
        ``effective-nsamples`` to set the ``target_eff_nsamples``.

        """
        if cp.has_option(section, "niterations"):
            niterations = int(cp.get(section, "niterations"))
        else:
            niterations = None
        if cp.has_option(section, "effective-nsamples"):
            nsamples = int(cp.get(section, "effective-nsamples"))
        else:
        self.set_target(niterations=niterations, eff_nsamples=nsamples)

    def set_burn_in_from_config(self, cp):
        """Sets the burn in class from the given config file.

        If no burn-in section exists in the file, then this just set the
        burn-in class to None.
        """
        try:
            bit = self.burn_in_class.from_config(cp, self)
        except ConfigParser.Error:
            bit = None
        self.set_burn_in(bit)

    def set_thin_interval_from_config(self, cp, section):
        """Sets thinning options from the given config file.
        """
        if cp.has_option(section, "thin-interval"):
            thin_interval = int(cp.get(section, "thin-interval"))
            logging.info("Will thin samples using interval %i", thin_interval)
        else:
            thin_interval = None
        if cp.has_option(section, "max-samples-per-chain"):
            max_samps_per_chain = int(cp.get(section, "max-samples-per-chain"))
            logging.info("Setting max samples per chain to %i",
                         max_samps_per_chain)
        else:
            max_samps_per_chain = None
        # check for consistency
        if thin_interval is not None and max_samps_per_chain is not None:
            raise ValueError("provide either thin-interval or "
                             "max-samples-per-chain, not both")
        # check that the thin interval is < then the checkpoint interval
        if thin_interval is not None and self.checkpoint_interval is not None \
                and thin_interval >= self.checkpoint_interval:
            raise ValueError("thin interval must be less than the checkpoint "
                             "interval")
        self.max_samples_per_chain = max_samps_per_chain

    @property
        """Dictionary of parameter names -> autocorrelation lengths.

        Depending on the sampler, the ACLs may be an integer, or an arrray of
        values per chain and/or per temperature.

        Returns ``None`` if no ACLs have been calculated.
        """
        return self._acls

        """Sets the raw acls."""
        self._acls = acls

        """The autocorrelation length.

        This method should convert the raw ACLs into an integer or array that
        can be used to extract independent samples from a chain.
        """
        pass

    @property
        """Dictionary of parameter names -> autocorrelation time(s).

        """
        acls = self.raw_acls
        if acls is None:
        return {p: acl * self.thin_interval
                for (p, acl) in acls.items()}

    @property
        """The autocorrelation time(s).

        The autocorrelation time is defined as the autocorrelation length times
        the ``thin_interval``. It gives the number of iterations between
        independent samples. Depending on the sampler, this may either be
        a single integer or an array of values.

        """
        acl = self.acl
        if acl is None:
        return acl * self.thin_interval

    def compute_acf(cls, filename, **kwargs):
        """A method to compute the autocorrelation function of samples in the
        given file."""
        pass

    def compute_acl(cls, filename, **kwargs):
        """A method to compute the autocorrelation length of samples in the
        pass

class EnsembleSupport(object):
    """Adds support for ensemble MCMC samplers."""

    @property
        """The number of walkers used.

        """
        return self.nchains

    @nwalkers.setter
    def nwalkers(self, value):
        """Sets the number of walkers."""
        self.nchains = value

    @property
        """The autocorrelation length of the ensemble.

        This is calculated by taking the maximum over all of the ``raw_acls``.
        This works for both single and parallel-tempered ensemble samplers.

        Returns ``None`` if no ACLs have been set.
        """
        return numpy.array(list(acls.values())).max()

    @property
        """
        if self.burn_in is not None and not self.burn_in.is_burned_in:
            # not burned in, so there's no effective samples
            return 0
        act = self.act
        if act is None:
            act = numpy.inf
        if self.burn_in is None:
            start_iter = 0
        else:
            start_iter = self.burn_in.burn_in_iteration
        nperwalker = nsamples_in_chain(start_iter, act, self.niterations)
        if self.burn_in is not None:
            # after burn in, we always have atleast 1 sample per walker
            nperwalker = max(nperwalker, 1)
        return int(self.nwalkers * nperwalker)

#
#
#              Functions for computing autocorrelation lengths
#
#

def ensemble_compute_acf(filename, start_index=None, end_index=None,
                         per_walker=False, walkers=None, parameters=None):
    """Computes the autocorrleation function for an ensemble MCMC.

    By default, parameter values are averaged over all walkers at each
    iteration. The ACF is then calculated over the averaged chain. An
    ACF per-walker will be returned instead if ``per_walker=True``.

        Name of a samples file to compute ACFs for.
    start_index : int, optional
        The start index to compute the acl from. If None (the default), will
        try to use the number of burn-in iterations in the file; otherwise,
        will start at the first sample.
    end_index : int, optional
        The end index to compute the acl to. If None (the default), will go to
        the end of the current iteration.
    per_walker : bool, optional
        Return the ACF for each walker separately. Default is False.
    walkers : int or array, optional
        Calculate the ACF using only the given walkers. If None (the
        default) all walkers will be used.
    parameters : str or array, optional
        Calculate the ACF for only the given parameters. If None (the
        default) will calculate the ACF for all of the model params.

    Returns
    -------
    dict :
        Dictionary of arrays giving the ACFs for each parameter. If
        ``per-walker`` is True, the arrays will have shape
        ``nwalkers x niterations``.
    """
    acfs = {}
            parameters = fp.variable_params
        if isinstance(parameters, string_types):
            parameters = [parameters]
            if per_walker:
                # just call myself with a single walker
                if walkers is None:
                    walkers = numpy.arange(fp.nwalkers)
                arrays = [
                    ensemble_compute_acf(filename, start_index=start_index,
                                         end_index=end_index,
                                         per_walker=False, walkers=ii,
                                         parameters=param)[param]
                    for ii in walkers]
                acfs[param] = numpy.vstack(arrays)
            else:
                samples = fp.read_raw_samples(
                    param, thin_start=start_index, thin_interval=1,
                    thin_end=end_index, walkers=walkers,
                    flatten=False)[param]
                samples = samples.mean(axis=0)
                acfs[param] = autocorrelation.calculate_acf(
                    samples).numpy()
    return acfs

def ensemble_compute_acl(filename, start_index=None, end_index=None,
                         min_nsamples=10):
    """Computes the autocorrleation length for an ensemble MCMC.

    Parameter values are averaged over all walkers at each iteration.
    The ACL is then calculated over the averaged chain. If an ACL cannot
    be calculated because there are not enough samples, it will be set
    to ``inf``.

        Name of a samples file to compute ACLs for.
        The start index to compute the acl from. If None, will try to use
        the number of burn-in iterations in the file; otherwise, will start
        at the first sample.
        The end index to compute the acl to. If None, will go to the end
        of the current iteration.
    min_nsamples : int, optional
        Require a minimum number of samples to compute an ACL. If the
        number of samples per walker is less than this, will just set to
        ``inf``. Default is 10.

    Returns
    -------
    dict
        A dictionary giving the ACL for each parameter.
    """
    acls = {}
        for param in fp.variable_params:
                param, thin_start=start_index, thin_interval=1,
                thin_end=end_index, flatten=False)[param]
            samples = samples.mean(axis=0)
            # if < min number of samples, just set to inf
            if samples.size < min_nsamples:
                acl = numpy.inf
            else:
                acl = autocorrelation.calculate_acl(samples)
            if acl <= 0:
            acls[param] = acl
        maxacl = numpy.array(list(acls.values())).max()
        logging.info("ACT: %s", str(maxacl*fp.thinned_by))
    return acls

#
#

#
#
#                                   Preamble
#
#
"""Provides constructor classes provide support for parallel tempered MCMC
samplers."""

from pycbc.inference.io import loadfile

class MultiTemperedSupport(object):
    """Provides methods for supporting multi-tempered samplers.
    """
    _ntemps = None

    @property
        """The number of temeratures that are set."""
        return self._ntemps

    def betas_from_config(cp, section):
        """Loads number of temperatures or betas from a config file.

        This looks in the given section for:

        * ``ntemps`` :
            The number of temperatures to use. Either this, or
            ``inverse-temperatures-file`` must be provided (but not both).
        * ``inverse-temperatures-file`` :
            Path to an hdf file containing the inverse temperatures ("betas")
            to use. The betas will be retrieved from the file's
            ``.attrs['betas']``. Either this or ``ntemps`` must be provided
            (but not both).

            Config file object to parse.

        Returns
        -------
        ntemps : int or None
            The number of temperatures to use, if it was provided.
        betas : array
            The array of betas to use, if a inverse-temperatures-file was
            provided.
        """
        if cp.has_option(section, "ntemps") and \
                cp.has_option(section, "inverse-temperatures-file"):
            raise ValueError("Must specify either ntemps or "
                             "inverse-temperatures-file, not both.")
        if cp.has_option(section, "inverse-temperatures-file"):
            # get the path of the file containing inverse temperatures values.
            inverse_temperatures_file = cp.get(section,
                                               "inverse-temperatures-file")
            betas = read_betas_from_hdf(inverse_temperatures_file)
            ntemps = betas.shape[0]
        else:
            # get the number of temperatures
            betas = None
            ntemps = int(cp.get(section, "ntemps"))
        return ntemps, betas

def read_betas_from_hdf(filename):
    """Loads inverse temperatures from the given file.
    """
    # get the path of the file containing inverse temperatures values.
    with h5py.File(filename, "r") as fp:
        try:
            betas = numpy.array(fp.attrs['betas'])
            # betas must be in decending order
            betas = numpy.sort(betas)[::-1]
            raise AttributeError("No attribute called betas")
    return betas

#
#
#
#

def compute_acf(filename, start_index=None, end_index=None,
                chains=None, parameters=None, temps=None):
    """Computes the autocorrleation function for independent MCMC chains with

        The start index to compute the acl from. If None (the default),
        will try to use the burn in iteration for each chain;
        otherwise, will start at the first sample.
    end_index : {None, int}
    chains : optional, int or array
        Calculate the ACF for only the given chains. If None (the
        default) ACFs for all chains will be estimated.
    parameters : optional, str or array
    temps : optional, (list of) int or 'all'
        The temperature index (or list of indices) to retrieve. If None
        (the default), the ACF will only be computed for the coldest (= 0)
        temperature chain. To compute an ACF for all temperates pass 'all',
        or a list of all of the temperatures.

    Returns
    -------
    dict :
        Dictionary parameter name -> ACF arrays. The arrays have shape
        ``ntemps x nchains x niterations``.
    """
    acfs = {}
        temps = _get_temps_idx(fp, temps)
            chains = numpy.arange(fp.nchains)
            subacfs = []
            for tk in temps:
                subsubacfs = []
                for ci in chains:
                    samples = fp.read_raw_samples(
                        param, thin_start=start_index, thin_interval=1,
                        thin_end=end_index, chains=ci, temps=tk)[param]
                    thisacf = autocorrelation.calculate_acf(samples).numpy()
                    subsubacfs.append(thisacf)
                # stack the chains
                subacfs.append(subsubacfs)
            # stack the temperatures
            acfs[param] = numpy.stack(subacfs)

def compute_acl(filename, start_index=None, end_index=None,
                min_nsamples=10):
    """Computes the autocorrleation length for independent MCMC chains with

    ACLs are calculated separately for each chain.

    start_index : {None, int}

    Returns
    -------
    dict
        A dictionary of ntemps x nchains arrays of the ACLs of each
    """
    # following is a convenience function to calculate the acl for each chain
    # defined here so that we can use map for this below
    def _getacl(si):
        # si: the samples loaded for a specific chain; may have nans in it
        si = si[~numpy.isnan(si)]
        if len(si) < min_nsamples:
            acl = numpy.inf
        else:
            acl = autocorrelation.calculate_acl(si)
        if acl <= 0:
        return acl
    acls = {}
        tidx = numpy.arange(fp.ntemps)
            these_acls = numpy.zeros((fp.ntemps, fp.nchains))
            for tk in tidx:
                    thin_end=end_index, temps=tk, flatten=False)[param]
                # flatten out the temperature
                samples = samples[0, ...]
                # samples now has shape nchains x maxiters
                if samples.shape[-1] < min_nsamples:
                    these_acls[tk, :] = numpy.inf
                else:
                    these_acls[tk, :] = list(map(_getacl, samples))
            acls[param] = these_acls
        # report the mean ACL: take the max over the temps and parameters
        act = acl_from_raw_acls(acls)*fp.thinned_by
        finite = act[numpy.isfinite(act)]
        logging.info("ACTs: min %s, mean (of finite) %s, max %s",
                     str(act.min()),
                     str(finite.mean() if finite.size > 0 else numpy.inf),
                     str(act.max()))

def acl_from_raw_acls(acls):
    """Calculates the ACL for one or more chains from a dictionary of ACLs.

    This is for parallel tempered MCMCs in which the chains are independent
    of each other.

    The ACL for each chain is maximized over the temperatures and parameters.

    acls : dict
        Dictionary of parameter names -> ntemps x nchains arrays of ACLs (the
        thing returned by :py:func:`compute_acl`).

    Returns
    -------
    array
        The ACL of each chain.
    """
    return numpy.array(list(acls.values())).max(axis=0).max(axis=0)

                         per_walker=False, walkers=None, parameters=None,
                         temps=None):
    """Computes the autocorrleation function for a parallel tempered, ensemble
    MCMC.

    iteration. The ACF is then calculated over the averaged chain for each
    temperature. An ACF per-walker will be returned instead if
    ``per_walker=True``.

    temps : (list of) int or 'all', optional

    Returns
    -------
    dict :
        ``ntemps x nwalkers x niterations``. Otherwise, the returned array
        will have shape ``ntemps x niterations``.
    """
    acfs = {}
                if per_walker:
                    # just call myself with a single walker
                    if walkers is None:
                        walkers = numpy.arange(fp.nwalkers)
                    arrays = [ensemble_compute_acf(filename,
                                                   start_index=start_index,
                                                   end_index=end_index,
                                                   per_walker=False,
                                                   walkers=ii,
                                                   parameters=param,
                                                   temps=tk)[param][0, :]
                              for ii in walkers]
                    # we'll stack all of the walker arrays to make a single
                    # nwalkers x niterations array; when these are stacked
                    # below, we'll get a ntemps x nwalkers x niterations
                    # array
                    subacfs.append(numpy.vstack(arrays))
                else:
                        param, thin_start=start_index,
                        thin_interval=1, thin_end=end_index,
                        walkers=walkers, temps=tk, flatten=False)[param]
                    # contract the walker dimension using the mean, and
                    # flatten the (length 1) temp dimension
                    samples = samples.mean(axis=1)[0, :]
                    thisacf = autocorrelation.calculate_acf(
                        samples).numpy()
                    subacfs.append(thisacf)

    """Computes the autocorrleation length for a parallel tempered, ensemble
    MCMC.

    Parameter values are averaged over all walkers at each iteration and
    temperature.  The ACL is then calculated over the averaged chain.

    Returns
    -------
    dict
        A dictionary of ntemps-long arrays of the ACLs of each parameter.
    """
    acls = {}
        if end_index is None:
            end_index = fp.niterations
            these_acls = numpy.zeros(fp.ntemps)
                # contract the walker dimension using the mean, and flatten
                # the (length 1) temp dimension
                samples = samples.mean(axis=1)[0, :]
                if samples.size < min_nsamples:
                    acl = numpy.inf
                else:
                    acl = autocorrelation.calculate_acl(samples)
                if acl <= 0:
                these_acls[tk] = acl

def _get_temps_idx(fp, temps):
    """Gets the indices of temperatures to load for computing ACF.
    """
    if isinstance(temps, int):
        temps = [temps]
    elif temps == 'all':
        temps = numpy.arange(fp.ntemps)
    elif temps is None:
        temps = [0]
    return temps

# Copyright (C) 2019  Collin Capano, Sumit Kumar, Prayush Kumar
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides classes and functions for using the dynesty sampler
packages for parameter estimation.
"""

import os
import dynesty, dynesty.dynesty, dynesty.nestedsamplers
from dynesty.utils import unitcheck, reflect
from pycbc.pool import choose_pool
from dynesty import utils as dyfunc
from pycbc.inference.io import (DynestyFile, validate_checkpoint_files,
                                loadfile)
from pycbc.distributions import read_constraints_from_config
from .base import (BaseSampler, setup_output)
from .base_mcmc import get_optional_arg_from_config
from .base_cube import setup_calls

#
#
#                                   Samplers
#
#

class DynestySampler(BaseSampler):
    """This class is used to construct an Dynesty sampler from the dynesty
    package.

    model : model
        A model from ``pycbc.inference.models``.
    nlive : int
        Number of live points to use in sampler.
    pool : function with map, Optional
        A provider of a map function that allows a function call to be run
        over multiple sets of arguments and possibly maps them to
        cores/nodes/etc.
    """
    name = "dynesty"
    _io = DynestyFile

    def __init__(self, model, nlive, nprocesses=1,
                 checkpoint_time_interval=None, maxcall=None,
                 loglikelihood_function=None, use_mpi=False,
                 no_save_state=False,
                 run_kwds=None, **kwargs):

        self.no_save_state = no_save_state
        log_likelihood_call, prior_call = setup_calls(
            model,
            loglikelihood_function=loglikelihood_function)
        # Set up the pool
        self.pool = choose_pool(mpi=use_mpi, processes=nprocesses)

        self.maxcall = maxcall
        self.checkpoint_time_interval = checkpoint_time_interval
        self.run_kwds = {} if run_kwds is None else run_kwds
        self.nlive = nlive
        self.names = model.sampling_params
        self.ndim = len(model.sampling_params)
        # Enable checkpointing if checkpoint_time_interval is set in config
        # file in sampler section
        if self.checkpoint_time_interval:
            self.run_with_checkpoint = True
            if self.maxcall is None:
                self.maxcall = 5000 * self.pool.size
            logging.info("Checkpointing enabled, will verify every %s calls"
                         " and try to checkpoint every %s seconds",
                         self.maxcall, self.checkpoint_time_interval)
        else:
            self.run_with_checkpoint = False

        # Check for cyclic boundaries
        periodic = []
        cyclic = self.model.prior_distribution.cyclic
        for i, param in enumerate(self.variable_params):
            if param in cyclic:
                logging.info('Param: %s will be cyclic', param)
                periodic.append(i)

        if len(periodic) == 0:
            periodic = None

        # Check for reflected boundaries. Dynesty only supports
        # reflection on both min and max of boundary.
        reflective = []
        reflect = self.model.prior_distribution.well_reflected
            if param in reflect:
                logging.info("Param: %s will be well reflected", param)
                reflective.append(i)

        if len(reflective) == 0:
            reflective = None

        if 'sample' in kwargs:
            if 'rwalk2' in kwargs['sample']:
                dynesty.dynesty._SAMPLING["rwalk"] = sample_rwalk_mod
                dynesty.nestedsamplers._SAMPLING["rwalk"] = sample_rwalk_mod
                kwargs['sample'] = 'rwalk'

        if self.nlive < 0:
            # Interpret a negative input value for the number of live points
            # (which is clearly an invalid input in all senses)
            # as the desire to dynamically determine that number
            self._sampler = dynesty.DynamicNestedSampler(log_likelihood_call,
                                                         prior_call, self.ndim,
                                                         pool=self.pool,
                                                         reflective=reflective,
                                                         periodic=periodic,
                                                         **kwargs)
            logging.info("Checkpointing not currently supported with"
                         "DYNAMIC nested sampler")
        else:
            self._sampler = dynesty.NestedSampler(log_likelihood_call,
                                                  prior_call, self.ndim,
                                                  nlive=self.nlive,
                                                  reflective=reflective,
                                                  periodic=periodic,
                                                  pool=self.pool, **kwargs)

        # properties of the internal sampler which should not be pickled
        self.no_pickle = ['loglikelihood',
                          'prior_transform',
                          'propose_point',
                          'update_proposal',
                          '_UPDATE', '_PROPOSE',
                          'evolve_point']

        diff_niter = 1
        if self.run_with_checkpoint is True:
            n_checkpointing = 1
            t0 = time.time()
            it = self._sampler.it

            logging.info('Starting from iteration: %s', it)
            while diff_niter != 0:
                self._sampler.run_nested(maxcall=self.maxcall, **self.run_kwds)

                delta_t = time.time() - t0
                diff_niter = self._sampler.it - it
                logging.info("Checking if we should checkpoint: %.2f s", delta_t)

                if delta_t >= self.checkpoint_time_interval:
                    logging.info('Checkpointing N={}'.format(n_checkpointing))
                    self.checkpoint()
                    n_checkpointing += 1
                    t0 = time.time()
                it = self._sampler.it
        else:
            self._sampler.run_nested(**self.run_kwds)

    @property
        return self._io

    @property
        return len(tuple(self.samples.values())[0])

                    use_mpi=False, loglikelihood_function=None):
        """Loads the sampler from the given config file. Many options are
        directly passed to the underlying dynesty sampler, see the official
        dynesty documentation for more details on these.

        The following options are retrieved in the ``[sampler]`` section:
        * ``name = STR`` :
            Required. This must match the sampler's name.
        * ``maxiter = INT``:
            The maximum number of iterations to run.
        * ``dlogz = FLOAT``:
            The target dlogz stopping condition.
        * ``logl_max = FLOAT``:
            The maximum logl stopping condition.
        * ``n_effective = INT``:
            Target effective number of samples stopping condition
        * ``sample = STR``:
            The method to sample the space. Should be one of 'uniform',
            'rwalk', 'rwalk2' (a modified version of rwalk), or 'slice'.
        * ``walk = INT``:
            Used for some of the walk methods. Sets the minimum number of
            steps to take when evolving a point.
        * ``maxmcmc = INT``:
            Used for some of the walk methods. Sets the maximum number of steps
            to take when evolving a point.
        * ``nact = INT``:
            used for some of the walk methods. Sets number of autorcorrelation
            lengths before terminating evolution of a point.
        * ``first_update_min_ncall = INT``:
            The minimum number of calls before updating the bounding region
            for the first time.
        * ``first_update_min_neff = FLOAT``:
            Don't update the the bounding region untill the efficiency drops
            below this value.
        * ``bound = STR``:
            The method of bounding of the prior volume.
            Should be one of 'single', 'balls', 'cubes', 'multi' or 'none'.
        * ``update_interval = INT``:
            Number of iterations between updating the bounding regions
        * ``enlarge = FLOAT``:
            Factor to enlarge the bonding region.
        * ``bootstrap = INT``:
            The number of bootstrap iterations to determine the enlargement
            factor.
        * ``maxcall = INT``:
            The maximum number of calls before checking if we should checkpoint
        * ``checkpoint_time_interval`` :
            Sets the time in seconds between checkpointing.
        * ``loglikelihood-function`` :
            The attribute of the model to use for the loglikelihood. If
            not provided, will default to ``loglikelihood``.

        model : pycbc.inference.model.BaseModel instance
            The model to use.
        output_file : str, optional
            The name of the output file to checkpoint and write results to.
        nprocesses : int, optional
            The number of parallel processes to use. Default is 1.
        use_mpi : bool, optional
            Use MPI for parallelization. Default is False.

        Returns
        -------
        DynestySampler :
            The sampler instance.
        """
        section = "sampler"
        # check name
        assert cp.get(section, "name") == cls.name, (
            "name in section [sampler] must match mine")
        # get the number of live points to use
        nlive = int(cp.get(section, "nlive"))
        loglikelihood_function = \
            get_optional_arg_from_config(cp, section, 'loglikelihood-function')

        no_save_state = cp.has_option(section, 'no-save-state')

        # optional run_nested arguments for dynesty
        rargs = {'maxiter': int,
                 'dlogz': float,
                 'logl_max': float,
                 'n_effective': int,
                 }

        # optional arguments for dynesty
        cargs = {'bound': str,
                 'maxcall': int,
                 'bootstrap': int,
                 'enlarge': float,
                 'update_interval': float,
                 'sample': str,
                 'checkpoint_time_interval': float,
                 'first_update_min_ncall': int,
                 'first_update_min_eff': float,
                 'walks': int,
                 'maxmcmc': int,
                 'nact': int,
                 }
        extra = {}
        run_extra = {}
        for karg in cargs:
            if cp.has_option(section, karg):
                extra[karg] = cargs[karg](cp.get(section, karg))

        #This arg needs to be a dict
        first_update = {}
        if 'first_update_min_ncall' in extra:
            first_update['min_ncall'] = extra.pop('first_update_min_ncall')
            logging.info('First update: min_ncall:%s',
                         first_update['min_ncall'])
        if 'first_update_min_eff' in extra:
            first_update['min_eff'] = extra.pop('first_update_min_eff')
            logging.info('First update: min_eff:%s', first_update['min_eff'])
        extra['first_update'] = first_update

        for karg in rargs:
                run_extra[karg] = rargs[karg](cp.get(section, karg))

        obj = cls(model, nlive=nlive, nprocesses=nprocesses,
                  loglikelihood_function=loglikelihood_function,
                  no_save_state=no_save_state,
                  use_mpi=use_mpi, run_kwds=run_extra, **extra)
        setup_output(obj, output_file, check_nsamples=False)

        if not obj.new_checkpoint:
            obj.resume_from_checkpoint()
        return obj

        """Checkpoint function for dynesty sampler
        """
        # Dynesty has its own __getstate__ which deletes
        # random state information and the pool
        saved = {}
        for key in self.no_pickle:
            if hasattr(self._sampler, key):
                saved[key] = getattr(self._sampler, key)
                setattr(self._sampler, key, None)
                # Write random state
                fp.write_random_state()

                # Write pickled data
                fp.write_pickled_data_into_checkpoint_file(self._sampler)

        # Restore properties that couldn't be pickled if we are continuing
        for key in saved:
            setattr(self._sampler, key, saved[key])

        try:
            with loadfile(self.checkpoint_file, 'r') as fp:
                sampler = fp.read_pickled_data_from_checkpoint_file()

                for key in sampler.__dict__:
                    if key not in self.no_pickle:
                        value = getattr(sampler, key)
                        setattr(self._sampler, key, value)

            self.set_state_from_file(self.checkpoint_file)
            logging.info("Found valid checkpoint file: %s",
                         self.checkpoint_file)
            print(e)
            logging.info("Failed to load checkpoint file")

        """Sets the state of the sampler back to the instance saved in a file.
        """
        with self.io(filename, 'r') as fp:
            numpy.random.set_state(fp.read_random_state())

        self._sampler.rstate = numpy.random
        #if self.nlive < 0:
        #    self._sampler.sampler.rstate = numpy.random

        """Finalze and write it to the results file
        """
        logz = self._sampler.results.logz[-1:][0]
        dlogz = self._sampler.results.logzerr[-1:][0]
        logging.info("log Z, dlog Z: {}, {}".format(logz, dlogz))

        if self.no_save_state:
            self.write_results(self.checkpoint_file)
        else:
            logging.info("Validating checkpoint and backup files")
            checkpoint_valid = validate_checkpoint_files(
                self.checkpoint_file, self.backup_file, check_nsamples=False)
            if not checkpoint_valid:
                raise IOError("error writing to checkpoint file")

    @property
        """Returns raw nested samples
        """
        results = self._sampler.results
        samples = results.samples
        nest_samp = {}
            nest_samp[param] = samples[:, i]
        nest_samp['logwt'] = results.logwt
        nest_samp['loglikelihood'] = results.logl
        return nest_samp

    def set_initial_conditions(self, initial_distribution=None,
                               samples_file=None):
        """Sets up the starting point for the sampler.

        Should also set the sampler's random state.
        """
        pass

        """Writes samples, model stats, acceptance fraction, and random state
        to the given file.

            The file to write to. The file is opened using the ``io`` class
            in an an append state.
        """
        with self.io(filename, 'a') as fp:
            # Write nested samples
            fp.write_raw_samples(self.samples)

            # Write logz and dlogz
            logz = self._sampler.results.logz[-1:][0]
            dlogz = self._sampler.results.logzerr[-1:][0]
            fp.write_logevidence(logz, dlogz)

    @property
        pass

    @property
    def logz(self):
        """
        return bayesian evidence estimated by
        dynesty sampler
        """
        return self._sampler.results.logz[-1:][0]

    @property
    def logz_err(self):
        """
        return error in bayesian evidence estimated by
        """
        return self._sampler.results.logzerr[-1:][0]

def sample_rwalk_mod(args):
    """ Modified version of dynesty.sampling.sample_rwalk

        Adapted from version used in bilby/dynesty
    """

    # Unzipping.
    (u, loglstar, axes, scale,
     prior_transform, loglikelihood, kwargs) = args
    rstate = numpy.random

    # Bounds
    nonbounded = kwargs.get('nonbounded', None)
    periodic = kwargs.get('periodic', None)
    reflective = kwargs.get('reflective', None)

    # Setup.
    n = len(u)
    walks = kwargs.get('walks', 10 * n)  # minimum number of steps
    maxmcmc = kwargs.get('maxmcmc', 2000)  # Maximum number of steps
    nact = kwargs.get('nact', 5)  # Number of ACT
    old_act = kwargs.get('old_act', walks)

    # Initialize internal variables
    accept = 0
    reject = 0
    nfail = 0
    act = numpy.inf
    u_list = []
    v_list = []
    logl_list = []

    ii = 0
    while ii < nact * act:
        ii += 1

        # Propose a direction on the unit n-sphere.
        drhat = rstate.randn(n)
        drhat /= numpy.linalg.norm(drhat)

        # Scale based on dimensionality.
        dr = drhat * rstate.rand() ** (1.0 / n)

        # Transform to proposal distribution.
        du = numpy.dot(axes, dr)
        u_prop = u + scale * du

        # Wrap periodic parameters
        if periodic is not None:
            u_prop[periodic] = numpy.mod(u_prop[periodic], 1)
        # Reflect
        if reflective is not None:
            u_prop[reflective] = reflect(u_prop[reflective])

        # Check unit cube constraints.
        if unitcheck(u_prop, nonbounded):
            pass
        else:
            nfail += 1
            # Only start appending to the chain once a single jump is made
            if accept > 0:
                u_list.append(u_list[-1])
                v_list.append(v_list[-1])
                logl_list.append(logl_list[-1])
            continue

        # Check proposed point.
        v_prop = prior_transform(numpy.array(u_prop))
        logl_prop = loglikelihood(numpy.array(v_prop))
        if logl_prop > loglstar:
            u = u_prop
            v = v_prop
            logl = logl_prop
            accept += 1
            u_list.append(u)
            v_list.append(v)
            logl_list.append(logl)
        else:
            reject += 1

        # If we've taken the minimum number of steps, calculate the ACT
        if accept + reject > walks:
            act = estimate_nmcmc(
                accept_ratio=accept / (accept + reject + nfail),
                old_act=old_act, maxmcmc=maxmcmc)

        # If we've taken too many likelihood evaluations then break
        if accept + reject > maxmcmc:
            logging.warning(
                "Hit maximum number of walks {} with accept={}, reject={}, "
                "and nfail={} try increasing maxmcmc"
                .format(maxmcmc, accept, reject, nfail))
            break

    # If the act is finite, pick randomly from within the chain
    if numpy.isfinite(act) and int(.5 * nact * act) < len(u_list):
        idx = numpy.random.randint(int(.5 * nact * act), len(u_list))
        u = u_list[idx]
        v = v_list[idx]
        logl = logl_list[idx]
    else:
        logging.debug("Unable to find a new point using walk: "
                      "returning a random point")
        u = numpy.random.uniform(size=n)
        v = prior_transform(u)
        logl = loglikelihood(v)

    blob = {'accept': accept, 'reject': reject, 'fail': nfail, 'scale': scale}
    kwargs["old_act"] = act

    ncall = accept + reject
    return u, v, logl, ncall, blob

def estimate_nmcmc(accept_ratio, old_act, maxmcmc, safety=5, tau=None):
    """ Estimate autocorrelation length of chain using acceptance fraction

    Using ACL = (2/acc) - 1 multiplied by a safety margin. Code adapated from
    CPNest:
        - https://github.com/johnveitch/cpnest/blob/master/cpnest/sampler.py
        - http://github.com/farr/Ensemble.jl

    accept_ratio: float [0, 1]
        Ratio of the number of accepted points to the total number of points
    old_act: int
        The ACT of the last iteration
    maxmcmc: int
        The maximum length of the MCMC chain to use
    safety: int
        A safety factor applied in the calculation
    tau: int (optional)
        The ACT, if given, otherwise estimated.

    """
    if tau is None:
        tau = maxmcmc / safety

    if accept_ratio == 0.0:
        Nmcmc_exact = (1 + 1 / tau) * old_act
    else:
        Nmcmc_exact = (
            (1. - 1. / tau) * old_act +
            (safety / tau) * (2. / accept_ratio - 1.)
        )
        Nmcmc_exact = float(min(Nmcmc_exact, maxmcmc))
    return max(safety, int(Nmcmc_exact))

#
#

#
#
#                                   Preamble
#
#
"""
This modules provides classes and functions for using the emcee sampler
"""

import emcee

from .base_mcmc import (BaseMCMC, EnsembleSupport,
                        ensemble_compute_acf, ensemble_compute_acl,
                        raw_samples_to_dict,
                        blob_data_to_dict, get_optional_arg_from_config)
from ..burn_in import EnsembleMCMCBurnInTests
from pycbc.inference.io import EmceeFile

#
#
#                                   Samplers
#
#

if emcee.__version__ >= '3.0.0':

class EmceeEnsembleSampler(EnsembleSupport, BaseMCMC, BaseSampler):
    """This class is used to construct an MCMC sampler from the emcee
    package's EnsembleSampler.

        Number of walkers to use in sampler.
    """
    name = "emcee"
    _io = EmceeFile
    burn_in_class = EnsembleMCMCBurnInTests

    def __init__(self, model, nwalkers,
                 checkpoint_interval=None, checkpoint_signal=None,
                 logpost_function=None, nprocesses=1, use_mpi=False):

        # create a wrapper for calling the model
        if logpost_function is None:
            logpost_function = 'logposterior'
        model_call = models.CallModel(model, logpost_function)

        # these are used to help paralleize over multiple cores / MPI
        models._global_instance = model_call
        model_call = models._call_global_model
        pool = choose_pool(mpi=use_mpi, processes=nprocesses)

        # set up emcee
        self.nwalkers = nwalkers
        ndim = len(model.variable_params)
        self._sampler = emcee.EnsembleSampler(nwalkers, ndim, model_call,
                                              pool=pool)
        # emcee uses it's own internal random number generator; we'll set it
        # to have the same state as the numpy generator
        self._sampler.random_state = rstate
        self._checkpoint_interval = checkpoint_interval
        self._checkpoint_signal = checkpoint_signal

    @property

    @property
        return (self.nwalkers,)

    @property
        """A dict mapping ``variable_params`` to arrays of samples currently
        in memory.

        The arrays have shape ``nwalkers x niterations``.
        """
        # emcee stores samples to it's chain attribute as a
        # nwalker x niterations x ndim array
        raw_samples = self._sampler.chain
        return raw_samples_to_dict(self, raw_samples)

    @property
        """A dict mapping the model's ``default_stats`` to arrays of values.

        The returned array has shape ``nwalkers x niterations``.
        """
        stats = self.model.default_stats
        return blob_data_to_dict(stats, self._sampler.blobs)

        """Clears the samples and stats from memory.
        """
        # store the iteration that the clear is occuring on
        self._lastclear = self.niterations
        # now clear the chain
        self._sampler.reset()
        self._sampler.clear_blobs()

        """
            rstate = fp.read_random_state()
        # set the numpy random state
        # set emcee's generator to the same state

        """Advance the ensemble for a number of samples.

        niterations : int
            Number of iterations to run the sampler for.
        """
            pos = self._p0
        res = self._sampler.run_mcmc(pos, niterations)
        p, _, _ = res[0], res[1], res[2]
        # update the positions
        self._pos = p

        """
            # write samples
            fp.write_samples(self.samples,
                             parameters=self.model.variable_params,
                             last_iteration=self.niterations)
            # write stats
            fp.write_samples(self.model_stats,
            # write accpetance
            fp.write_acceptance_fraction(self._sampler.acceptance_fraction)
            # write random state
            fp.write_random_state(state=self._sampler.random_state)

        """All data is written by the last checkpoint in the run method, so
        this just passes."""
        pass

    def compute_acf(filename, **kwargs):
        r"""Computes the autocorrelation function.

        Calls :py:func:`base_mcmc.ensemble_compute_acf`; see that

            Name of a samples file to compute ACFs for.
            :py:func:`base_mcmc.ensemble_compute_acf`.

        Returns
        -------
        dict :
            Dictionary of arrays giving the ACFs for each parameter. If
            ``per-walker`` is True, the arrays will have shape
            ``nwalkers x niterations``.
        """
        return ensemble_compute_acf(filename, **kwargs)

    def compute_acl(filename, **kwargs):
        r"""Computes the autocorrelation length.

        Calls :py:func:`base_mcmc.ensemble_compute_acl`; see that

            Name of a samples file to compute ACLs for.

        Returns
        -------
        dict
            A dictionary giving the ACL for each parameter.
        """
        return ensemble_compute_acl(filename, **kwargs)

        """Loads the sampler from the given config file."""
        # get the number of walkers to use
        nwalkers = int(cp.get(section, "nwalkers"))
        # get the checkpoint interval, if it's specified
        checkpoint_interval = cls.checkpoint_from_config(cp, section)
        checkpoint_signal = cls.ckpt_signal_from_config(cp, section)
        # get the logpost function
        lnpost = get_optional_arg_from_config(cp, section, 'logpost-function')
        obj = cls(model, nwalkers,
                  checkpoint_interval=checkpoint_interval,
                  checkpoint_signal=checkpoint_signal,
                  logpost_function=lnpost, nprocesses=nprocesses,
                  use_mpi=use_mpi)
        # set target
        obj.set_target_from_config(cp, section)
        # add burn-in if it's specified
        obj.set_burn_in_from_config(cp)
        # set prethin options
        obj.set_thin_interval_from_config(cp, section)
        # Set up the output file
        setup_output(obj, output_file)
        else:
            obj.set_start_from_config(cp)
        return obj

#
#

"""
This modules provides classes and functions for using the emcee_pt sampler
"""

from .base_mcmc import (BaseMCMC, EnsembleSupport, raw_samples_to_dict,
                        get_optional_arg_from_config)
from .base_multitemper import (MultiTemperedSupport,
                               ensemble_compute_acf, ensemble_compute_acl)
from ..burn_in import EnsembleMultiTemperedMCMCBurnInTests
from pycbc.inference.io import EmceePTFile

class EmceePTSampler(MultiTemperedSupport, EnsembleSupport, BaseMCMC,
                     BaseSampler):
    """This class is used to construct a parallel-tempered MCMC sampler from
    the emcee package's PTSampler.

        Number of temeratures to use in the sampler.
    betas : array
        An array of inverse temperature values to be used in emcee_pt's
        temperature ladder. If not provided, ``emcee_pt`` will use the number
        of temperatures and the number of dimensions of the parameter space to
        construct the ladder with geometrically spaced temperatures.
    loglikelihood_function : str, optional
        Set the function to call from the model for the ``loglikelihood``.
        Default is ``loglikelihood``.
    nprocesses : int, optional
        The number of parallel processes to use. Default is 1
        (no paralleliztion).
    use_mpi : bool, optional
        Use MPI for parallelization. Default (False) will use python's
        multiprocessing.
    """
    name = "emcee_pt"
    _io = EmceePTFile
    burn_in_class = EnsembleMultiTemperedMCMCBurnInTests

    def __init__(self, model, ntemps, nwalkers, betas=None,
                 loglikelihood_function=None,
                 nprocesses=1, use_mpi=False):

        # frustratingly, emcee_pt does not support blob data, so we have to
        # turn it off
        model_call = models.CallModel(model, loglikelihood_function,
                                      return_all_stats=False)

        prior_call = models._call_global_model_logprior

        # construct the sampler: PTSampler needs the likelihood and prior
        # functions separately
        self._sampler = emcee.PTSampler(ntemps, nwalkers, ndim,
                                        model_call, prior_call, pool=self.pool,
                                        betas=betas)
        self._ntemps = ntemps

    @property

    @property
        return (self.ntemps, self.nwalkers,)

    @property
        return self._sampler.betas

        Calls :py:func:`base_multitemper.ensemble_compute_acf`; see that

            :py:func:`base_multitemper.ensemble_compute_acf`.

        Returns
        -------
        dict :
            ``per-walker=True`` is passed as a keyword argument, the arrays
            will have shape ``ntemps x nwalkers x niterations``. Otherwise, the
            returned array will have shape ``ntemps x niterations``.
        """

        Calls :py:func:`base_multitemper.ensemble_compute_acl`; see that

            :py:func:`base_multitemper.ensemble_compute_acl`.

        Returns
        -------
        dict
            A dictionary of ntemps-long arrays of the ACLs of each parameter.
        """

        """Loads the sampler from the given config file.

        * ``name`` :
            Required. This must match the samlper's name.
        * ``nwalkers`` :
            Required. The number of walkers to use.
        * ``niterations`` :
            The number of iterations to run the sampler for. Either this or
            ``effective-nsamples`` must be provided (but not both).
        * ``effective-nsamples`` :
            Run the sampler until the given number of effective samples are
            obtained. A ``checkpoint-interval`` must also be provided in this
            case. Either this or ``niterations`` must be provided (but not
            both).
        * ``thin-interval`` :
            Thin the samples by the given value before saving to disk. May
            provide this, or ``max-samples-per-chain``, but not both. If
            neither options are provided, will save all samples.
        * ``max-samples-per-chain`` :
            Thin the samples such that the number of samples per chain per
            temperature that are saved to disk never exceeds the given value.
            May provide this, or ``thin-interval``, but not both. If neither
            options are provided, will save all samples.
        * ``checkpoint-interval`` :
            Sets the checkpoint interval to use. Must be provided if using
            ``effective-nsamples``.
        * ``checkpoint-signal`` :
            Set the checkpoint signal, e.g., "USR2". Optional.
        * ``logl-function`` :

        Settings for burn-in tests are read from ``[sampler-burn_in]``. In
        particular, the ``burn-in-test`` option is used to set the burn in
        tests to perform. See
        :py:func:`MultiTemperedMCMCBurnInTests.from_config` for details. If no
        ``burn-in-test`` is provided, no burn in tests will be carried out.

        Returns
        -------
        EmceePTSampler :
        """
        # get the temps/betas
        ntemps, betas = cls.betas_from_config(cp, section)
        # get the loglikelihood function
        logl = get_optional_arg_from_config(cp, section, 'logl-function')
        obj = cls(model, ntemps, nwalkers, betas=betas,
                  loglikelihood_function=logl, nprocesses=nprocesses,
        else:
        return obj

    @property
        in memory.

        The arrays have shape ``ntemps x nwalkers x niterations``.
        """

    @property
        """Returns the log likelihood ratio and log prior as a dict of arrays.

        The returned array has shape ntemps x nwalkers x niterations.

        Unfortunately, because ``emcee_pt`` does not have blob support, this
        will only return the loglikelihood and logprior (with the logjacobian
        set to zero) regardless of what stats the model can return.

        .. warning::
            Since the `logjacobian` is not saved by `emcee_pt`, the `logprior`
            returned here is the log of the prior pdf in the sampling
            coordinate frame rather than the variable params frame. This
            differs from the variable params frame by the log of the Jacobian
            of the transform from one frame to the other. If no sampling
            transforms were used, then the `logprior` is the same.
        """
        # likelihood has shape ntemps x nwalkers x niterations
        logl = self._sampler.lnlikelihood
        # get prior from posterior
        logp = self._sampler.lnprobability - logl
        logjacobian = numpy.zeros(logp.shape)
        return {'loglikelihood': logl, 'logprior': logp,
                'logjacobian': logjacobian}

        """Clears the chain and blobs from memory.
        """

        """

            Number of samples to get from sampler.
        """

        """
            fp.write_samples(self.model_stats, last_iteration=self.niterations)
            fp.write_random_state()

    def calculate_logevidence(cls, filename, thin_start=None, thin_end=None,
                              thin_interval=None):
        """Calculates the log evidence from the given file using ``emcee_pt``'s
        thermodynamic integration.

            Name of the file to read the samples from. Should be an
            ``EmceePTFile``.
        thin_start : int
            Index of the sample to begin returning stats. Default is to read
            stats after burn in. To start from the beginning set thin_start
            to 0.
            Interval to accept every i-th sample. Default is to use the
            `fp.acl`. If `fp.acl` is not set, then use all stats
            (set thin_interval to 1).
        thin_end : int
            Index of the last sample to read. If not given then
            `fp.niterations` is used.

        Returns
        -------
        lnZ : float
            The estimate of log of the evidence.
        dlnZ : float
            The error on the estimate.
        """
        with cls._io(filename, 'r') as fp:
            logls = fp.read_raw_samples(['loglikelihood'],
                                        thin_start=thin_start,
                                        thin_end=thin_end,
                                        temps='all', flatten=False)
            logls = logls['loglikelihood']
            # we need the betas that were used
            betas = fp.betas
            # annoyingly, theromdynaimc integration in PTSampler is an instance
            # method, so we'll implement a dummy one
            ntemps = fp.ntemps
            nwalkers = fp.nwalkers
            ndim = len(fp.variable_params)
        dummy_sampler = emcee.PTSampler(ntemps, nwalkers, ndim, None,
                                        None, betas=betas)
        return dummy_sampler.thermodynamic_integration_log_evidence(
            logls=logls, fburnin=0.)

    def _correctjacobian(self, samples):
        """Corrects the log jacobian values stored on disk.

            Dictionary of the samples.
        """
        # flatten samples for evaluating
        orig_shape = list(samples.values())[0].shape
        flattened_samples = {p: arr.ravel()
                             for p, arr in list(samples.items())}
        # convert to a list of tuples so we can use map function
        params = list(flattened_samples.keys())
        size = flattened_samples[params[0]].size
        logj = numpy.zeros(size)
        for ii in range(size):
            these_samples = {p: flattened_samples[p][ii] for p in params}
            these_samples = self.model.sampling_transforms.apply(these_samples)
            self.model.update(**these_samples)
            logj[ii] = self.model.logjacobian
        return logj.reshape(orig_shape)

        """Calculates the log evidence and writes to the checkpoint file.

        If sampling transforms were used, this also corrects the jacobian
        stored on disk.

        The thin start/interval/end for calculating the log evidence are
        retrieved from the checkpoint file's thinning attributes.
        """
        if self.model.sampling_transforms is not None:
            # fix the lobjacobian values stored on disk
            logging.info("Correcting logjacobian values on disk")
            with self.io(self.checkpoint_file, 'r') as fp:
                samples = fp.read_raw_samples(self.variable_params,
                                              thin_start=0,
                                              thin_interval=1, thin_end=None,
                                              temps='all', flatten=False)
            logjacobian = self._correctjacobian(samples)
            # write them back out
                    fp[fp.samples_group]['logjacobian'][()] = logjacobian
        logging.info("Calculating log evidence")
        # get the thinning settings
        with self.io(self.checkpoint_file, 'r') as fp:
        # calculate
        logz, dlogz = self.calculate_logevidence(
            self.checkpoint_file, thin_start=thin_start, thin_end=thin_end,
            thin_interval=thin_interval)
        # write to both the checkpoint and backup
                fp.write_logevidence(logz, dlogz)

#
#

"""This module provides classes for interacting with epsie samplers.
"""

import epsie
from epsie.samplers import ParallelTemperedSampler

# we'll use emcee_pt's default beta ladder for temperature levels
from emcee.ptsampler import default_beta_ladder

from .base_mcmc import (BaseMCMC, get_optional_arg_from_config,
                        nsamples_in_chain)
from .base_multitemper import (MultiTemperedSupport, compute_acf, compute_acl,
                               acl_from_raw_acls)
from ..burn_in import MultiTemperedMCMCBurnInTests
from ..jump import epsie_proposals_from_config
from ..io import EpsieFile

class EpsieSampler(MultiTemperedSupport, BaseMCMC, BaseSampler):
    """Constructs an MCMC sampler using epsie's parallel-tempered sampler.

    nchains : int
        Number of chains to use in the sampler.
    ntemps : int, optional
        Number of temperatures to use in the sampler. A geometrically-spaced
        temperature ladder with the gievn number of levels will be constructed
        based on the number of parameters. If not provided, must provide
        ``betas``.
    betas : array, optional
        An array of inverse temperature values to be used in for the
        temperature ladder. If not provided, must provide ``ntemps``.
    proposals : list, optional
        List of proposals to use. Any parameters that do not have a proposal
        provided will use the ``default_propsal``. **Note:** proposals should
        be specified for the sampling parameters, not the
        variable parameters.
    default_proposal : an epsie.Proposal class, optional
        The default proposal to use for parameters not in ``proposals``.
        Default is :py:class:`epsie.proposals.Normal`.
    default_proposal_args : dict, optional
        Dictionary of arguments to pass to the default proposal.
    swap_interval : int, optional
        The number of iterations between temperature swaps. Default is 1.
        Seed for epsie's random number generator. If None provided, will create
        one.
    checkpoint_interval : int, optional
        Specify the number of iterations to do between checkpoints. If not
        provided, no checkpointin will be done.
    checkpoint_signal : str, optional
        Set the signal to use when checkpointing. For example, 'USR2'.
    """
    name = "epsie"
    _io = EpsieFile
    burn_in_class = MultiTemperedMCMCBurnInTests

    def __init__(self, model, nchains, ntemps=None, betas=None,
                 proposals=None, default_proposal=None,
                 default_proposal_args=None, seed=None,
                 swap_interval=1,

        # create the betas if not provided
        if betas is None:
            betas = default_beta_ladder(len(model.variable_params),
                                        ntemps=ntemps)
        model_call = _EpsieCallModel(model, loglikelihood_function)

        # initialize the sampler
        self._sampler = ParallelTemperedSampler(
            model.sampling_params, model_call, nchains, betas=betas,
            swap_interval=swap_interval,
            proposals=proposals, default_proposal=default_proposal,
            default_proposal_args=default_proposal_args,
            seed=seed, pool=pool)
        # set other parameters

    @property

    @property
        return (self.ntemps, self.nchains,)

    @property
        """The inverse temperatures being used."""

    @property
    def seed(self):
        """The seed used for epsie's random bit generator.

        This is not the same as the seed used for the prior distributions.
        """
        return self._sampler.seed

    @property
        """Number of iterations between temperature swaps."""
        return self._sampler.swap_interval

        Calls :py:func:`base_multitemper.compute_acf`; see that

            :py:func:`base_multitemper.compute_acf`.

        Returns
        -------
        dict :
            Dictionary of arrays giving the ACFs for each parameter. The arrays
            will have shape ``ntemps x nchains x niterations``.
        """
        return compute_acf(filename, **kwargs)

        Calls :py:func:`base_multitemper.compute_acl`; see that

            :py:func:`base_multitemper.compute_acl`.

        Returns
        -------
        dict
        """
        return compute_acl(filename, **kwargs)

    @property
    def acl(self):  # pylint: disable=invalid-overridden-method
        """The autocorrelation lengths of the chains.
        """
        return acl_from_raw_acls(self.raw_acls)

    @property
    def effective_nsamples(self):  # pylint: disable=invalid-overridden-method
        """
        else:
        nperchain = nsamples_in_chain(start_iter, act, self.niterations)
            # ensure that any chain not burned in has zero samples
            nperchain[~self.burn_in.is_burned_in] = 0
            # and that any chain that is burned in has at least one sample
            nperchain[self.burn_in.is_burned_in & (nperchain < 1)] = 1
        return int(nperchain.sum())

    @property
        in memory.

        The arrays have shape ``ntemps x nchains x niterations``.

        The dictionary also contains sampling parameters.
        """
        samples = epsie.array2dict(self._sampler.positions)
        samples = self.model.prior_distribution.apply_boundary_conditions(
            **samples)
        # apply transforms to go to model's variable params space
            samples = self.model.sampling_transforms.apply(
                samples, inverse=True)

    @property

        """
        return epsie.array2dict(self._sampler.blobs)

        """
        # now clear the sampler
        self._sampler.clear()

        """
            # get the numpy state
            numpy_rstate_group = '/'.join([fp.sampler_group,
                                           'numpy_random_state'])
            rstate = fp.read_random_state(group=numpy_rstate_group)
            # set the sampler state for epsie
            self._sampler.set_state_from_checkpoint(fp, path=fp.sampler_group)
        # set the global numpy random state for pycbc

        p0 = super(EpsieSampler, self).set_p0(samples_file=samples_file,
                                              prior=prior)
        self._sampler.start_position = p0

    @property
        """A dictionary of the current chain positions."""
        # we override BaseMCMC's pos property because this can be directly
        # retrieved from epsie
        return self._sampler.current_positions

        """Advance the chains for a number of iterations.

        """
        self._sampler.run(niterations)

        """Writes samples, model stats, acceptance ratios, and random state

        """
            # write accpetance ratio
            acceptance = self._sampler.acceptance
            fp.write_acceptance_ratio(acceptance['acceptance_ratio'],
                                      last_iteration=self.niterations)
            # write temperature data
            if self.ntemps > 1:
                temp_ar = self._sampler.temperature_acceptance
                temp_swaps = self._sampler.temperature_swaps
                fp.write_temperature_data(temp_swaps, temp_ar,
                                          self.swap_interval,
                                          last_iteration=self.niterations)
            # write numpy's global state (for the distributions)
            fp.write_random_state(group=numpy_rstate_group)
            # write the sampler's state
            self._sampler.checkpoint(fp, path=fp.sampler_group)

        pass

            (required) must match the samlper's name
        * ``nchains`` :
            (required) the number of chains to use
            both).
        * ``seed`` :
            The seed to use for epsie's random number generator. If not
            provided, epsie will create one.
        * ``swap-interval`` :
            The number of iterations between temperature swaps. Default is 1.

        Jump proposals must be provided for every sampling
        parameter. These are retrieved from subsections
        ``[jump_proposal-{params}]``, where params is a
        :py:const:`pycbc.VARARGS_DELIM` separated list of parameters the
        proposal should be used for. See
        :py:func:`inference.jump.epsie_proposals_from_config` for
        details.

        .. note::
            Jump proposals should be specified for **sampling parameters**,
            not **variable parameters**.

        Returns
        -------
        EpsiePTSampler :
        """
        nchains = int(cp.get(section, "nchains"))
        seed = get_optional_arg_from_config(cp, section, 'seed', dtype=int)
        # get the swap interval
        swap_interval = get_optional_arg_from_config(cp, section,
                                                     'swap-interval',
                                                     dtype=int)
        if swap_interval is None:
            swap_interval = 1
        # get the proposals
        proposals = epsie_proposals_from_config(cp)
        # check that all of the sampling parameters have a specified
        # proposal
        sampling_params = set(model.sampling_params)
        proposal_params = set(param for prop in proposals
                              for param in prop.parameters)
        missing = sampling_params - proposal_params
        if missing:
            raise ValueError("Missing jump proposals for sampling parameters "
                             "{}".format(', '.join(missing)))
        # initialize
        obj = cls(model, nchains,
                  ntemps=ntemps, betas=betas, proposals=proposals,
                  swap_interval=swap_interval, seed=seed,
                  loglikelihood_function=logl,
                  nprocesses=nprocesses, use_mpi=use_mpi)
        if obj.new_checkpoint:
        else:
        return obj

class _EpsieCallModel(object):
    """Model wrapper for epsie.

    Allows model to be called like a function. Returns the loglikelihood
    function, logprior, and the model's default stats.
    """

    def __init__(self, model, loglikelihood_function=None):

    def __call__(self, **kwargs):
        """Calls update, then calls the loglikelihood and logprior."""
        self.model.update(**kwargs)
        logp = self.model.logprior
            # don't try to call the log likelihood if the prior rules it out
            logl = numpy.nan
        else:
            logl = getattr(self.model, self.loglikelihood_function)
        return logl, logp, self.model.current_stats

# Copyright (C) 2018  Daniel Finstad
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides classes and functions for using the Multinest sampler
"""

import sys

from pycbc.inference.io import (MultinestFile, validate_checkpoint_files)
from pycbc.pool import is_main_process
from pycbc.transforms import apply_transforms

#
#
#                                   Samplers
#
#

class MultinestSampler(BaseSampler):
    """This class is used to construct a nested sampler from
    the Multinest package.

    nlivepoints : int
    """
    name = "multinest"
    _io = MultinestFile

    def __init__(self, model, nlivepoints, checkpoint_interval=1000,
                 importance_nested_sampling=False,
                 evidence_tolerance=0.1, sampling_efficiency=0.01,
                 constraints=None):
        try:
            loglevel = logging.getLogger().getEffectiveLevel()
            logging.getLogger().setLevel(logging.WARNING)
            from pymultinest import Analyzer, run
            self.run_multinest = run
            self.analyzer = Analyzer
            logging.getLogger().setLevel(loglevel)
            raise ImportError("pymultinest is not installed.")

        super(MultinestSampler, self).__init__(model)

        self._constraints = constraints
        self._nlivepoints = nlivepoints
        self._ndim = len(model.variable_params)
        self._random_state = numpy.random.get_state()
        self._ztol = evidence_tolerance
        self._eff = sampling_efficiency
        self._ins = importance_nested_sampling
        self._samples = None
        self._itercount = None
        self._logz = None
        self._dlogz = None
        self._importance_logz = None
        self._importance_dlogz = None
        self.is_main_process = is_main_process()

    @property

    @property
        """Get the current number of iterations.
        """
        itercount = self._itercount
        if itercount is None:
            itercount = 0
        return itercount

    @property
        """Get the number of iterations between checkpoints.
        """

    @property
    def nlivepoints(self):
        """Get the number of live points used in sampling.
        """
        return self._nlivepoints

    @property
        """Get the current estimate of the log evidence.
        """
        return self._logz

    @property
    def dlogz(self):
        """Get the current error estimate of the log evidence.
        """
        return self._dlogz

    @property
    def importance_logz(self):
        """Get the current importance weighted estimate of the log
        evidence.
        """
        return self._importance_logz

    @property
    def importance_dlogz(self):
        """Get the current error estimate of the importance
        weighted log evidence.
        """
        return self._importance_dlogz

    @property
        in memory.
        """
        samples_dict = {p: self._samples[:, i] for i, p in
                        enumerate(self.model.variable_params)}
        return samples_dict

    @property
        """
        stats = []
        for sample in self._samples:
            params = dict(zip(self.model.variable_params, sample))
                params = self.model.sampling_transforms.apply(params)
            self.model.update(**params)
            self.model.logposterior
            stats.append(self.model.get_current_stats())
        stats = numpy.array(stats)
        return {s: stats[:, i] for i, s in enumerate(self.model.default_stats)}

    def get_posterior_samples(self):
        """Read posterior samples from ASCII output file created by
        multinest.
        """
        post_file = self.backup_file[:-9]+'-post_equal_weights.dat'
        return numpy.loadtxt(post_file, ndmin=2)

    def check_if_finished(self):
        """Estimate remaining evidence to see if desired evidence-tolerance
        stopping criterion has been reached.
        """
        resume_file = self.backup_file[:-9] + '-resume.dat'
        current_vol, _, _ = numpy.loadtxt(
            resume_file, skiprows=6, unpack=True)
        maxloglike = max(self.get_posterior_samples()[:, -1])
        logz_remain = numpy.exp(maxloglike +
                                numpy.log(current_vol) - self.logz)
        logging.info("Estimate of remaining logZ is %s", logz_remain)
        done = logz_remain < self._ztol
        return done

        """Sets the initial starting point for the sampler.

        If a starting samples file is provided, will also load the random
        state from it.
        """
        # use samples file to set the state of the sampler
            self.set_state_from_file(samples_file)

        """Resume sampler from checkpoint
        """
        pass

        """
        with self.io(filename, 'r') as f_p:
            rstate = f_p.read_random_state()
        # set sampler's generator to the same state
        self._random_state = rstate

    def loglikelihood(self, cube, *extra_args):
        """Log likelihood evaluator that gets passed to multinest.
        """
        params = {p: v for p, v in zip(self.model.variable_params, cube)}
        # apply transforms
            params = self.model.sampling_transforms.apply(params)
        if self.model.waveform_transforms is not None:
            params = apply_transforms(params, self.model.waveform_transforms)
        # apply constraints
        if (self._constraints is not None and
                not all([c(params) for c in self._constraints])):
        return self.model.loglikelihood

    def transform_prior(self, cube, *extra_args):
        """Transforms the unit hypercube that multinest makes its draws
        from, into the prior space defined in the config file.
        """

        """Runs the sampler until the specified evidence tolerance
        is reached.
        """
            self._itercount = 0
        else:
            self.set_initial_conditions(samples_file=self.checkpoint_file)
            with self.io(self.checkpoint_file, "r") as f_p:
                self._itercount = f_p.niterations
        outputfiles_basename = self.backup_file[:-9] + '-'
        analyzer = self.analyzer(self._ndim,
                                 outputfiles_basename=outputfiles_basename)
        done = False
        while not done:
            logging.info("Running sampler for %s to %s iterations",
                         self.niterations, self.niterations + iterinterval)
            # run multinest
            self.run_multinest(self.loglikelihood, self.transform_prior,
                               self._ndim, n_live_points=self.nlivepoints,
                               evidence_tolerance=self._ztol,
                               sampling_efficiency=self._eff,
                               importance_nested_sampling=self._ins,
                               max_iter=iterinterval,
                               n_iter_before_update=iterinterval,
                               seed=numpy.random.randint(0, 1e6),
                               outputfiles_basename=outputfiles_basename,
                               multimodal=False, verbose=True)
            # parse results from multinest output files
            nest_stats = analyzer.get_mode_stats()
            self._logz = nest_stats["nested sampling global log-evidence"]
            self._dlogz = nest_stats[
                "nested sampling global log-evidence error"]
            if self._ins:
                self._importance_logz = nest_stats[
                    "nested importance sampling global log-evidence"]
                self._importance_dlogz = nest_stats[
                    "nested importance sampling global log-evidence error"]
            self._samples = self.get_posterior_samples()[:, :-1]
            logging.info("Have %s posterior samples", self._samples.shape[0])
            self._itercount += iterinterval
            # make sure there's at least 1 posterior sample
            if self._samples.shape[0] == 0:
                continue
            if self.is_main_process:
                self.checkpoint()
            # check if we're finished
            done = self.check_if_finished()
        if not self.is_main_process:
            sys.exit()

        """
        with self.io(filename, 'a') as f_p:
            f_p.write_samples(self.samples, self.model.variable_params)
            f_p.write_samples(self.model_stats)
            # write evidence
            f_p.write_logevidence(self.logz, self.dlogz,
                                  self.importance_logz,
                                  self.importance_dlogz)
            # write random state (use default numpy.random_state)
            f_p.write_random_state()

        logging.info("Writing samples to files")
        for f_n in [self.checkpoint_file, self.backup_file]:
            self.write_results(f_n)
            with self.io(f_n, "a") as f_p:
                f_p.write_niterations(self.niterations)
            self.checkpoint_file, self.backup_file, check_nsamples=False)

    def setup_output(self, output_file):
        """Sets up the sampler's checkpoint and output files.

        The checkpoint file has the same name as the output file, but with
        ``.checkpoint`` appended to the name. A backup file will also be
        created.

        sampler : sampler instance
            Sampler
            Name of the output file.
        """
        if self.is_main_process:
            setup_output(self, output_file)
        else:
            # child processes just store filenames
            checkpoint_file = output_file + '.checkpoint'
            backup_file = output_file + '.bkup'
            self.checkpoint_file = checkpoint_file
            self.backup_file = backup_file
            self.checkpoint_valid = True
            self.new_checkpoint = True

        pass

        nlivepoints = int(cp.get(section, "nlivepoints"))
        checkpoint = get_optional_arg_from_config(
            cp, section, 'checkpoint-interval', dtype=int)
        # get the evidence tolerance, if specified
        ztol = get_optional_arg_from_config(cp, section, 'evidence-tolerance',
                                            dtype=float)
        # get the sampling efficiency, if specified
        eff = get_optional_arg_from_config(cp, section, 'sampling-efficiency',
                                           dtype=float)
        # get importance nested sampling setting, if specified
        ins = get_optional_arg_from_config(cp, section,
                                           'importance-nested-sampling',
                                           dtype=bool)
        # get constraints since we can't use the joint prior distribution
        constraints = read_constraints_from_config(cp)
        # build optional kwarg dict
        kwarg_names = ['evidence_tolerance', 'sampling_efficiency',
                       'importance_nested_sampling',
                       'checkpoint_interval']
        optional_kwargs = {k: v for k, v in
                           zip(kwarg_names, [ztol, eff, ins, checkpoint]) if
                           v is not None}
        obj = cls(model, nlivepoints, constraints=constraints,
                  **optional_kwargs)
        obj.setup_output(output_file)
        return obj

#
#

"""
"""

import ptemcee

from .base_multitemper import (read_betas_from_hdf,
from pycbc.inference.io import PTEmceeFile

class PTEmceeSampler(EnsembleSupport, BaseMCMC, BaseSampler):
    """This class is used to construct the parallel-tempered ptemcee sampler.

        Specify the number of temps to use. Either this, ``Tmax``, or ``betas``
        must be specified.
    Tmax : float, optional
        Specify the maximum temperature to use. This may be used with
        ``ntemps``; see :py:func:`ptemcee.make_ladder` for details. Either
        this, ``ntemps``, or ``betas`` must be specified.
    betas : list of float, optional
        Specify the betas to use. Must be provided if ``ntemps`` and ``Tmax``
        are not given. Will override ``ntemps`` and ``Tmax`` if provided.
    adaptive : bool, optional
        Whether or not to use adaptive temperature levels. Default is False.
    adaptation_lag : int, optional
        Only used if ``adaptive`` is True; see :py:mod:`ptemcee.Sampler` for
        details. If not provided, will use ``ptemcee``'s default.
    adaptation_time : int, optional
    scale_factor : float, optional
        Scale factor used for the stretch proposal; see
        :py:mod:`ptemcee.Sampler` for details. If not provided, will use
        ``ptemcee``'s default.
    """
    name = "ptemcee"
    _io = PTEmceeFile

    def __init__(self, model, nwalkers, ntemps=None, Tmax=None, betas=None,
                 adaptive=False, adaptation_lag=None, adaptation_time=None,
                 scale_factor=None,

        # create temperature ladder if needed
        if ntemps is None and Tmax is None and betas is None:
            raise ValueError("must provide either ntemps/Tmax or betas")
            betas = ptemcee.make_ladder(ndim, ntemps=ntemps, Tmax=Tmax)
        # construct the keyword arguments to pass; if a kwarg is None, we
        # won't pass it, resulting in ptemcee's defaults being used
        kwargs = {}
        kwargs['adaptive'] = adaptive
        kwargs['betas'] = betas
        if adaptation_lag is not None:
            kwargs['adaptation_lag'] = adaptation_lag
        if adaptation_time is not None:
            kwargs['adaptation_time'] = adaptation_time
        if scale_factor is not None:
            kwargs['scale_factor'] = scale_factor
        # frustratingly, ptemcee does not support blob data, so we have to
        # construct the sampler
        self._sampler = ptemcee.Sampler(nwalkers=nwalkers, ndim=ndim,
                                        logl=model_call, logp=prior_call,
                                        mapper=self.pool.map, **kwargs)
        # we'll initialize ensemble and chain to None
        self._chain = None
        self._ensemble = None

    @property

    @property

    @property

    @property
        """Returns the beta history currently in memory."""
        # chain betas has shape niterations x ntemps; transpose to
        # ntemps x niterations
        return self._chain.betas.transpose()

    @property
        """Returns the betas that were used at startup."""
        # the initial betas that were used

    @property
    def adaptive(self):
        """Whether or not the betas are adapted."""
        return self._sampler.adaptive

    @property
    def adaptation_lag(self):
        """The adaptation lag for the beta evolution."""
        return self._sampler.adaptation_lag

    @property
    def adaptation_time(self):
        """The adaptation time for the beta evolution."""
        return self._sampler.adaptation_time

    @property
    def scale_factor(self):
        """The scale factor used by ptemcee."""
        return self._sampler.scale_factor

    @property
    def ensemble(self):
        """Returns the current ptemcee ensemble.

        The ensemble stores the current location of and temperatures of the
        walkers. If the ensemble hasn't been setup yet, will set one up
        using p0 for the positions. If set_p0 hasn't been run yet, this will
        result in a ValueError.
        """
        if self._ensemble is None:
            if self._p0 is None:
                raise ValueError("initial positions not set; run set_p0")
            # use the global numpy random state
            rstate = numpy.random.mtrand._rand
            # self._p0 has base_shape x ndim = ntemps x nwalkers x ndim (see
            # BaseMCMC.set_p0). ptemcee's Ensemble expects
            # ntemps x nwalkers x ndim... so we're good
            self._ensemble = self._sampler.ensemble(self._p0, rstate)
        return self._ensemble

    @property
    def _pos(self):
        """Uses the ensemble for the position."""
        # BaseMCMC expects _pos to have shape ntemps x nwalkers x ndim,
        # which is the same shape as ensemble.x
        return self.ensemble.x

    @property
    def chain(self):
        """The current chain of samples in memory.
        The chain is returned as a :py:mod:`ptemcee.chain.Chain` instance. If
        no chain has been created yet (``_chain`` is None), then will create
        a new chain using the current ``ensemble``.
        """
        if self._chain is None:
            # create a chain
            self._chain = ptemcee.chain.Chain(self.ensemble)
        return self._chain

        """
        # set _chain to None; this will both cause the current chain to
        # get garbage collected, and will cause a new chain to be created
        # the next time self.chain is called

    @property
        in memory.
        """
        # chain.x has shape niterations x ntemps x nwalkers x ndim
        # we'll transpose to ntemps x nwalkers x niterations x ndim
        raw_samples = self._chain.x.transpose((1, 2, 0, 3))

    @property

        Unfortunately, because ``ptemcee`` does not have blob support, this

            Since the ``logjacobian`` is not saved by ``ptemcee``, the
            ``logprior`` returned here is the log of the prior pdf in the
            sampling coordinate frame rather than the variable params frame.
            This differs from the variable params frame by the log of the
            Jacobian of the transform from one frame to the other. If no
            sampling transforms were used, then the ``logprior`` is the same.
        """
        # log likelihood and prior have shape
        # niterations x ntemps x nwalkers; we'll tranpose to have shape
        # ntemps x nwalkers x niterations
        logl = self._chain.logl.transpose((1, 2, 0))
        logp = self._chain.logP.transpose((1, 2, 0))

        """
            # set the numpy random state
            numpy.random.set_state(rstate)
            # set the ensemble to its last state
            ensemble = self.ensemble
            for attr, val in fp.read_ensemble_attrs().items():
                setattr(ensemble, attr, val)
            ensemble.betas = fp.read_betas(iteration=-1)
            ensemble.time = fp.niterations

        """
        self.chain.run(niterations)

        """Calculates the log evidence from the given file.
        This uese ``ptemcee``'s thermodynamic integration.
            ``PTEmceeFile``.
            to 0.
        Returns
        -------
        lnZ : float
        """
            betas = fp.read_betas(thin_start=thin_start,
                                  thin_interval=thin_interval,
                                  thin_end=thin_end)
            # we'll separate betas out by their unique temperatures
            # there's probably a faster way to do this...
            mean_logls = []
            unique_betas = []
            for ti in range(ntemps):
                ubti, idx = numpy.unique(betas[ti, :], return_inverse=True)
                unique_idx = numpy.unique(idx)
                loglsti = logls[ti, :, :]
                for ii in unique_idx:
                    # average over the walkers and iterations with the same
                    # betas
                    getiters = numpy.where(ii == unique_idx)[0]
                    mean_logls.append(loglsti[:, getiters].mean())
                    unique_betas.append(ubti[ii])
        return ptemcee.util.thermodynamic_integration_log_evidence(
            numpy.array(unique_betas), numpy.array(mean_logls))

        Returns
        -------
        dict :
        """

        Returns
        -------
        dict
        """

        * ``nwalkers = INT`` :
        * ``ntemps = INT`` :
            The number of temperatures to use. This may be used in combination
            with ``Tmax``. Either this, ``Tmax``, ``betas`` or ``betas-file``
            must be provided.
        * ``tmax = FLOAT`` :
            The maximum temperature to use. This may be used in combination
            with ``ntemps``, or alone.
        * ``betas = FLOAT1 FLOAT2 [...]`` :
            Space-separated list of (intial) inverse temperatures ("betas") to
            use. This sets both the number of temperatures and the tmax. A
            ``ValueError`` will be raised if both this and ``ntemps`` or
            ``Tmax`` are provided.
        * ``betas-file = STR`` :
            ``.attrs['betas']``. A ``ValueError`` will be raised if both this
            and ``betas`` are provided.
        * ``adaptive =`` :
            If provided, temperature adaptation will be turned on.
        * ``adaptation-lag = INT`` :
            The adaptation lag to use (see ptemcee for details).
        * ``adaptation-time = INT`` :
            The adaptation time to use (see ptemcee for details).
        * ``scale-factor = FLOAT`` :
            The scale factor to use for the emcee stretch.
        * ``niterations = INT`` :
        * ``effective-nsamples = INT`` :
            both).
        * ``thin-interval = INT`` :
        * ``max-samples-per-chain = INT`` :
        * ``checkpoint-interval = INT`` :
        * ``checkpoint-signal = STR`` :
        * ``logl-function = STR`` :

        :py:func:`EnsembleMultiTemperedMCMCBurnInTests.from_config` for
        details. If no ``burn-in-test`` is provided, no burn in tests will be
        carried out.

        Returns
        -------
        """
        optargs = {}
        # get the temperature level settings
        ntemps = get_optional_arg_from_config(cp, section, 'ntemps', int)
        if ntemps is not None:
            optargs['ntemps'] = ntemps
        tmax = get_optional_arg_from_config(cp, section, 'tmax', float)
        if tmax is not None:
            optargs['Tmax'] = tmax
        betas = get_optional_arg_from_config(cp, section, 'betas')
        if betas is not None:
            # convert to list sorted in descencding order
            betas = numpy.sort(list(map(float, shlex.split(betas))))[::-1]
            optargs['betas'] = betas
        betas_file = get_optional_arg_from_config(cp, section, 'betas-file')
        if betas_file is not None:
            optargs['betas'] = read_betas_from_hdf(betas_file)
        if betas is not None and betas_file is not None:
            raise ValueError("provide either betas or betas-file, not both")
        if 'betas' in optargs and (ntemps is not None or tmax is not None):
            raise ValueError("provide either ntemps/tmax or betas/betas-file, "
                             "not both")
        # adaptation parameters
        adaptive = get_optional_arg_from_config(cp, section, 'adaptive')
        if adaptive is not None:
            optargs['adaptive'] = True
        else:
            optargs['adaptive'] = False
        adaptation_lag = get_optional_arg_from_config(cp, section,
                                                      'adaptation-lag', int)
            optargs['adaptation_lag'] = adaptation_lag
        adaptation_time = get_optional_arg_from_config(cp, section,
                                                       'adaptation-time', int)
            optargs['adaptation_time'] = adaptation_time
        scale_factor = get_optional_arg_from_config(cp, section,
                                                    'scale-factor', float)
            optargs['scale_factor'] = scale_factor
                  use_mpi=use_mpi, **optargs)
        else:
        return obj

        """
            # write betas
            fp.write_betas(self.betas, last_iteration=self.niterations)
            # write attributes of the ensemble
            fp.write_ensemble_attrs(self.ensemble)

        """

        """

# Copyright (C) 2020  Alex Nitz
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides classes and functions for using the ultranest sampler
"""

import sys

from pycbc.inference.io.ultranest import UltranestFile
from pycbc.io.hdf import dump_state
from pycbc.pool import use_mpi

#
#
#                                   Samplers
#
#

class UltranestSampler(BaseSampler):
    """This class is used to construct an Ultranest sampler from the ultranest
    package.

    log_dir : str
        Folder where files should be stored for resuming (optional).
    stepsampling : bool
        If false, uses rejection sampling. If true, uses
        hit-and-run sampler, which scales better with dimensionality.
    """
    name = "ultranest"
    _io = UltranestFile

    def __init__(self, model, log_dir=None,
                 stepsampling=False,
                 enable_plots=False,
        super(UltranestSampler, self).__init__(model)

        import ultranest
        log_likelihood_call, prior_call = setup_calls(model, copy_prior=True)

        for param in self.variable_params:
                periodic.append(True)
            else:
                periodic.append(False)

        self._sampler = ultranest.ReactiveNestedSampler(
            list(self.model.variable_params),
            log_likelihood_call,
            prior_call, log_dir=log_dir,
            wrapped_params=periodic,
            resume=True)

        if stepsampling:
            import ultranest.stepsampler
            self._sampler.stepsampler = ultranest.stepsampler.RegionBallSliceSampler(
                nsteps=100, adaptive_nsteps='move-distance',
                region_filter=True)

        self.enable_plots = enable_plots
        self.nlive = 0
        self.ndim = len(self.model.variable_params)
        self.result = None
        self.kwargs = kwargs  # Keywords for the run method of ultranest

        do_mpi, _, rank = use_mpi()
        self.main = (not do_mpi) or (rank == 0)

        self.result = self._sampler.run(**self.kwargs)
        if not self.main:
            sys.exit(0)
        self._sampler.print_results()

        if self.enable_plots:
            self._sampler.plot()

    @property

    @property
        return self.result['niter']

    def from_config(cls, cp, model, output_file=None, **kwds):
        """
        Loads the sampler from the given config file.
        """
        skeys = {}
        opts = {'update_interval_iter_fraction': float,
                'update_interval_ncall': int,
                'log_interval': int,
                'show_status': bool,
                'dlogz': float,
                'dKL': float,
                'frac_remain': float,
                'Lepsilon': float,
                'min_ess': int,
                'max_iters': int,
                'max_ncalls': int,
                'log_dir': str,
                'stepsampling': bool,
                'enable_plots': bool,
                'max_num_improvement_loops': int,
                'min_num_live_points': int,
                'cluster_num_live_points:': int}
        for opt_name in opts:
            if cp.has_option('sampler', opt_name):
                value = cp.get('sampler', opt_name)
                skeys[opt_name] = opts[opt_name](value)
        inst = cls(model, **skeys)

        if not do_mpi or (rank == 0):
            setup_output(inst, output_file)
        return inst

        pass

        pass

    @property
        return {}

    @property
        from ultranest.utils import resample_equal

        # we'll do the resampling ourselves so we can pick up
        # additional parameters
        try:  # Remove me on next ultranest release
            wsamples = self.result['weighted_samples']['v']
            weights = self.result['weighted_samples']['w']
            logl = self.result['weighted_samples']['L']
            wsamples = self.result['weighted_samples']['points']
            weights = self.result['weighted_samples']['weights']
            logl = self.result['weighted_samples']['logl']

        wsamples = numpy.column_stack((wsamples, logl))
        params = list(self.model.variable_params) + ['loglikelihood']
        samples = resample_equal(wsamples, weights / weights.sum())
        samples_dict = {p: samples[:, i] for i, p in enumerate(params)}

        """
            fp.write_samples(self.samples, self.samples.keys())
            # write log evidence
            fp.write_logevidence(self.logz, self.logz_err)

            # write full ultranest formatted results
            dump_state(self.result, fp,
                       path='sampler_info',
                       dsetname='presult')

    @property
        """
        ultranest sampler
        """
        return self.result['logz']

    @property
        """
        """
        return self.result['logzerr']

#
#

"""
This modules provides a list of implemented samplers for parameter estimation.
"""

# pylint: disable=unused-import
from .base import (initial_dist_from_config, create_new_output_file)
from .multinest import MultinestSampler
from .ultranest import UltranestSampler

# list of available samplers
samplers = {cls.name: cls for cls in (
    MultinestSampler,
    UltranestSampler,
)}

try:
    from .emcee import EmceeEnsembleSampler
    from .emcee_pt import EmceePTSampler
    samplers[EmceeEnsembleSampler.name] = EmceeEnsembleSampler
    samplers[EmceePTSampler.name] = EmceePTSampler
    pass

try:
    from .epsie import EpsieSampler
    samplers[EpsieSampler.name] = EpsieSampler
    pass

try:
    from .ptemcee import PTEmceeSampler
    samplers[PTEmceeSampler.name] = PTEmceeSampler
    pass

try:
    from .cpnest import CPNestSampler
    samplers[CPNestSampler.name] = CPNestSampler
    pass

try:
    from .dynesty import DynestySampler
    samplers[DynestySampler.name] = DynestySampler
    pass

def load_from_config(cp, model, **kwargs):
    """Loads a sampler from the given config file.

    This looks for a name in the section ``[sampler]`` to determine which
    sampler class to load. That sampler's ``from_config`` is then called.

        Config parser to read from.
    model : pycbc.inference.model
        Which model to pass to the sampler.
        All other keyword arguments are passed directly to the sampler's
        ``from_config`` file.

    Returns
    -------
    sampler :
        The initialized sampler.
    """
    name = cp.get('sampler', 'name')
    return samplers[name].from_config(cp, model, **kwargs)

# Copyright (C) 2012  Alex Nitz, Tito Dal Canton
#
#
#
#

#
#
#                                   Preamble
#
#
"""This module provides utilities for injecting signals into data"""

import os
import lal
import lalsimulation as sim
from pycbc import frame
from pycbc.opt import LimitedSizeDict
from pycbc.waveform import get_td_waveform, utils as wfutils
from pycbc.waveform import ringdown_td_approximants
from pycbc.types import float64, float32, TimeSeries, load_timeseries
from pycbc.conversions import tau0_from_mass1_mass2
from pycbc.filter import resample_to_delta_t
import pycbc.io

injection_func_map = {
    np.dtype(float32): sim.SimAddInjectionREAL4TimeSeries,
    np.dtype(float64): sim.SimAddInjectionREAL8TimeSeries
}

#
# Remove everything between the dashed lines once we get rid of xml
#
from glue.ligolw import utils as ligolw_utils
from glue.ligolw import ligolw, table, lsctables

# dummy class needed for loading LIGOLW files
class LIGOLWContentHandler(ligolw.LIGOLWContentHandler):
    pass

lsctables.use_in(LIGOLWContentHandler)

# Map parameter names used in pycbc to names used in the sim_inspiral
# table, if they are different
sim_inspiral_map = {
    'ra': 'longitude',
    'dec': 'latitude',
    'approximant': 'waveform',
    }

def set_sim_data(inj, field, data):
    """Sets data of a SimInspiral instance."""
    try:
        sim_field = sim_inspiral_map[field]
        sim_field = field
    # for tc, map to geocentric times
    if sim_field == 'tc':
        inj.geocent_end_time = int(data)
        inj.geocent_end_time_ns = int(1e9*(data % 1))
    else:
        setattr(inj, sim_field, data)

def projector(detector_name, inj, hp, hc, distance_scale=1):
    """ Use the injection row to project the polarizations into the
    detector frame
    """
    detector = Detector(detector_name)

    hp /= distance_scale
    hc /= distance_scale

    try:
        tc = inj.tc
        ra = inj.ra
        dec = inj.dec
    except:
        tc = inj.get_time_geocent()
        ra = inj.longitude
        dec = inj.latitude

    hp.start_time += tc
    hc.start_time += tc

    # taper the polarizations
    try:
        hp_tapered = wfutils.taper_timeseries(hp, inj.taper)
        hc_tapered = wfutils.taper_timeseries(hc, inj.taper)
        hp_tapered = hp
        hc_tapered = hc

    projection_method = 'lal'
    if hasattr(inj, 'detector_projection_method'):
        projection_method = inj.detector_projection_method

    logging.info('Injecting at %s, method is %s', tc, projection_method)

    # compute the detector response and add it to the strain
    signal = detector.project_wave(hp_tapered, hc_tapered,
                                   ra, dec, inj.polarization,
                                   method=projection_method,
                                   reference_time=tc,)
    return signal

def legacy_approximant_name(apx):
    """Convert the old style xml approximant name to a name
    and phase_order. Alex: I hate this function. Please delete this when we
    use Collin's new tables.
    """
    apx = str(apx)
    try:
        order = sim.GetOrderFromString(apx)
    except:
        print("Warning: Could not read phase order from string, using default")
        order = -1
    name = sim.GetStringFromApproximant(sim.GetApproximantFromString(apx))
    return name, order

class _XMLInjectionSet(object):

    """Manages sets of injections: reads injections from LIGOLW XML files
    and injects them into time series.

    sim_file : string
        Path to a LIGOLW XML file containing a SimInspiralTable
        with injection definitions.

    indoc
    table
    """

    def __init__(self, sim_file, **kwds):
        self.indoc = ligolw_utils.load_filename(
            sim_file, False, contenthandler=LIGOLWContentHandler)
        self.table = table.get_table(self.indoc, lsctables.SimInspiralTable.tableName)
        self.extra_args = kwds

    def apply(self, strain, detector_name, f_lower=None, distance_scale=1,
              simulation_ids=None,
              inj_filter_rejector=None,
              injection_sample_rate=None,):
        """Add injections (as seen by a particular detector) to a time series.

        strain : TimeSeries
            Time series to inject signals into, of type float32 or float64.
        detector_name : string
            Name of the detector used for projecting injections.
        f_lower : {None, float}, optional
            Low-frequency cutoff for injected signals. If None, use value
            provided by each injection.
        distance_scale: {1, float}, optional
            Factor to scale the distance of an injection with. The default is
            no scaling.
        simulation_ids: iterable, optional
            If given, only inject signals with the given simulation IDs.
        inj_filter_rejector: InjFilterRejector instance; optional, default=None
            If given send each injected waveform to the InjFilterRejector
            instance so that it can store a reduced representation of that
            injection if necessary.
        injection_sample_rate: float, optional
            The sample rate to generate the signal before injection

        Returns
        -------
        None

        Raises
        ------
        TypeError
            For invalid types of `strain`.
        """
        if strain.dtype not in (float32, float64):
            raise TypeError("Strain dtype must be float32 or float64, not " \
                    + str(strain.dtype))

        lalstrain = strain.lal()
        earth_travel_time = lal.REARTH_SI / lal.C_SI
        t0 = float(strain.start_time) - earth_travel_time
        t1 = float(strain.end_time) + earth_travel_time

        # pick lalsimulation injection function
        add_injection = injection_func_map[strain.dtype]

        delta_t = strain.delta_t
        if injection_sample_rate is not None:
            delta_t = 1.0 / injection_sample_rate

        injections = self.table
        if simulation_ids:
            injections = [inj for inj in injections \
                          if inj.simulation_id in simulation_ids]
        injection_parameters = []
        for inj in injections:
            f_l = inj.f_lower if f_lower is None else f_lower
            # roughly estimate if the injection may overlap with the segment
            # Add 2s to end_time to account for ringdown and light-travel delay
            end_time = inj.get_time_geocent() + 2
            inj_length = tau0_from_mass1_mass2(inj.mass1, inj.mass2, f_l)
            # Start time is taken as twice approx waveform length with a 1s
            # safety buffer
            start_time = inj.get_time_geocent() - 2 * (inj_length + 1)
            if end_time < t0 or start_time > t1:
                continue
            signal = self.make_strain_from_inj_object(inj, delta_t,
                    detector_name, f_lower=f_l, distance_scale=distance_scale)
            signal = resample_to_delta_t(signal, strain.delta_t, method='ldas')
            if float(signal.start_time) > t1:
                continue

            signal = signal.astype(strain.dtype)
            signal_lal = signal.lal()
            add_injection(lalstrain, signal_lal, None)
            injection_parameters.append(inj)
            if inj_filter_rejector is not None:
                sid = inj.simulation_id
                inj_filter_rejector.generate_short_inj_from_inj(signal, sid)

        strain.data[:] = lalstrain.data.data[:]

        injected = copy.copy(self)
        injected.table = lsctables.SimInspiralTable()
        injected.table += injection_parameters
        if inj_filter_rejector is not None:
            inj_filter_rejector.injection_params = injected
        return injected

    def make_strain_from_inj_object(self, inj, delta_t, detector_name,
                                    f_lower=None, distance_scale=1):
        """Make a h(t) strain time-series from an injection object as read from
        a sim_inspiral table, for example.

        inj : injection object
            The injection object to turn into a strain h(t).
        delta_t : float
            Sample rate to make injection at.

        Returns
        --------
        signal : float
            h(t) corresponding to the injection.
        """
        f_l = inj.f_lower if f_lower is None else f_lower

        name, phase_order = legacy_approximant_name(inj.waveform)

        # compute the waveform time series
        hp, hc = get_td_waveform(
            inj, approximant=name, delta_t=delta_t,
            phase_order=phase_order,
            f_lower=f_l, distance=inj.distance,
            **self.extra_args)
        return projector(detector_name,
                         inj, hp, hc, distance_scale=distance_scale)

    def end_times(self):
        """Return the end times of all injections"""
        return [inj.get_time_geocent() for inj in self.table]

    def write(filename, samples, write_params=None, static_args=None):
        """Writes the injection samples to the given xml.

            The name of the file to write to.
        samples : io.FieldArray
            FieldArray of parameters.
        write_params : list, optional
            Only write the given parameter names. All given names must be keys
            in ``samples``. Default is to write all parameters in ``samples``.
        static_args : dict, optional
            Dictionary mapping static parameter names to values. These are
            written to the ``attrs``.
        """
        xmldoc = ligolw.Document()
        xmldoc.appendChild(ligolw.LIGO_LW())
        simtable = lsctables.New(lsctables.SimInspiralTable)
        xmldoc.childNodes[0].appendChild(simtable)
        if static_args is None:
            static_args = {}
        if write_params is None:
            write_params = samples.fieldnames
        for ii in range(samples.size):
            sim = lsctables.SimInspiral()
            # initialize all elements to None
            for col in sim.__slots__:
                setattr(sim, col, None)
            for field in write_params:
                data = samples[ii][field]
                set_sim_data(sim, field, data)
            # set any static args
            for (field, value) in static_args.items():
                set_sim_data(sim, field, value)
            simtable.append(sim)
        ligolw_utils.write_filename(xmldoc, filename,
                                    gz=filename.endswith('gz'))

class _HDFInjectionSet(object):
    """Manages sets of injections: reads injections from hdf files

        Path to an hdf file containing injections.
    \**kwds :
        The rest of the keyword arguments are passed to the waveform generation
        function when generating injections.

    filehandler
    table
    static_args
    extra_args
    required_params : tuple
        Parameter names that must exist in the injection HDF file in order to
        create an injection of that type.
    """
    _tableclass = pycbc.io.FieldArray
    injtype = None
    required_params = ()

    def __init__(self, sim_file, hdf_group=None, **kwds):
        # open the file
        fp = h5py.File(sim_file, 'r')
        group = fp if hdf_group is None else fp[hdf_group]
        self.filehandler = fp
        # get parameters
        parameters = list(group.keys())
        # get all injection parameter values
        injvals = {param: group[param][()] for param in parameters}
        # make sure Numpy S strings are loaded as strings and not bytestrings
        # (which could mess with approximant names, for example)
        for k in injvals:
            if injvals[k].dtype.kind == 'S':
                injvals[k] = injvals[k].astype('U')
        # if there were no variable args, then we only have a single injection
        if len(parameters) == 0:
            numinj = 1
        else:
            numinj = tuple(injvals.values())[0].size
        # add any static args in the file
        try:
            # ensure parameter names are string types
            self.static_args = group.attrs['static_args'].astype('U')
            self.static_args = []
        parameters.extend(self.static_args)
        # we'll expand the static args to be arrays with the same size as
        # the other values
        for param in self.static_args:
            val = group.attrs[param]
            # if val is a list or numpy array, we need to store it as an
            # object; otherwise, we'll get a shape mismatch between fields
            if isinstance(val, (np.ndarray, list, tuple)):
                arr = np.empty(numinj, dtype=object)
                for ii in range(numinj):
                    arr[ii] = val
            else:
                # otherwise, we can just repeat the value the needed number of
                # times
                arr = np.repeat(val, numinj)
            # make sure any byte strings are stored as strings instead
            if arr.dtype.char == 'S':
                arr = arr.astype('U')
            injvals[param] = arr
        # make sure required parameters are provided
        missing = set(self.required_params) - set(injvals.keys())
            raise ValueError("required parameter(s) {} not found in the given "
                             "injection file".format(', '.join(missing)))
        # initialize the table
        self.table = self._tableclass.from_kwargs(**injvals)
        # save the extra arguments

    def apply(self, strain, detector_name, distance_scale=1,
              simulation_ids=None, inj_filter_rejector=None,
              **kwargs):
        """Adds injections to a detector's time series."""
        pass

                                    distance_scale=1, **kwargs):
        """Make a h(t) strain time-series from an injection object.
        """
        pass

        pass

    def supported_approximants(self):
        """Return a list of the supported approximants."""
        pass

    def write(cls, filename, samples, write_params=None, static_args=None,
              **metadata):
        """Writes the injection samples to the given hdf file.

        \**metadata :
            All other keyword arguments will be written to the file's attrs.
        """
        with h5py.File(filename, 'w') as fp:
            # write metadata
            if static_args is None:
                static_args = {}
            fp.attrs["static_args"] = list(static_args.keys())
            fp.attrs['injtype'] = cls.injtype
            for key, val in metadata.items():
                fp.attrs[key] = val
            if write_params is None:
                write_params = samples.fieldnames
            for arg, val in static_args.items():
                try:
                    fp.attrs[arg] = val
                except TypeError:
                    # can get this in python 3 if the val was numpy.str_ type
                    # try decoding it and writing
                    fp.attrs[arg] = str(val)
                try:
                    fp[field] = samples[field]
                except TypeError as e:
                    # can get this in python 3 if the val was a numpy.str_ type
                    # we'll try again as a string type
                    if samples[field].dtype.char == 'U':
                        fp[field] = samples[field].astype('S')
                    else:
                        raise e

class CBCHDFInjectionSet(_HDFInjectionSet):
    """Manages CBC injections.
    """
    _tableclass = pycbc.io.WaveformArray
    injtype = 'cbc'
    required_params = ('tc',)

        Returns
        -------
        None

        Raises
        ------
        TypeError
        """

            injections = injections[list(simulation_ids)]
        for ii, inj in enumerate(injections):
            end_time = inj.tc + 2
            start_time = inj.tc - 2 * (inj_length + 1)
                continue
                     detector_name, f_lower=f_l,
                     distance_scale=distance_scale)
                continue

                inj_filter_rejector.generate_short_inj_from_inj(signal, ii)

        injected.table = injections

            The injection object to turn into a strain h(t). Can be any
            object which has waveform parameters as attributes, such as an
            element in a ``WaveformArray``.

        Returns
        --------
        """
        if f_lower is None:
            f_l = inj.f_lower
        else:
            f_l = f_lower

        hp, hc = get_td_waveform(inj, delta_t=delta_t, f_lower=f_l,
                                 **self.extra_args)

        return self.table.tc

    def supported_approximants():
        all_apprxs = []
        for d in [waveform.waveform.td_wav, waveform.waveform.fd_wav]:
            for key in d:
                all_apprxs.extend(d[key])
        return list(set(all_apprxs))

class RingdownHDFInjectionSet(_HDFInjectionSet):
    """Manages a ringdown injection: reads injection from hdf file
    and injects it into time series.
    """
    injtype = 'ringdown'

              simulation_ids=None, inj_filter_rejector=None):
        """Add injection (as seen by a particular detector) to a time series.

        distance_scale: float, optional
            Factor to scale the distance of an injection with. The default (=1)
            is no scaling.
        inj_filter_rejector: InjFilterRejector instance, optional
            Not implemented. If not ``None``, a ``NotImplementedError`` will
            be raised.

        Returns
        -------
        None

        Raises
        ------
        NotImplementedError
            If an ``inj_filter_rejector`` is provided.
        TypeError
        """
            raise NotImplementedError("Ringdown injections do not support "
                                      "inj_filter_rejector")

        for ii in range(injections.size):
            injection = injections[ii]
            signal = self.make_strain_from_inj_object(
                injection, strain.delta_t, detector_name,
                distance_scale=distance_scale)

            strain.data[:] = lalstrain.data.data[:]

                                    distance_scale=1):

        Returns
        --------
        """
        hp, hc = ringdown_td_approximants[inj['approximant']](
            inj, delta_t=delta_t, **self.extra_args)

        """Return the approximate end times of all injections.

        Currently, this just assumes all ringdowns are 2 seconds long.
        """
        # the start times are the tcs
        tcs = self.table.tc
        # FIXME: this could be figured out using the ringdown module
        return tcs + 2

        return list(waveform.ringdown_td_approximants.keys())

class IncoherentFromFileHDFInjectionSet(_HDFInjectionSet):
    """Manages injecting an arbitrary time series loaded from a file.

    The injections must have the following attributes set:

    * ``filename``: (str) the name of the file to load containing the time
      series. The file type and format can be a frame file or anything
      understood by :py:func:`pycbc.types.timeseries.load_timeseries`. If a
      frame file (ends in ``.gwf``) is specified, a ``channel`` attribute must
      also be set.

    * ``DETECTOR_gps_time``: (float) The GPS time at which the time series
      should be added to the ``DETECTOR`` data, where ``DETECTOR`` is the name
      of the instrument to inject into (e.g., ``h1_gps_time``). **The time
      series will only be injected into a detector if a GPS time is given for
      that detector.** Set to -inf, nan, or do not provide a GPS time for a
      particular detector if you do not want to inject into that detector.

    * ``ref_point``: (str or float) What to use as the reference time of the
      injected time series. The time series will be injected into the detector
      such that the ``ref_point`` in the time series occurs at the specifed
      ``DETECTOR_gps_time``. Options are: ``'start'``, ``'end'``, ``'center'``,
      ``'absmax'``, or a float giving the number of seconds into the time
      series.

    In addition, the following attributes may optionally be provided:

    * ``channel``: (str): If the filename points to a frame file, the channel
      to load in that file. Must be provided for frame files.

    * ``DETECTOR_phase_shift``: (float) Apply a phase shift to the time series
      before adding it to detector ``DETECTOR``.

    * ``DETECTOR_amp_scale``: (float) Scale the amplitude by the given amount

    * ``slice_start``: (float) Slice the time series starting at
      ``ref_point + slice_start`` before injecting into the data. Measured in
      seconds.

    * ``slice_end``: (float) Slice the time series ending at
      ``ref_point + slice_end`` before injecting into the data. Measured in
      seconds.

    * ``left_taper_width``: (float) Taper the start of the time series (after
      slicing) using half a kaiser window over the given number of seconds.
      See `:py:func:waveform.utils.td_taper` for more details.

    * ``right_taper_width``: (float) Taper the end of the time series (after

    The signal will be resampled to the same sample rate as the data it is
    being injected into.

    In order to use with ``pycbc_create_injections``, set the ``approximant``
    name to ``'incoherent_from_file'``.
    """
    injtype = 'incoherent_from_file'
    required_params = ('filename', 'ref_point')
    _buffersize = 10
    _buffer = None
    _rtbuffer = None

        raise NotImplementedError("IncoherentFromFile times cannot be "
                                  "determined without loading time series")

        return ['incoherent_from_file']

    def loadts(self, inj):
        """Loads an injection time series.

        After the first time a time series is loaded it will be added to an
        internal buffer for faster in case another injection uses the same
        series.
        """
        if self._buffer is None:
            # create the buffer
            self._buffer = LimitedSizeDict(size_limit=self._buffersize)
        try:
            return self._buffer[inj.filename]
            pass
        # not in buffer, so load
        if inj.filename.endswith('.gwf'):
            try:
                channel = inj.channel
            except AttributeError as _err:
                # Py3.XX: uncomment the "from _err" when we drop 2.7
                raise ValueError("Must provide a channel for "
                                 "frame files") #from _err
            ts = frame.read_frame(inj.filename, channel)
        else:
            ts = load_timeseries(inj.filename)
        # cache
        self._buffer[inj.filename] = ts
        return ts

    def set_ref_time(self, inj, ts):
        """Sets t=0 of the given time series based on what the given
        injection's ``ref_point`` is.
        """
        try:
            ref_point = inj.ref_point
        except AttributeError as _err:
            # Py3.XX: uncomment the "from _err" when we drop 2.7
            raise ValueError("Must provide a ref_point for {} injections"
                             .format(self.injtype))  #from _err
        # try to get from buffer
        if self._rtbuffer is None:
            self._rtbuffer = LimitedSizeDict(size_limit=self._buffersize)
        try:
            reftime = self._rtbuffer[inj.filename, ref_point]
            if ref_point == "start":
                reftime = 0.
            elif ref_point == "end":
                reftime = -len(ts)*ts.delta_t
            elif ref_point == "center":
                reftime = -len(ts)*ts.delta_t/2.
            elif ref_point == "absmax":
                reftime = -ts.abs_arg_max()*ts.delta_t
            elif isinstance(ref_point, (float, int)):
                reftime = -float(ref_point)
            else:
                raise ValueError("Unrecognized ref_point {} provided"
                                 .format(ref_point))
            self._rtbuffer[inj.filename, ref_point] = reftime
        ts._epoch = reftime

    def slice_and_taper(inj, ts):
        """Slices and tapers a timeseries based on the injection settings.

        This assumes that ``set_ref_time`` has been applied to the timeseries
        first. A copy of the time series will be returned even if no slicing
        or tapering is done.
        """
        try:
            tstart = inj.slice_start
            tstart = ts.start_time
        try:
            tend = inj.slice_end
            tend = ts.end_time
        ts = ts.time_slice(tstart, tend).copy()
        # now taper
        try:
            twidth = inj.left_taper_width
            twidth = 0
        if twidth:
            ts = wfutils.td_taper(ts, ts.start_time, ts.start_time+twidth,
                                  side='left')
        try:
            twidth = inj.right_taper_width
            twidth = 0
        if twidth:
            ts = wfutils.td_taper(ts, ts.end_time-twidth, ts.end_time,
                                  side='right')
        return ts

              injection_sample_rate=None, inj_filter_rejector=None):
            raise NotImplementedError("IncoherentFromFile injections do not "
                                      "support inj_filter_rejector")
            delta_t = 1./injection_sample_rate
        else:
            delta_t = strain.delta_t
            # Check if we should inject or not...
            # loading the time series like this is a bit brute-force, since
            # we really only need to know the delta_t and length of the
            # timeseries if the ref_point is anything but absmax, but that
            # would require adding logic to figure out how to get that metadata
            # based on the filetype and ref_point
            ts = self.loadts(inj)
            # set the ref time
            self.set_ref_time(inj, ts)
            # determine if we inject or not based on the times
            try:
                injtime = inj['{}_gps_time'.format(detector_name).lower()]
                injtime = -np.inf
            if np.isnan(injtime):
                # nan means don't inject
            start_time = injtime + ts.start_time
            end_time = injtime + ts.end_time
            inject = (start_time < strain.end_time and
                      end_time > strain.start_time)
            if inject:
                ts = self.make_strain_from_inj_object(
                    inj, delta_t, detector_name,
                    distance_scale=distance_scale, ts=ts)
                if ts.delta_t != strain.delta_t:
                    ts = resample_to_delta_t(ts, strain.delta_t, method='ldas')
                strain.inject(ts, copy=False)

                                    distance_scale=1, ts=None):
        if ts is None:
        # slice and taper
        ts = self.slice_and_taper(inj, ts)
        # shift reference to the detector time
        ts._epoch += inj['{}_gps_time'.format(detector_name).lower()]
        # resample
        ts = resample_to_delta_t(ts, delta_t, method='ldas')
        # apply any phase shift
        try:
            phase_shift = inj[
                '{}_phase_shift'.format(detector_name).lower()]
            phase_shift = 0
        if phase_shift:
            fs = ts.to_frequencyseries()
            fs *= np.exp(1j*phase_shift)
            ts = fs.to_timeseries()
        # apply any scaling
        try:
            amp_scale = inj[
                '{}_amp_scale'.format(detector_name).lower()]
            amp_scale = 1.
        amp_scale /= distance_scale
        ts *= amp_scale
        return ts

hdfinjtypes = {
    CBCHDFInjectionSet.injtype: CBCHDFInjectionSet,
    RingdownHDFInjectionSet.injtype: RingdownHDFInjectionSet,
    IncoherentFromFileHDFInjectionSet.injtype:
    IncoherentFromFileHDFInjectionSet,
}

def get_hdf_injtype(sim_file):
    """Gets the HDFInjectionSet class to use with the given file.

    This looks for the ``injtype`` in the given file's top level ``attrs``. If
    that attribute isn't set, will default to :py:class:`CBCHDFInjectionSet`.

    sim_file : str
        Name of the file. The file must already exist.

    Returns
    -------
    HDFInjectionSet :
        The type of HDFInjectionSet to use.
    """
    with h5py.File(sim_file, 'r') as fp:
        try:
            ftype = fp.attrs['injtype']
            ftype = CBCHDFInjectionSet.injtype
    try:
        return hdfinjtypes[ftype]
        # may get a key error if the file type was stored as unicode instead
        # of string; if so, try decoding it
        try:
            ftype = str(ftype.decode())
            # not actually a byte error; passing will reraise the KeyError
            pass

def hdf_injtype_from_approximant(approximant):
    """Gets the HDFInjectionSet class to use with the given approximant.

    approximant : str
        Name of the approximant.

    Returns
    -------
    """
    retcls = None
    for cls in hdfinjtypes.values():
        if approximant in cls.supported_approximants():
            retcls = cls
    if retcls is None:
        # none were found, raise an error
        raise ValueError("Injection file type unknown for approximant {}"
                         .format(approximant))
    return retcls

class InjectionSet(object):
    """Manages sets of injections and injects them into time series.

    Injections are read from either LIGOLW XML files or HDF files.

        Path to an hdf file or a LIGOLW XML file that contains a
        SimInspiralTable.
    \**kwds :

    table
    """

        ext = os.path.basename(sim_file)
        if ext.endswith(('.xml', '.xml.gz', '.xmlgz')):
            self._injhandler = _XMLInjectionSet(sim_file, **kwds)
            self.indoc = self._injhandler.indoc
        else:
            # assume hdf file
            self._injhandler = get_hdf_injtype(sim_file)(sim_file, **kwds)
        self.table = self._injhandler.table
        self.extra_args = self._injhandler.extra_args
        self.apply = self._injhandler.apply
        self.make_strain_from_inj_object = \
            self._injhandler.make_strain_from_inj_object
        self.end_times = self._injhandler.end_times

    def write(filename, samples, write_params=None, static_args=None,
              injtype=None, **metadata):

        injtype : str, optional
            Specify which `HDFInjectionSet` class to use for writing. If not
            provided, will try to determine it by looking for an approximant in
            the ``static_args``, followed by the ``samples``.
        """
        # DELETE the following "if" once xml is dropped
        ext = os.path.basename(filename)
            _XMLInjectionSet.write(filename, samples, write_params,
                                   static_args)
        else:
            # try determine the injtype if it isn't given
            if injtype is None:
                if static_args is not None and 'approximant' in static_args:
                    injcls = hdf_injtype_from_approximant(
                        static_args['approximant'])
                elif 'approximant' in samples.fieldnames:
                    apprxs = np.unique(samples['approximant'])
                    # make sure they all correspond to the same injection type
                    injcls = [hdf_injtype_from_approximant(a) for a in apprxs]
                    if not all(c == injcls[0] for c in injcls):
                        raise ValueError("injections must all be of the same "
                                         "type")
                    injcls = injcls[0]
                else:
                    raise ValueError("Could not find an approximant in the "
                                     "static args or samples to determine the "
                                     "injection type. Please specify an "
                                     "injtype instead.")
            else:
                injcls = hdfinjtypes[injtype]
            injcls.write(filename, samples, write_params, static_args,
                         **metadata)

    def from_cli(opt):
        """Return an instance of InjectionSet configured as specified
        on the command line.
        """
        if opt.injection_file is None:

        kwa = {}
        if opt.injection_f_ref is not None:
            kwa['f_ref'] = opt.injection_f_ref
        if opt.injection_f_final is not None:
            kwa['f_final'] = opt.injection_f_final
        return InjectionSet(opt.injection_file, **kwa)

class SGBurstInjectionSet(object):
    """Manages sets of sine-Gaussian burst injections: reads injections
    from LIGOLW XML files and injects them into time series.

        Path to a LIGOLW XML file containing a SimBurstTable

    indoc
    table
    """

        self.table = table.get_table(
            self.indoc, lsctables.SimBurstTable.tableName)

    def apply(self, strain, detector_name, f_lower=None, distance_scale=1):

        distance_scale: {1, foat}, optional

        Returns
        -------
        None

        Raises
        ------
        TypeError
        """

        #detector = Detector(detector_name)

        for inj in self.table:
            end_time = inj.get_time_geocent()
            #CHECK: This is a hack (10.0s); replace with an accurate estimate
            inj_length = 10.0
            eccentricity = 0.0
            polarization = 0.0
            start_time = end_time - 2 * inj_length
                continue

            # compute the waveform time series
            hp, hc = sim.SimBurstSineGaussian(float(inj.q),
                float(inj.frequency),float(inj.hrss),float(eccentricity),
                float(polarization),float(strain.delta_t))
            hp = TimeSeries(hp.data.data[:], delta_t=hp.deltaT, epoch=hp.epoch)
            hc = TimeSeries(hc.data.data[:], delta_t=hc.deltaT, epoch=hc.epoch)
            hp._epoch += float(end_time)
            hc._epoch += float(end_time)
            if float(hp.start_time) > t1:
                continue

            # compute the detector response, taper it if requested
            # and add it to the strain
            strain = wfutils.taper_timeseries(strain, inj.taper)
            signal_lal = hp.astype(strain.dtype).lal()

# Copyright (C) 2006--2016  Kipp Cannon
#
#
#

#
#
#                                   Preamble
#
#

"""
While the ligolw module provides classes and parser support for reading and
writing LIGO Light Weight XML documents, this module supplements that code
with classes and parsers that add intelligence to the in-RAM document
representation.

In particular, the document tree associated with a Table element is
enhanced.  During parsing, the Stream element in this module converts the
character data contained within it into a list of objects.  The list
contains one object for each row of the table, and the objects' attributes
are the names of the table's columns.  When the document is written out
again, the Stream element serializes the row objects back into character
data.

The Table element exports a list-like interface to the rows.  The Column
elements also provide list-like access to the values in the corresponding
columns of the table.
"""

import re
from six.moves import zip
import sys
from xml.sax.saxutils import escape as xmlescape
from xml.sax.xmlreader import AttributesImpl

from glue import git_version
from . import ligolw
from . import tokenizer
from . import types as ligolwtypes

__author__ = "Kipp Cannon <kipp.cannon@ligo.org>"
__version__ = "git id %s" % git_version.id
__date__ = git_version.date

#
#
#                                  Utilities
#
#

def get_table(xmldoc, name):
	"""
	Scan xmldoc for a Table element named name.  Raises ValueError if
	not exactly 1 such table is found.

	NOTE:  if a Table sub-class has its .tableName attribute set, then
	its .get_table() class method can be used instead.  This is true
	for all Table classes in the glue.ligolw.lsctables module, and it
	is recommended to always use the .get_table() class method of those
	classes to retrieve those standard tables instead of calling this
	function and passing the .tableName attribute.  The example below
	shows both techniques.

	Example:

	>>> from . import ligolw, lsctables
	>>> xmldoc = ligolw.Document()
	>>> xmldoc.appendChild(ligolw.LIGO_LW()).appendChild(lsctables.New(lsctables.SnglInspiralTable))
	[]
	>>> # find table with this function
	>>> sngl_inspiral_table = get_table(xmldoc, lsctables.SnglInspiralTable.tableName)
	>>> # find table with .get_table() class method (preferred)
	>>> sngl_inspiral_table = lsctables.SnglInspiralTable.get_table(xmldoc)

	See also the .get_table() class method of the Table class.
	"""
	tables = Table.getTablesByName(xmldoc, name)
	if len(tables) != 1:
		raise ValueError("document must contain exactly one %s table" % Table.TableName(name))
	return tables[0]

def reassign_ids(elem):
	"""
	Recurses over all Table elements below elem whose next_id
	attributes are not None, and uses the .get_next_id() method of each
	of those Tables to generate and assign new IDs to their rows.  The
	modifications are recorded, and finally all ID attributes in all
	rows of all tables are updated to fix cross references to the
	modified IDs.

	This function is used by ligolw_add to assign new IDs to rows when
	merging documents in order to make sure there are no ID collisions.
	Using this function in this way requires the .get_next_id() methods
	of all Table elements to yield unused IDs, otherwise collisions
	will result anyway.  See the .sync_next_id() method of the Table
	class for a way to initialize the .next_id attributes so that
	collisions will not occur.

	Example:

	[]
	>>> reassign_ids(xmldoc)
	"""
	mapping = {}
	for tbl in elem.getElementsByTagName(ligolw.Table.tagName):
		if tbl.next_id is not None:
			tbl.updateKeyMapping(mapping)
		tbl.applyKeyMapping(mapping)

#
#
#                                Column Element
#
#

class Column(ligolw.Column):
	"""
	High-level column element that provides list-like access to the
	values in a column.

	Example:

	>>> from xml.sax.xmlreader import AttributesImpl
	>>> import sys
	>>> tbl = Table(AttributesImpl({u"Name": u"test"}))
	>>> col = tbl.appendChild(Column(AttributesImpl({u"Name": u"test:snr", u"Type": u"real_8"})))
	>>> tbl.appendChild(TableStream(AttributesImpl({u"Name": u"test"})))	# doctest: +ELLIPSIS
	<glue.ligolw.table.TableStream object at ...>
	>>> tbl._update_column_info()
	>>> assert col.Name == 'snr'
	>>> assert col.Type == 'real_8'
	>>> # append 3 rows (with nothing in them)
	>>> tbl.append(tbl.RowType())
	>>> # assign values to the rows, in order, in this column
	>>> col[:] = [8.0, 10.0, 12.0]
	>>> col[:]
	[8.0, 10.0, 12.0]
	>>> col.asarray()	# doctest: +NORMALIZE_WHITESPACE
	array([  8.,  10.,  12.])
	>>> tbl.write(sys.stdout)	# doctest: +NORMALIZE_WHITESPACE
	<Table Name="test">
		<Column Name="test:snr" Type="real_8"/>
		<Stream Name="test">
			8,
			10,
			12
		</Stream>
	</Table>
	>>> col.index(10)
	1
	>>> 12 in col
	True
	>>> col[0] = 9.
	>>> col[1] = 9.
	>>> col[2] = 9.
	>>> tbl.write(sys.stdout)		# doctest: +NORMALIZE_WHITESPACE
			9,
			9,
			9
		</Stream>
	</Table>
	>>> col.count(9)
	3

	NOTE:  the .Name attribute returns the stripped "Name" attribute of
	the element, e.g. with the table suffix removed, but when assigning
	to the .Name attribute the value provided is stored without
	modification, i.e. there is no attempt to reattach the table's name
	to the string.  The calling code is responsible for doing the
	correct manipulations.  Therefore, the assignment operation below

	>>> assert col.Name, col.getAttribute("Name") == ('snr', 'test:snr')
	>>> col.Name = col.Name
	>>> assert col.Name, col.getAttribute("Name") == ('snr', 'snr')

	does not preserve the value of the "Name" attribute (though it does
	preserve the stripped form reported by the .Name property).  This
	asymmetry is necessary because the correct table name string to
	reattach to the attribute's value cannot always be known, e.g., if
	the Column object is not part of an XML tree and does not have a
	parent node.
	"""
	# FIXME: the pattern should be
	#
	# r"(?:\A[a-z0-9_]+:|\A)(?P<FullName>(?:[a-z0-9_]+:|\A)(?P<Name>[a-z0-9_]+))\Z"
	#
	# but people are putting upper case letters in names!!!!!  Someone
	# is going to get the beats.  There is a reason for requiring names
	# to be all lower case:  SQL table and column names are case
	# insensitive, therefore (i) when converting a document to SQL the
	# columns "Rho" and "rho" would become indistinguishable and so it
	# would be impossible to convert a document with columns having
	# names like this into an SQL database;  and (ii) even if that
	# degeneracy is not encountered the case cannot be preserved and so
	# when converting back to XML the correct capitalization is lost.
	# Requiring names to be all lower-case creates the same
	# degeneracies in XML representations that exist in SQL
	# representations ensuring compatibility and defines the correct
	# case to restore the names to when converting to XML.  Other rules
	# can be imagined that would work as well, this is the one that got
	# chosen.
	class ColumnName(ligolw.LLWNameAttr):
		dec_pattern = re.compile(r"(?:\A\w+:|\A)(?P<FullName>(?:(?P<Table>\w+):|\A)(?P<Name>\w+))\Z")
		enc_pattern = u"%s"

	Name = ligolw.attributeproxy(u"Name", enc = ColumnName.enc, dec = ColumnName)

	def __len__(self):
		"""
		The number of values in this column.
		"""
		return len(self.parentNode)

	def __getitem__(self, i):
		"""
		Retrieve the value in this column in row i.
		"""
		if isinstance(i, slice):
			return [getattr(r, self.Name) for r in self.parentNode[i]]
		else:
			return getattr(self.parentNode[i], self.Name)

	def __setitem__(self, i, value):
		"""
		Set the value in this column in row i.  i may be a slice.

		NOTE:  Unlike normal Python lists, the length of the Column
		cannot be changed as it is tied to the number of rows in
		the Table.  Therefore, if i is a slice, value should be an
		iterable with exactly the correct number of items.  No
		check is performed to ensure that this is true:  if value
		contains too many items the extras will be ignored, and if
		value contains too few items only as many rows will be
		updated as there are items.
		"""
			for r, val in zip(self.parentNode[i], value):
				setattr(r, self.Name, val)
		else:
			setattr(self.parentNode[i], self.Name, value)

	def __delitem__(self, *args):
		raise NotImplementedError

	def __iter__(self):
		"""
		Return an iterator object for iterating over values in this
		column.
		"""
		for row in self.parentNode:
			yield getattr(row, self.Name)

	def count(self, value):
		"""
		Return the number of rows with this column equal to value.
		"""
		return sum(x == value for x in self)

	def index(self, value):
		"""
		Return the smallest index of the row(s) with this column
		equal to value.
		"""
		for i, x in enumerate(self):
			if x == value:
				return i
		raise ValueError(value)

	def __contains__(self, value):
		"""
		Returns True or False if there is or is not, respectively,
		a row containing val in this column.
		"""
		return value in iter(self)

	def asarray(self):
		"""
		Construct a numpy array from this column.  Note that this
		creates a copy of the data, so modifications made to the
		array will *not* be recorded in the original document.
		"""
		# most codes don't use this feature, this is the only place
		# numpy is used here, and importing numpy can be
		# time-consuming, so we derfer the import until needed.
		import numpy
		try:
			dtype = ligolwtypes.ToNumPyType[self.Type]
		except KeyError as e:
			raise TypeError("cannot determine numpy dtype for Column '%s': %s" % (self.getAttribute("Name"), e))
		return numpy.fromiter(self, dtype = dtype)

	@classmethod
	def getColumnsByName(cls, elem, name):
		"""
		Return a list of Column elements named name under elem.
		"""
		name = cls.ColumnName(name)
		return elem.getElements(lambda e: (e.tagName == cls.tagName) and (e.Name == name))

#
#
#                                Stream Element
#
#

#
# A subclass of tokenizer.RowBuilder that interns strings.
#

class InterningRowBuilder(tokenizer.RowBuilder):
	"""
	This subclass of the tokenizer.RowBuilder class respects the
	"interning" hints provided by table definitions, and attempts to
	replace the values of row attributes associated with interned
	columns with references to shared instances of those values.  This
	results in a reduction in memory use which is small for most
	documents, but can be subtantial when dealing with tables
	containing large volumes of repeated information.

	Example:

	>>> class Row(object):
	...	pass
	...
	>>> # 3rd arg is optional list of attributes to intern
	>>> rows = InterningRowBuilder(Row, ["name", "age"], ("name",))
	>>> l = list(rows.append(["Dick", 20., "Jane", 75., "Dick", 22.]))
	>>> l[0].name
	'Dick'
	>>> l[2].name
	'Dick'
	>>> l[2].name is l[0].name
	True

	Note that Python naturally interns short strings, so this example
	would return True regardless;  it is intended only to demonstrate
	the use of the class.

	The values are stored in a dictionary that is shared between all
	instances of this class, and which survives forever.  Nothing is
	ever naturally "uninterned", so the string dictionary grows without
	bound as more documents are processed.  This can be a problem in
	some use cases, and the work-around is to run

	>>> InterningRowBuilder.strings.clear()

	to reset the dictionary at appropriate points in the application.
	Typically this would be done immediately after each document is
	loaded.
	"""
	strings = {}
	def append(self, tokens):
		interns = self.interns
		setdefault = self.strings.setdefault
		for row in super(InterningRowBuilder, self).append(tokens):
			for col in interns:
				val = getattr(row, col)
				setattr(row, col, setdefault(val, val))
			yield row

#
# Stream class
#

class TableStream(ligolw.Stream):
	"""
	High-level Stream element for use inside Tables.  This element
	knows how to parse the delimited character stream into row objects
	that it appends into the list-like parent element, and knows how to
	turn the parent's rows back into a character stream.
	"""
	#
	# Select the RowBuilder class to use when parsing tables.
	#

	RowBuilder = tokenizer.RowBuilder

	def config(self, parentNode):
		# some initialization that requires access to the
		# parentNode, and so cannot be done inside the __init__()
		# function.
		loadcolumns = set(parentNode.columnnames)
		if parentNode.loadcolumns is not None:
			# FIXME:  convert loadcolumns attributes to sets to
			# avoid the conversion.
			loadcolumns &= set(parentNode.loadcolumns)
		self._tokenizer = tokenizer.Tokenizer(self.Delimiter)
		self._tokenizer.set_types([(pytype if colname in loadcolumns else None) for pytype, colname in zip(parentNode.columnpytypes, parentNode.columnnames)])
		columnnames = [name for name in parentNode.columnnames if name in loadcolumns]
		# FIXME:  convert interncolumns attributes to sets to
		# simplify computing the intersection
		interncolumns = [name for name in (parentNode.interncolumns or set()) if name in columnnames]
		self._rowbuilder = self.RowBuilder(parentNode.RowType, columnnames, interncolumns)
		return self

	def appendData(self, content):
		# tokenize buffer, pack into row objects, and append to
		# table
		appendfunc = self.parentNode.append
		for row in self._rowbuilder.append(self._tokenizer.append(content)):
			appendfunc(row)

	def endElement(self):
		# stream tokenizer uses delimiter to identify end of each
		# token, so add a final delimiter to induce the last token
		# to get parsed but only if there's something other than
		# whitespace left in the tokenizer's buffer.  the writing
		# code will have put a final delimiter into the stream if
		# the final token was pure whitespace in order to
		# unambiguously indicate that token's presence
		if not self._tokenizer.data.isspace():
			self.appendData(self.Delimiter)
		# now we're done with these
		del self._tokenizer
		del self._rowbuilder
		# call parent's _end_of_rows() hook.
		self.parentNode._end_of_rows()

	def write(self, fileobj = sys.stdout, indent = u""):
		# retrieve the .write() method of the file object to avoid
		# doing the attribute lookup in loops
		w = fileobj.write
		# loop over parent's rows.  This is complicated because we
		# need to not put a delimiter at the end of the last row
		# unless it ends with a null token
		w(self.start_tag(indent))
		rowdumper = tokenizer.RowDumper(self.parentNode.columnnames, [ligolwtypes.FormatFunc[coltype] for coltype in self.parentNode.columntypes], self.Delimiter)
		rowdumper.dump(self.parentNode)
		try:
			line = next(rowdumper)
		except StopIteration:
			# table is empty
			pass
		else:
			# write first row
			newline = u"\n" + indent + ligolw.Indent
			w(newline)
			# the xmlescape() call replaces things like "<"
			# with "&lt;" so that the string will not confuse
			# an XML parser when the file is read.  turning
			# "&lt;" back into "<" during file reading is
			# handled by the XML parser, so there is no code
			# in Glue related to that.
			w(xmlescape(line))
			# now add delimiter and write the remaining rows
			newline = rowdumper.delimiter + newline
			for line in rowdumper:
				w(newline)
				w(xmlescape(line))
			if rowdumper.tokens and rowdumper.tokens[-1] == u"":
				# the last token of the last row was null:
				# add a final delimiter to indicate that a
				# token is present
				w(rowdumper.delimiter)
		w(u"\n" + self.end_tag(indent) + u"\n")

#
#
#                                Table Element
#
#

class Table(ligolw.Table, list):
	"""
	High-level Table element that knows about its columns and rows.
	"""
	class TableName(ligolw.LLWNameAttr):
		dec_pattern = re.compile(r"(?:\A[a-z0-9_]+:|\A)(?P<Name>[a-z0-9_]+):table\Z")
		enc_pattern = u"%s:table"

	Name = ligolw.attributeproxy(u"Name", enc = TableName.enc, dec = TableName)

	validcolumns = None
	loadcolumns = None
	interncolumns = None
	constraints = None
	how_to_index = None
	next_id = None

	class RowType(object):
		"""
		Helpful parent class for row objects.  Also used as the
		default row class by Table instances.  Provides an
		__init__() method that accepts keyword arguments from which
		the object's attributes are initialized.

		Example:

		>>> x = Table.RowType(a = 0.0, b = "test", c = True)
		>>> x.a
		0.0
		>>> x.b
		'test'
		>>> x.c
		True

		Also provides .__getstate__() and .__setstate__() methods
		to allow row objects to be pickled (otherwise, because they
		all use __slots__ to reduce their memory footprint, they
		aren't pickleable).
		"""
		def __init__(self, **kwargs):
			for key, value in kwargs.items():
				setattr(self, key, value)

		def __getstate__(self):
			return dict((key, getattr(self, key)) for key in self.__slots__ if hasattr(self, key))

		def __setstate__(self, state):
			self.__init__(**state)

	def __init__(self, *args):
		"""
		Initialize
		"""
		super(Table, self).__init__(*args)
		self.columnnames = []
		self.columntypes = []
		self.columnpytypes = []

	#
	# Table retrieval
	#

	def getTablesByName(cls, elem, name):
		"""
		Return a list of Table elements named name under elem.
		"""
		name = cls.TableName(name)

	def get_table(cls, xmldoc):
		"""
		Equivalent to the module-level function get_table(), but
		uses the .tableName attribute of this class to provide the
		name of the table to search for.  The Table parent class
		does not provide a .tableName attribute, but sub-classes,
		especially those in lsctables.py, do provide a value for
		that attribute, and in those cases this class method
		provides a cleaner way to retrieve them.

		Example:

		>>> from . import ligolw, lsctables
		>>> xmldoc = ligolw.Document()
		>>> xmldoc.appendChild(ligolw.LIGO_LW()).appendChild(lsctables.New(lsctables.SnglInspiralTable))
		[]
		>>> sngl_inspiral_table = lsctables.SnglInspiralTable.get_table(xmldoc)
		"""
		return get_table(xmldoc, cls.tableName)

	def copy(self):
		"""
		Construct and return a new Table document subtree whose
		structure is the same as this table, that is it has the
		same columns etc..  The rows are not copied.  Note that a
		fair amount of metadata is shared between the original and
		new tables.  In particular, a copy of the Table object
		itself is created (but with no rows), and copies of the
		child nodes are created.  All other object references are
		shared between the two instances, such as the RowType
		attribute on the Table object.
		"""
		new = copy.copy(self)
		new.childNodes = []	# got reference to original list
		for elem in self.childNodes:
			new.appendChild(copy.copy(elem))
		del new[:]
		new._end_of_columns()
		new._end_of_rows()
		return new

	def CheckElement(cls, elem):
		"""
		Return True if element is a Table element whose Name
		attribute matches the .tableName attribute of this class ;
		return False otherwise.  See also .CheckProperties().
		"""
		return cls.CheckProperties(elem.tagName, elem.attributes)

	def CheckProperties(cls, tagname, attrs):
		"""
		Return True if tagname and attrs are the XML tag name and
		element attributes, respectively, of a Table element whose
		Name attribute matches the .tableName attribute of this
		class;  return False otherwise.  The Table parent class
		that attribute.  See also .CheckElement()

		Example:

		>>> from . import lsctables
		>>> lsctables.ProcessTable.CheckProperties(u"Table", {u"Name": u"process:table"})
		True
		"""
		return tagname == cls.tagName and cls.TableName(attrs[u"Name"]) == cls.tableName

	#
	# Column access
	#

	def getColumnByName(self, name):
		"""
		Retrieve and return the Column child element named name.
		The comparison is done using the stripped names.  Raises
		KeyError if this table has no column by that name.

		Example:

		>>> tbl = lsctables.New(lsctables.SnglInspiralTable)
		>>> col = tbl.getColumnByName("mass1")
		"""
		try:
			col, = Column.getColumnsByName(self, name)
		except ValueError:
			# did not find exactly 1 matching child
			raise KeyError(name)
		return col

	def appendColumn(self, name):
		"""
		Append a Column element named "name" to the table.  Returns
		the new child.  Raises ValueError if the table already has
		a column by that name, and KeyError if the validcolumns
		attribute of this table does not contain an entry for a
		column by that name.

		Note that the name string is assumed to be "pre-stripped",
		that is it is the significant portion of the elements Name
		attribute.  The Column element's Name attribute will be
		constructed by pre-pending the stripped Table element's
		name and a colon.

		Example:

		>>> process_table = lsctables.New(lsctables.ProcessTable, [])
		>>> col = process_table.appendColumn("program")
		>>> assert col.getAttribute("Name") == 'process:program'
		>>> assert col.Name == 'program'
		"""
		try:
			self.getColumnByName(name)
			# if we get here the table already has that column
			raise ValueError("duplicate Column '%s'" % name)
		except KeyError:
			pass
		column = Column(AttributesImpl({u"Name": "%s:%s" % (self.Name, name), u"Type": self.validcolumns[name]}))
		streams = self.getElementsByTagName(ligolw.Stream.tagName)
		if streams:
			self.insertBefore(column, streams[0])
		else:
			self.appendChild(column)
		return column

	#
	# Row access
	#

	def appendRow(self, *args, **kwargs):
		"""
		Create and append a new row to this table, then return it

		All positional and keyword arguments are passed to the RowType
		constructor for this table.
		"""
		row = self.RowType(*args, **kwargs)
		self.append(row)
		return row

	#
	# Element methods
	#

	def _update_column_info(self):
		"""
		Used for validation during parsing, and additional
		book-keeping.  For internal use only.
		"""
		del self.columnnames[:]
		del self.columntypes[:]
		del self.columnpytypes[:]
		for child in self.getElementsByTagName(ligolw.Column.tagName):
			if self.validcolumns is not None:
				try:
					if self.validcolumns[child.Name] != child.Type:
						raise ligolw.ElementError("invalid type '%s' for Column '%s' in Table '%s', expected type '%s'" % (child.Type, child.getAttribute("Name"), self.getAttribute("Name"), self.validcolumns[child.Name]))
				except KeyError:
					raise ligolw.ElementError("invalid Column '%s' for Table '%s'" % (child.getAttribute("Name"), self.getAttribute("Name")))
			if child.Name in self.columnnames:
				raise ligolw.ElementError("duplicate Column '%s' in Table '%s'" % (child.getAttribute("Name"), self.getAttribute("Name")))
			self.columnnames.append(child.Name)
			self.columntypes.append(child.Type)
			try:
				self.columnpytypes.append(ligolwtypes.ToPyType[child.Type])
			except KeyError:
				raise ligolw.ElementError("unrecognized Type '%s' for Column '%s' in Table '%s'" % (child.Type, child.getAttribute("Name"), self.getAttribute("Name")))

	def _verifyChildren(self, i):
		"""
		"""
		super(Table, self)._verifyChildren(i)
		child = self.childNodes[i]
		if child.tagName == ligolw.Column.tagName:
			self._update_column_info()
		elif child.tagName == ligolw.Stream.tagName:
			# require agreement of non-stripped strings
			if child.getAttribute("Name") != self.getAttribute("Name"):
				raise ligolw.ElementError("Stream name '%s' does not match Table name '%s'" % (child.getAttribute("Name"), self.getAttribute("Name")))

	def _end_of_columns(self):
		"""
		Called during parsing to indicate that the last Column
		child element has been added.  Subclasses can override this
		to perform any special action that should occur following
		the addition of the last Column element.
		"""
		pass

	def _end_of_rows(self):
		"""
		Called during parsing to indicate that the last row has
		been added.  Subclasses can override this to perform any
		special action that should occur following the addition of
		the last row.
		"""
		pass

	def removeChild(self, child):
		"""
		Remove a child from this element.  The child element is
		returned, and it's parentNode element is reset.
		"""
		super(Table, self).removeChild(child)
		return child

	def unlink(self):
		"""
		Break internal references within the document tree rooted
		on this element to promote garbage collection.
		"""
		super(Table, self).unlink()
		del self[:]

		# Table elements are allowed to contain 0 Stream children,
		# but _end_of_columns() and _end_of_rows() hooks must be
		# called regardless, so we do that here if needed.
		if self.childNodes[-1].tagName != ligolw.Stream.tagName:
			self._end_of_columns()
			self._end_of_rows()

	#
	# Row ID manipulation
	#

	def get_next_id(cls):
		"""
		Returns the current value of the next_id class attribute,
		and increments the next_id class attribute by 1.  Raises
		ValueError if the table does not have an ID generator
		associated with it.
		"""
		# = None if no ID generator
		id = cls.next_id
		cls.next_id += 1
		return id

	def set_next_id(cls, id):
		"""
		Sets the value of the next_id class attribute.  This is a
		convenience function to help prevent accidentally assigning
		a value to an instance attribute instead of the class
		attribute.
		"""
		cls.next_id = id

	def reset_next_id(cls):
		"""
		If the current value of the next_id class attribute is not
		None then set it to 0, otherwise it is left unmodified.

		Example:

		>>> for cls in lsctables.TableByName.values(): cls.reset_next_id()
		"""
		if cls.next_id is not None:
			cls.set_next_id(type(cls.next_id)(0))

	def sync_next_id(self):
		"""
		Determines the highest-numbered ID in this table, and sets
		the table's .next_id attribute to the next highest ID in
		sequence.  If the .next_id attribute is already set to a
		value greater than the highest value found, then it is left
		unmodified.  The return value is the ID identified by this
		method.  If the table's .next_id attribute is None, then
		this function is a no-op.

		Note that tables of the same name typically share a common
		.next_id attribute (it is a class attribute, not an
		attribute of each instance) so that IDs can be generated
		that are unique across all tables in the document.  Running
		sync_next_id() on all the tables in a document that are of
		the same type will have the effect of setting the ID to the
		next ID higher than any ID in any of those tables.

		Example:

		>>> tbl = lsctables.New(lsctables.ProcessTable)
		>>> print(tbl.sync_next_id())
		process:process_id:0
		"""
		if self.next_id is not None:
			if len(self):
				n = max(self.getColumnByName(self.next_id.column_name)) + 1
			else:
				n = type(self.next_id)(0)
			if n > self.next_id:
				self.set_next_id(n)
		return self.next_id

	def updateKeyMapping(self, mapping):
		"""
		Used as the first half of the row key reassignment
		algorithm.  Accepts a dictionary mapping old key --> new
		key.  Iterates over the rows in this table, using the
		table's next_id attribute to assign a new ID to each row,
		recording the changes in the mapping.  Returns the mapping.
		Raises ValueError if the table's next_id attribute is None.
		"""
		if self.next_id is None:
			raise ValueError(self)
		try:
			column = self.getColumnByName(self.next_id.column_name)
			# table is missing its ID column, this is a no-op
			return mapping
		for i, old in enumerate(column):
			if old is None:
				raise ValueError("null row ID encountered in Table '%s', row %d" % (self.getAttribute("Name"), i))
			if old in mapping:
				column[i] = mapping[old]
			else:
				column[i] = mapping[old] = self.get_next_id()
		return mapping

	def applyKeyMapping(self, mapping):
		"""
		Used as the second half of the key reassignment algorithm.
		Loops over each row in the table, replacing references to
		old row keys with the new values from the mapping.
		"""
		for coltype, colname in zip(self.columntypes, self.columnnames):
			if coltype in ligolwtypes.IDTypes and (self.next_id is None or colname != self.next_id.column_name):
				column = self.getColumnByName(colname)
				for i, old in enumerate(column):
					try:
						column[i] = mapping[old]
					except KeyError:
						pass

#
#
#                               Content Handler
#
#

#
# Override portions of a ligolw.LIGOLWContentHandler class
#

def use_in(ContentHandler):
	"""
	Modify ContentHandler, a sub-class of
	glue.ligolw.LIGOLWContentHandler, to cause it to use the Table,
	Column, and Stream classes defined in this module when parsing XML
	documents.

	Example:

	>>> from glue.ligolw import ligolw
	>>> class LIGOLWContentHandler(ligolw.LIGOLWContentHandler):
	...	pass
	...
	>>> use_in(LIGOLWContentHandler)
	<class 'glue.ligolw.table.LIGOLWContentHandler'>
	"""
	def startColumn(self, parent, attrs):
		return Column(attrs)

	def startStream(self, parent, attrs, __orig_startStream = ContentHandler.startStream):
		if parent.tagName == ligolw.Table.tagName:
			parent._end_of_columns()
			return TableStream(attrs).config(parent)
		return __orig_startStream(self, parent, attrs)

	def startTable(self, parent, attrs):
		return Table(attrs)

	ContentHandler.startColumn = startColumn
	ContentHandler.startStream = startStream
	ContentHandler.startTable = startTable

	return ContentHandler

#
#
#

#
#
#                                   Preamble
#
#

"""
LSC Table definitions.  These must be kept synchronized with the official
definitions in the LDAS CVS repository at
http://www.ldas-sw.ligo.caltech.edu/cgi-bin/cvsweb.cgi/ldas/dbms/db2/sql.
Maintenance of the table definitions is left to the conscience of
interested users.
"""

try:
	from functools import total_ordering
	from functools32 import total_ordering
import operator
from xml import sax

from ligo import segments

from glue import iterutils
import lal
from lal import LIGOTimeGPS
from . import table
from . import ilwd

#
#
#                            Convenience Functions
#
#

def New(cls, columns = None, **kwargs):
	"""
	Construct a pre-defined LSC table.  The optional columns argument
	is a sequence of the names of the columns the table should be
	constructed with.  If columns = None, then the table is constructed
	with all valid columns (use columns = [] to create a table with no
	columns).

	Example:

	>>> tbl = New(ProcessTable, [u"process_id", u"start_time", u"end_time", u"comment"])
	<Table Name="process:table">
		<Column Name="process:comment" Type="lstring"/>
		<Column Name="process:end_time" Type="int_4s"/>
		<Column Name="process:process_id" Type="ilwd:char"/>
		<Column Name="process:start_time" Type="int_4s"/>
		<Stream Delimiter="," Name="process:table" Type="Local">
		</Stream>
	</Table>
	"""
	new = cls(sax.xmlreader.AttributesImpl({u"Name": cls.TableName.enc(cls.tableName)}), **kwargs)
	colnamefmt = new.Name + u":%s"
	if columns is not None:
		for key in sorted(columns):
			if key not in new.validcolumns:
				raise ligolw.ElementError("invalid Column '%s' for Table '%s'" % (key, new.Name))
			new.appendChild(table.Column(sax.xmlreader.AttributesImpl({u"Name": colnamefmt % key, u"Type": new.validcolumns[key]})))
	else:
		for key, value in sorted(new.validcolumns.items()):
			new.appendChild(table.Column(sax.xmlreader.AttributesImpl({u"Name": colnamefmt % key, u"Type": value})))
	new._end_of_columns()
	new.appendChild(table.TableStream(sax.xmlreader.AttributesImpl({u"Name": new.getAttribute(u"Name"), u"Delimiter": table.TableStream.Delimiter.default, u"Type": table.TableStream.Type.default})))
	return new

def HasNonLSCTables(elem):
	"""
	Return True if the document tree below elem contains non-LSC
	tables, otherwise return False.
	"""
	return any(t.Name not in TableByName for t in elem.getElementsByTagName(ligolw.Table.tagName))

class instrumentsproperty(object):
	def __init__(self, name):
		self.name = name

	@staticmethod
	def get(ifos):
		"""
		Parse the values stored in the "ifos" and "instruments"
		columns found in many tables.  This function is mostly for
		internal use by the .instruments properties of the
		corresponding row classes.  The mapping from input to
		output is as follows (rules are applied in order):

		input is None --> output is None

		input contains "," --> output is set of strings split on
		"," with leading and trailing whitespace stripped from each
		piece and empty strings removed from the set

		input contains "+" --> output is set of strings split on
		"+" with leading and trailing whitespace stripped from each

		else, after stripping input of leading and trailing
		whitespace,

		input has an even length greater than two --> output is set
		of two-character pieces

		input is a non-empty string --> output is a set containing
		input as single value

		else output is an empty set.

		NOTE:  the complexity of this algorithm is a consequence of
		there being several conventions in use for encoding a set
		of instruments into one of these columns;  it has been
		proposed that L.L.W.  documents standardize on the
		comma-delimited variant of the encodings recognized by this
		function, and for this reason the inverse function,
		instrumentsproperty.set(), implements that encoding only.

		NOTE:  to force a string containing an even number of
		characters to be interpreted as a single instrument name
		and not to be be split into two-character pieces, add a ","
		character to the end to force the comma-delimited decoding
		to be used.  instrumentsproperty.set() does this for you.

		Example:

		>>> print(instrumentsproperty.get(None))
		None
		>>> assert instrumentsproperty.get(u"") == set()
		>>> assert instrumentsproperty.get(u"  ,  ,,") == set()
		>>> assert instrumentsproperty.get(u"H1") == {'H1'}
		>>> assert instrumentsproperty.get(u"SWIFT") == {'SWIFT'}
		>>> assert instrumentsproperty.get(u"H1L1") == {'H1', 'L1'}
		>>> assert instrumentsproperty.get(u"H1L1,") == {'H1L1'}
		>>> assert instrumentsproperty.get(u"H1,L1") == {'H1', 'L1'}
		>>> assert instrumentsproperty.get(u"H1+L1") == {'H1', 'L1'}
		"""
		if ifos is None:
			return None
		if u"," in ifos:
			result = set(ifo.strip() for ifo in ifos.split(u","))
			result.discard(u"")
			return result
		if u"+" in ifos:
			result = set(ifo.strip() for ifo in ifos.split(u"+"))
		ifos = ifos.strip()
		if len(ifos) > 2 and not len(ifos) % 2:
			# if ifos is a string with an even number of
			# characters greater than two, split it into
			# two-character pieces.  FIXME:  remove this when
			# the inspiral codes don't write ifos strings like
			# this anymore
			return set(ifos[n:n+2] for n in range(0, len(ifos), 2))
		if ifos:
			return set([ifos])
		return set()

	def set(instruments):
		"""
		Convert an iterable of instrument names into a value
		suitable for storage in the "ifos" column found in many
		tables.  This function is mostly for internal use by the
		.instruments properties of the corresponding row classes.
		The input can be None or an iterable of zero or more
		instrument names, none of which may be zero-length, consist
		exclusively of spaces, or contain "," or "+" characters.
		The output is a single string containing the unique
		instrument names concatenated using "," as a delimiter.
		instruments will only be iterated over once and so can be a
		generator expression.  Whitespace is allowed in instrument
		names but might not be preserved.  Repeated names will not
		be preserved.

		NOTE:  in the special case that there is 1 instrument name
		in the iterable and it has an even number of characters > 2
		in it, the output will have a "," appended in order to
		force instrumentsproperty.get() to parse the string back
		into a single instrument name.  This is a special case
		included temporarily to disambiguate the encoding until all
		codes have been ported to the comma-delimited encoding.
		This behaviour will be discontinued at that time.  DO NOT
		WRITE CODE THAT RELIES ON THIS!  You have been warned.

		Example:

		>>> print(instrumentsproperty.set(None))
		None
		>>> assert instrumentsproperty.set(()) == ''
		>>> assert instrumentsproperty.set((u"H1",)) == 'H1'
		>>> assert instrumentsproperty.set((u"H1",u"H1",u"H1")) == 'H1'
		>>> assert instrumentsproperty.set((u"H1",u"L1")) == 'H1,L1'
		>>> assert instrumentsproperty.set((u"SWIFT",)) == 'SWIFT'
		>>> assert instrumentsproperty.set((u"H1L1",)) == 'H1L1,'
		"""
		if instruments is None:
		_instruments = sorted(set(instrument.strip() for instrument in instruments))
		# safety check:  refuse to accept blank names, or names
		# with commas or pluses in them as they cannot survive the
		# encode/decode process
		if not all(_instruments) or any(u"," in instrument or u"+" in instrument for instrument in _instruments):
			raise ValueError(instruments)
		if len(_instruments) == 1 and len(_instruments[0]) > 2 and not len(_instruments[0]) % 2:
			# special case disambiguation.  FIXME:  remove when
			# everything uses the comma-delimited encoding
			return u"%s," % _instruments[0]
		return u",".join(_instruments)

	def __get__(self, obj, cls = None):
		return self.get(getattr(obj, self.name))

	def __set__(self, obj, instruments):
		setattr(obj, self.name, self.set(instruments))

instrument_set_from_ifos = instrumentsproperty.get
ifos_from_instrument_set = instrumentsproperty.set

class gpsproperty(object):
	"""
	Descriptor used internally to implement LIGOTimeGPS-valued
	properties.
	"""
	def __init__(self, s_name, ns_name):
		self.s_name = s_name
		self.get_s = operator.attrgetter(s_name)
		self.ns_name = ns_name
		self.get_ns = operator.attrgetter(ns_name)

	posinf = 0x7FFFFFFF, 0xFFFFFFFF
	neginf = 0xFFFFFFFF, 0xFFFFFFFF
	infs = posinf, neginf

		s = self.get_s(obj)
		ns = self.get_ns(obj)
		if s is None and ns is None:
		if ns == 0xFFFFFFFF:
			return segments.PosInfinity if (s, ns) == self.posinf else segments.NegInfinity
		return LIGOTimeGPS(s, ns)

	def __set__(self, obj, gps):
		if gps is None:
			s = ns = None
		elif isinstance(gps, segments.infinity) or math.isinf(gps):
			if gps > 0:
				s, ns = self.posinf
			elif gps < 0:
				s, ns = self.neginf
			else:
				raise ValueError(gps)
		else:
			try:
				s = gps.gpsSeconds
				ns = gps.gpsNanoSeconds
			except AttributeError:
				# try converting and going again
				return self.__set__(obj, LIGOTimeGPS(gps))
			if abs(ns) > 999999999:
				raise ValueError("denormalized LIGOTimeGPS not allowed")
		setattr(obj, self.s_name, s)
		setattr(obj, self.ns_name, ns)

class gpsproperty_with_gmst(gpsproperty):
	def __init__(self, s_name, ns_name, gmst_name):
		super(gpsproperty_with_gmst, self).__init__(s_name, ns_name)
		self.gmst_name = gmst_name

		super(gpsproperty_with_gmst, self).__set__(obj, gps)
			setattr(obj, self.gmst_name, None)
		else:
			# re-retrieve the value in case it required type
			# conversion
			gps = self.__get__(obj)
			setattr(obj, self.gmst_name, lal.GreenwichMeanSiderealTime(gps))

class segmentproperty(object):
	"""
	Descriptor used internally to expose pairs of GPS-valued properties
	as segment-valued properties.
	"""
	def __init__(self, start_name, stop_name):
		self.start = start_name
		self.stop = stop_name

		start = getattr(obj, self.start)
		stop = getattr(obj, self.stop)
		if start is None and stop is None:
		return segments.segment(start, stop)

	def __set__(self, obj, seg):
		if seg is None:
			start = stop = None
		else:
			start, stop = seg
		setattr(obj, self.start, start)
		setattr(obj, self.stop, stop)

#
#
#                                process:table
#
#

ProcessID = ilwd.get_ilwdchar_class(u"process", u"process_id")

class ProcessTable(table.Table):
	tableName = "process"
	validcolumns = {
		"program": "lstring",
		"version": "lstring",
		"cvs_repository": "lstring",
		"cvs_entry_time": "int_4s",
		"comment": "lstring",
		"is_online": "int_4s",
		"node": "lstring",
		"username": "lstring",
		"unix_procid": "int_4s",
		"start_time": "int_4s",
		"end_time": "int_4s",
		"jobid": "int_4s",
		"domain": "lstring",
		"ifos": "lstring",
		"process_id": "ilwd:char"
	}
	constraints = "PRIMARY KEY (process_id)"
	next_id = ProcessID(0)

	def get_ids_by_program(self, program):
		"""
		Return a set containing the process IDs from rows whose
		program string equals the given program.
		"""
		return set(row.process_id for row in self if row.program == program)

class Process(table.Table.RowType):
	"""
	Example:

	>>> x = Process()
	>>> x.instruments = (u"H1", u"L1")
	>>> assert x.ifos == 'H1,L1'
	>>> assert x.instruments == {'H1', 'L1'}
	"""
	__slots__ = tuple(ProcessTable.validcolumns.keys())

	instruments = instrumentsproperty("ifos")

	def get_ifos(self):
		"""
		Return a set of the instruments for this row.
		"""
		return self.instruments

	def set_ifos(self, instruments):
		"""
		Serialize a sequence of instruments into the ifos
		attribute.  The instrument names must not contain the ","
		character.
		"""
		self.instruments = instruments

ProcessTable.RowType = Process

#
#
#                                lfn:table
#
#

LfnID = ilwd.get_ilwdchar_class(u"lfn", u"lfn_id")

class LfnTable(table.Table):
	tableName = "lfn"
		"process_id": "ilwd:char",
		"lfn_id": "ilwd:char",
		"name": "lstring",
		"end_time": "int_4s"
	}
	constraints = "PRIMARY KEY (lfn_id)"
	next_id = LfnID(0)

class Lfn(table.Table.RowType):
	__slots__ = tuple(LfnTable.validcolumns.keys())

LfnTable.RowType = Lfn

#
#
#                             process_params:table
#
#

class ProcessParamsTable(table.Table):
	tableName = "process_params"
		"param": "lstring",
		"type": "lstring",
		"value": "lstring"
	}
	# FIXME: these constraints break ID remapping in the DB backend.
	# an index is used instead.  switch back to the constraints when I
	# can figure out how not to break remapping.
	#constraints = "PRIMARY KEY (process_id, param)"
	how_to_index = {
		"pp_pip_index": ("process_id", "param"),
	}

	def append(self, row):
		if row.type is not None and row.type not in ligolwtypes.Types:
			raise ligolw.ElementError("unrecognized type '%s'" % row.type)
		table.Table.append(self, row)

class ProcessParams(table.Table.RowType):
	"""
	Example:

	>>> x = ProcessParams()
	>>> x.pyvalue = u"test"
	>>> assert x.type == 'lstring'
	>>> assert x.value == 'test'
	>>> assert x.pyvalue == 'test'
	>>> x.pyvalue = 6.
	>>> assert x.type == 'real_8'
	>>> assert x.value == '6'
	>>> x.pyvalue
	6.0
	>>> x.pyvalue = None
	>>> print(x.type)
	None
	>>> print(x.value)
	None
	>>> print(x.pyvalue)
	None
	>>> x.pyvalue = True
	>>> assert(x.type) == 'int_4s'
	>>> assert x.value == '1'
	1
	"""
	__slots__ = tuple(ProcessParamsTable.validcolumns.keys())

	@property
	def pyvalue(self):
		if self.value is None:
		try:
			parsefunc = ligolwtypes.ToPyType[self.type]
			raise ValueError("invalid type '%s'" % self.type)
		return parsefunc(self.value)

	@pyvalue.setter
	def pyvalue(self, value):
		if value is None:
			self.type = self.value = None
		else:
			try:
				self.type = ligolwtypes.FromPyType[type(value)]
				raise ValueError("type not supported: %s" % repr(type(value)))
			self.value = value if self.type in ligolwtypes.StringTypes else ligolwtypes.FormatFunc[self.type](value)

ProcessParamsTable.RowType = ProcessParams

#
#
#                             search_summary:table
#
#

class SearchSummaryTable(table.Table):
	tableName = "search_summary"
		"shared_object": "lstring",
		"lalwrapper_cvs_tag": "lstring",
		"lal_cvs_tag": "lstring",
		"in_start_time": "int_4s",
		"in_start_time_ns": "int_4s",
		"in_end_time": "int_4s",
		"in_end_time_ns": "int_4s",
		"out_start_time": "int_4s",
		"out_start_time_ns": "int_4s",
		"out_end_time": "int_4s",
		"out_end_time_ns": "int_4s",
		"nevents": "int_4s",
		"nnodes": "int_4s"
	}
		"ss_pi_index": ("process_id",),
	}

	def get_inlist(self):
		"""
		Return a segmentlist object describing the times spanned by
		the input segments of all rows in the table.

		Note:  the result is not coalesced, the segmentlist
		contains the segments as they appear in the table.
		"""
		return segments.segmentlist(row.in_segment for row in self)

	def get_outlist(self):
		"""
		the output segments of all rows in the table.

		"""
		return segments.segmentlist(row.out_segment for row in self)

	def get_in_segmentlistdict(self, process_ids = None):
		"""
		Return a segmentlistdict mapping instrument to in segment
		list.  If process_ids is a sequence of process IDs, then
		only rows with matching IDs are included otherwise all rows
		are included.

		Note:  the result is not coalesced, each segmentlist
		contains the segments listed for that instrument as they
		appeared in the table.
		"""
		seglists = segments.segmentlistdict()
		for row in self:
			ifos = row.instruments or (None,)
			if process_ids is None or row.process_id in process_ids:
				seglists.extend(dict((ifo, segments.segmentlist([row.in_segment])) for ifo in ifos))
		return seglists

	def get_out_segmentlistdict(self, process_ids = None):
		"""
		Return a segmentlistdict mapping instrument to out segment

		"""
				seglists.extend(dict((ifo, segments.segmentlist([row.out_segment])) for ifo in ifos))

class SearchSummary(table.Table.RowType):
	"""
	Example:

	>>> x = SearchSummary()
	>>> x.in_start = x.out_start = LIGOTimeGPS(0)
	>>> x.in_end = x.out_end = LIGOTimeGPS(10)
	>>> x.in_segment
	segment(LIGOTimeGPS(0, 0), LIGOTimeGPS(10, 0))
	>>> x.out_segment
	>>> x.in_segment = x.out_segment = None
	>>> print(x.in_segment)
	None
	>>> print(x.out_segment)
	None
	"""
	__slots__ = tuple(SearchSummaryTable.validcolumns.keys())

	in_start = gpsproperty("in_start_time", "in_start_time_ns")
	in_end = gpsproperty("in_end_time", "in_end_time_ns")
	out_start = gpsproperty("out_start_time", "out_start_time_ns")
	out_end = gpsproperty("out_end_time", "out_end_time_ns")

	in_segment = segmentproperty("in_start", "in_end")
	out_segment = segmentproperty("out_start", "out_end")

		"""
		"""

		"""
		"""

	def get_in(self):
		"""
		Get the input segment.
		"""
		return self.in_segment

	def set_in(self, seg):
		"""
		Set the input segment.
		"""
		self.in_segment = seg

	def get_out(self):
		"""
		Get the output segment.
		"""
		return self.out_segment

	def set_out(self, seg):
		"""
		Set the output segment.
		"""
		self.out_segment = seg

SearchSummaryTable.RowType = SearchSummary

#
#
#                            search_summvars:table
#
#

SearchSummVarsID = ilwd.get_ilwdchar_class(u"search_summvars", u"search_summvar_id")

class SearchSummVarsTable(table.Table):
	tableName = "search_summvars"
		"search_summvar_id": "ilwd:char",
		"string": "lstring",
		"value": "real_8"
	}
	constraints = "PRIMARY KEY (search_summvar_id)"
	next_id = SearchSummVarsID(0)

class SearchSummVars(table.Table.RowType):
	__slots__ = tuple(SearchSummVarsTable.validcolumns.keys())

SearchSummVarsTable.RowType = SearchSummVars

#
#
#                            experiment:table
#
#

ExpDefID = ilwd.get_ilwdchar_class(u"experiment", u"experiment_id")

class ExperimentTable(table.Table):
	tableName = "experiment"
		"experiment_id": "ilwd:char",
		"search_group": "lstring",
		"search": "lstring",
		"lars_id": "lstring",
		"instruments": "lstring",
		"gps_start_time": "int_4s",
		"gps_end_time": "int_4s",
		"comments": "lstring"
	}
	constraints = "PRIMARY KEY (experiment_id)"
	next_id = ExpDefID(0)

	def get_expr_id(self, search_group, search, lars_id, instruments, gps_start_time, gps_end_time, comments = None):
		"""
		Return the expr_def_id for the row in the table whose
		values match the givens.
		If a matching row is not found, returns None.

		@search_group: string representing the search group (e.g., cbc)
		@serach: string representing search (e.g., inspiral)
		@lars_id: string representing lars_id
		@instruments: the instruments; must be a python set
		@gps_start_time: string or int representing the gps_start_time of the experiment
		@gps_end_time: string or int representing the gps_end_time of the experiment
		"""
		# create string from instrument set
		instruments = ifos_from_instrument_set(instruments)

		# look for the ID
			if (row.search_group, row.search, row.lars_id, row.instruments, row.gps_start_time, row.gps_end_time, row.comments) == (search_group, search, lars_id, instruments, gps_start_time, gps_end_time, comments):
				# found it
				return row.experiment_id

		# experiment not found in table
		return None

	def write_new_expr_id(self, search_group, search, lars_id, instruments, gps_start_time, gps_end_time, comments = None):
		"""
		Creates a new def_id for the given arguments and returns it. 
		If an entry already exists with these, will just return that id.

		"""
		
		# check if id already exists
		check_id = self.get_expr_id( search_group, search, lars_id, instruments, gps_start_time, gps_end_time, comments = comments )
		if check_id:
			return check_id

		row = self.RowType()
		row.experiment_id = self.get_next_id()
		row.search_group = search_group
		row.search = search
		row.lars_id = lars_id
		row.instruments = ifos_from_instrument_set(instruments)
		row.gps_start_time = gps_start_time
		row.gps_end_time = gps_end_time
		row.comments = comments

		# return new ID
		return row.experiment_id

	def get_row_from_id(self, experiment_id):
		"""
		Returns row in matching the given experiment_id.
		"""
		row = [row for row in self if row.experiment_id == experiment_id]
		if len(row) > 1:
			raise ValueError("duplicate ids in experiment table")
		if len(row) == 0:
			raise ValueError("id '%s' not found in table" % experiment_id)

		return row[0]

class Experiment(table.Table.RowType):
	__slots__ = tuple(ExperimentTable.validcolumns.keys())

	def get_instruments(self):
		"""
		"""
		return instrument_set_from_ifos(self.instruments)

	def set_instruments(self, instruments):
		"""
		"""
		self.instruments = ifos_from_instrument_set(instruments)

ExperimentTable.RowType = Experiment

#
#
#                            experiment_summary:table
#
#

ExpSummID = ilwd.get_ilwdchar_class(u"experiment_summary", u"experiment_summ_id")

class ExperimentSummaryTable(table.Table):
	tableName = "experiment_summary"
		"experiment_summ_id": "ilwd:char",
		"time_slide_id": "ilwd:char",
		"veto_def_name": "lstring",
		"datatype": "lstring",
		"sim_proc_id": "ilwd:char",
		"duration": "int_4s",
		"nevents": "int_4u"
	}
	constraints = "PRIMARY KEY (experiment_summ_id)"
		"es_ei_index": ("experiment_id",),
		"es_dt_index": ("datatype",)
	}
	next_id = ExpSummID(0)

	datatypes = ['slide', 'all_data', 'playground', 'exclude_play', 'simulation']
		
	def as_id_dict(self):
		"""
		Return table as a dictionary mapping experiment_id, time_slide_id,
		veto_def_name, and sim_proc_id (if it exists) to the expr_summ_id.
		"""
		d = {}
			if row.experiment_id not in d:
				d[row.experiment_id] = {}
			if (row.time_slide_id, row.veto_def_name, row.datatype, row.sim_proc_id) in d[row.experiment_id]:
				# entry already exists, raise error
				raise KeyError("duplicate entries in experiment_summary table")
			d[row.experiment_id][(row.time_slide_id, row.veto_def_name, row.datatype, row.sim_proc_id)] = row.experiment_summ_id

		return d

	def get_expr_summ_id(self, experiment_id, time_slide_id, veto_def_name, datatype, sim_proc_id = None):
		"""
		Return the expr_summ_id for the row in the table whose experiment_id, 
		time_slide_id, veto_def_name, and datatype match the given. If sim_proc_id,
		will retrieve the injection run matching that sim_proc_id.
		"""

			if (row.experiment_id, row.time_slide_id, row.veto_def_name, row.datatype, row.sim_proc_id) == (experiment_id, time_slide_id, veto_def_name, datatype, sim_proc_id):
				# found it
				return row.experiment_summ_id

		# if get to here, experiment not found in table

	def write_experiment_summ(self, experiment_id, time_slide_id, veto_def_name, datatype, sim_proc_id = None ):
		"""
		Writes a single entry to the experiment_summ table. This can be used
		for either injections or non-injection experiments. However, it is
		recommended that this only be used for injection experiments; for
		non-injection experiments write_experiment_summ_set should be used to
		ensure that an entry gets written for every time-slide performed.
		"""
		# check if entry alredy exists; if so, return value
		check_id = self.get_expr_summ_id(experiment_id, time_slide_id, veto_def_name, datatype, sim_proc_id = sim_proc_id)

		row.experiment_summ_id = self.get_next_id()
		row.experiment_id = experiment_id
		row.time_slide_id = time_slide_id
		row.veto_def_name = veto_def_name
		row.datatype = datatype
		row.sim_proc_id = sim_proc_id
		row.nevents = None
		row.duration = None
		
		return row.experiment_summ_id

	def write_non_injection_summary(self, experiment_id, time_slide_dict, veto_def_name, write_all_data = True, write_playground = True, write_exclude_play = True, return_dict = False):
		"""
		Method for writing a new set of non-injection experiments to the experiment
		summary table. This ensures that for every entry in the 
		experiment table, an entry for every slide is added to
		the experiment_summ table, rather than just an entry for slides that
		have events in them. Default is to write a 3 rows for zero-lag: one for
		all_data, playground, and exclude_play. (If all of these are set to false,
		will only slide rows.)
		
		Note: sim_proc_id is hard-coded to None because time-slides
		are not performed with injections.

		@experiment_id: the experiment_id for this experiment_summary set
		@time_slide_dict: the time_slide table as a dictionary; used to figure out
			what is zero-lag and what is slide
		@veto_def_name: the name of the vetoes applied
		@write_all_data: if set to True, writes a zero-lag row who's datatype column
			is set to 'all_data'
		@write_playground: same, but datatype is 'playground'
		@write_exclude_play: same, but datatype is 'exclude_play'
		@return_dict: if set to true, returns an id_dict of the table
		"""
		for slide_id in time_slide_dict:
			# check if it's zero_lag or not
			if not any( time_slide_dict[slide_id].values() ):
				if write_all_data:
					self.write_experiment_summ( experiment_id, slide_id, veto_def_name, 'all_data', sim_proc_id = None )
				if write_playground:
					self.write_experiment_summ( experiment_id, slide_id, veto_def_name, 'playground', sim_proc_id = None )
				if write_exclude_play:
					self.write_experiment_summ( experiment_id, slide_id, veto_def_name, 'exclude_play', sim_proc_id = None )
			else:
				self.write_experiment_summ( experiment_id, slide_id, veto_def_name, 'slide', sim_proc_id = None )

		if return_dict:
			return self.as_id_dict()

	def add_nevents(self, experiment_summ_id, num_events, add_to_current = True):
		"""
		Add num_events to the nevents column in a specific entry in the table. If
		add_to_current is set to False, will overwrite the current nevents entry in
		the row with num_events. Otherwise, default is to add num_events to
		the current value.

		Note: Can subtract events by passing a negative number to num_events.
		"""
			if row.experiment_summ_id != experiment_summ_id:
				continue
			if row.nevents is None:
				row.nevents = 0
			if add_to_current:
				row.nevents += num_events
				return row.nevents
			else:
				row.nevents = num_events
				
		# if get to here, couldn't find experiment_summ_id in the table
		raise ValueError("'%s' could not be found in the table" % (str(experiment_summ_id)))

class ExperimentSummary(table.Table.RowType):
	__slots__ = tuple(ExperimentSummaryTable.validcolumns.keys())

ExperimentSummaryTable.RowType = ExperimentSummary

#
#
#                            experiment_map:table
#
#

class ExperimentMapTable(table.Table):
	tableName = "experiment_map"
		"coinc_event_id": "ilwd:char",
	}
		"em_esi_index": ("experiment_summ_id",),
		"em_cei_index": ("coinc_event_id",)
	}

	def get_experiment_summ_ids( self, coinc_event_id ):
		"""
		Gets all the experiment_summ_ids that map to a given coinc_event_id.
		"""
		experiment_summ_ids = []
			if row.coinc_event_id == coinc_event_id:
				experiment_summ_ids.append(row.experiment_summ_id)
		if len(experiment_summ_ids) == 0:
			raise ValueError("'%s' could not be found in the experiment_map table" % coinc_event_id)
		return experiment_summ_ids

class ExperimentMap(table.Table.RowType):
	__slots__ = tuple(ExperimentMapTable.validcolumns.keys())

ExperimentMapTable.RowType = ExperimentMap

#
#
#                               gds_trigger:table
#
#

GDSTriggerID = ilwd.get_ilwdchar_class(u"gds_trigger", u"event_id")

class GDSTriggerTable(table.Table):
	tableName = "gds_trigger"
		"creator_db": "int_4s",
		"process_id": "ilwd:char_u",
		"filter_id": "ilwd:char",
		"subtype": "lstring",
		"ifo": "lstring",
		"start_time_ns": "int_4s",
		"duration": "real_4",
		"priority": "int_4s",
		"disposition": "int_4s",
		"size": "real_4",
		"significance": "real_4",
		"frequency": "real_4",
		"bandwidth": "real_4",
		"time_peak": "real_4",
		"time_average": "real_4",
		"time_sigma": "real_4",
		"freq_peak": "real_4",
		"freq_average": "real_4",
		"freq_sigma": "real_4",
		"noise_power": "real_4",
		"signal_power": "real_4",
		"pixel_count": "int_4s",
		"confidence": "real_4",
		"binarydata": "ilwd:char_u",
		"binarydata_length": "int_4s",
		"event_id": "ilwd:char"
	}
	constraints = "PRIMARY KEY (event_id)"
	next_id = GDSTriggerID(0)
	interncolumns = ("process_id", "ifo", "subtype")

class GDSTrigger(table.Table.RowType):
	__slots__ = tuple(GDSTriggerTable.validcolumns.keys())

	#
	# Tile properties
	#

	def get_start(self):
		return LIGOTimeGPS(self.start_time, self.start_time_ns)

	def set_start(self, gps):
		self.start_time, self.start_time_ns = gps.seconds, gps.nanoseconds

	def get_stop(self):
		return LIGOTimeGPS(self.start_time, self.start_time_ns) + self.duration

	def get_peak(self):
		return LIGOTimeGPS(self.time_peak, self.time_peak)

	def set_peak(self, gps):
		self.time_peak, self.peak_time_ns = gps.seconds, gps.nanoseconds

	def get_period(self):
		start = LIGOTimeGPS(self.start_time, self.start_time_ns)
		return segments.segment(start, start + self.duration)

	def set_period(self, period):
		self.start_time, self.start_time_ns = period[0].seconds, period[0].nanoseconds
		self.duration = float(abs(period))

	def get_band(self):
		low = self.frequency
		return segments.segment(low, low + self.bandwidth)

	def set_band(self, band):
		self.frequency = band[0]
		self.bandwidth = abs(band)

GDSTriggerTable.RowType = GDSTrigger

#
#
#                               sngl_burst:table
#
#

SnglBurstID = ilwd.get_ilwdchar_class(u"sngl_burst", u"event_id")

class SnglBurstTable(table.Table):
	tableName = "sngl_burst"
		"channel": "lstring",
		"stop_time": "int_4s",
		"stop_time_ns": "int_4s",
		"flow": "real_4",
		"fhigh": "real_4",
		"central_freq": "real_4",
		"amplitude": "real_4",
		"snr": "real_4",
		"chisq": "real_8",
		"chisq_dof": "real_8",
		"tfvolume": "real_4",
		"hrss": "real_4",
		"time_lag": "real_4",
		"peak_time": "int_4s",
		"peak_time_ns": "int_4s",
		"peak_frequency": "real_4",
		"peak_strain": "real_4",
		"peak_time_error": "real_4",
		"peak_frequency_error": "real_4",
		"peak_strain_error": "real_4",
		"ms_start_time": "int_4s",
		"ms_start_time_ns": "int_4s",
		"ms_stop_time": "int_4s",
		"ms_stop_time_ns": "int_4s",
		"ms_duration": "real_4",
		"ms_flow": "real_4",
		"ms_fhigh": "real_4",
		"ms_bandwidth": "real_4",
		"ms_hrss": "real_4",
		"ms_snr": "real_4",
		"ms_confidence": "real_4",
		"param_one_name": "lstring",
		"param_one_value": "real_8",
		"param_two_name": "lstring",
		"param_two_value": "real_8",
		"param_three_name": "lstring",
		"param_three_value": "real_8",
	}
	next_id = SnglBurstID(0)
	interncolumns = ("process_id", "ifo", "search", "channel")

	def get_column(self, column):
		"""@returns: an array of column values for each row in the table

		@param column:
			name of column to return
		@returntype:
			numpy.ndarray
		"""
		if column.lower() == 'q':
			return self.get_q
		else:
			return self.getColumnByName(column).asarray()

		"""@returns: the peak time of each row in the table
		@returntype: numpy.ndarray
		"""
		return numpy.asarray([row.get_peak() for row in self])

		"""@returns: the start time of each row in the table
		"""
		return numpy.asarray([row.get_start() for row in self])

	def get_ms_start(self):
		"""@returns: the start time of the most significant tile for
		each row in the table
		"""
		return numpy.asarray([row.get_ms_start() for row in self])

		"""@returns: the stop time of each row in the table
		"""
		return numpy.asarray([row.get_stop() for row in self])

	def get_ms_stop(self):
		"""@returns: the stop time of the most significant tile for
		"""
		return numpy.asarray([row.get_ms_stop() for row in self])

	def get_q(self):
		"""@returns: the Q of each row in the table
		"""
		return numpy.asarray([row.get_q() for row in self])

	def get_z(self):
		"""@returns: the Z (Omega-Pipeline energy) of each row in the
		table
		"""
		return numpy.asarray([row.get_z() for row in self])

		"""@returns: the period segment of each row in the table
		@returntype: ligo.segments.segmentlist
		"""
		return segments.segmentlist([row.get_period() for row in self])

	def get_ms_period(self):
		"""@returns: the period segment for the most significant tile
		of each row in the table
		"""
		return segments.segmentlist([row.get_ms_period() for row in self])

		"""@returns: the frequency band of each row in the table
		"""
		return segments.segmentlist([row.get_band() for row in self])

	def get_ms_band(self):
		"""@returns: the frequency band of the most significant tile
		for each row in the table
		"""
		return segments.segmentlist([row.get_ms_band() for row in self])

	def veto(self, seglist):
		"""@returns: those rows of the table that don't lie within a
		given seglist
		"""
		keep = self.copy()
			time = row.get_peak()
			if time not in seglist:
				keep.append(row)
		return keep

	def vetoed(self, seglist):
		"""@returns: those rows of the table that lie within a given
		seglist
		"""
		vetoed = self.copy()
			if time in seglist:
				vetoed.append(row)
		return vetoed

	def veto_seglistdict(self, seglistdict):
			if time not in seglistdict[row.ifo]:

	def vetoed_seglistdict(self, seglistdict):
			if time in seglistdict[row.ifo]:

class SnglBurst(table.Table.RowType):
	__slots__ = tuple(SnglBurstTable.validcolumns.keys())

	#
	#

	start = gpsproperty("start_time", "start_time_ns")
	stop = gpsproperty("stop_time", "stop_time_ns")
	peak = gpsproperty("peak_time", "peak_time_ns")

	@property
	def period(self):
		start = self.start
		try:
			stop = self.stop
		except AttributeError:
			stop = None
		# special case:  use duration if stop is not recorded
		if start is not None and stop is None and self.duration is not None:
			stop = start + self.duration

	@period.setter
	def period(self, seg):
			self.start = self.stop = self.duration = None
		else:
			self.start, self.stop = seg
			self.duration = float(abs(seg))

	@property
	def band(self):
		if self.central_freq is None and self.bandwidth is None:
		return segments.segment(self.central_freq - self.bandwidth / 2., self.central_freq + self.bandwidth / 2.)

	@band.setter
	def band(self, seg):
			try:
				self.flow = self.fhigh = None
				# not in LAL C version
				pass
			self.central_freq = self.bandwidth = None
		else:
			try:
				self.flow, self.fhigh = seg
				pass
			self.central_freq = sum(seg) / 2.
			self.bandwidth = abs(seg)

	#
	# "Most significant pixel" properties
	#

	ms_start = gpsproperty("ms_start_time", "ms_start_time_ns")
	ms_stop = gpsproperty("ms_stop_time", "ms_stop_time_ns")
	ms_peak = gpsproperty("ms_peak_time", "ms_peak_time_ns")

	@property
	def ms_period(self):
		start = self.ms_start
		stop = self.ms_stop
		if start is not None and stop is None and self.ms-duration is not None:
			stop = start + self.ms_duration

	@ms_period.setter
	def ms_period(self, seg):
			self.ms_start = self.ms_stop = self.ms_duration = None
		else:
			self.ms_start, self.ms_stop = seg
			self.ms_duration = float(abs(seg))

	@property
	def ms_band(self):
		if self.ms_flow is None and self.ms_bandwidth is None:
		return segments.segment(self.ms_flow, self.ms_flow + self.ms_bandwidth)

	@ms_band.setter
	def ms_band(self, seg):
			self.ms_bandwidth = self.ms_flow = self.ms_fhigh = None
		else:
			self.ms_flow, self.ms_fhigh = seg
			self.ms_bandwidth = abs(seg)

	# legacy compatibility.  DO NOT USE

		return self.start

		self.start = gps

		return self.stop

	def set_stop(self, gps):
		self.stop = gps

		return self.peak

		self.peak = gps

		return self.period

		self.period = period

		return self.band

		self.band = band

		return self.ms_start

	def set_ms_start(self, gps):
		self.ms_start = gps

		return self.ms_stop

	def set_ms_stop(self, gps):
		self.ms_stop = gps

		return self.ms_period

	def set_ms_period(self, period):
		self.ms_period = period

		return self.ms_band

	def set_ms_band(self, band):
		self.ms_band = band

	#
	# Omega-Pipeline properties
	#

		return self.duration * 2 * numpy.pi**(1/2.) * self.central_freq

		return self.snr ** 2 / 2.

SnglBurstTable.RowType = SnglBurst

#
#
#                              multi_burst:table
#
#

#
# FIXME:  I think extra columns have been added here that aren't in other
# places where this table is defined.
#

class MultiBurstTable(table.Table):
	tableName = "multi_burst"
		"false_alarm_rate": "real_4",
		"ligo_axis_ra": "real_4",
		"ligo_axis_dec": "real_4",
		"ligo_angle": "real_4",
		"ligo_angle_sig": "real_4",
		"coinc_event_id": "ilwd:char"
	}
	# FIXME:  like some other tables here, this table should have the
	# constraint that the coinc_event_id column is a primary key.  this
	# breaks ID reassignment in ligolw_sqlite, so until that is fixed
	# the constraint is being replaced with an index.
	#constraints = "PRIMARY KEY (coinc_event_id)"
		"mb_cei_index": ("coinc_event_id",)
	}

class MultiBurst(table.Table.RowType):
	__slots__ = tuple(MultiBurstTable.validcolumns.keys())

	@property
		if start is None and self.duration is None:

			self.start = self.duration = None
		else:
			self.start = seg[0]

	@property

		else:

MultiBurstTable.RowType = MultiBurst

#
#
#                             sngl_inspiral:table
#
#

SnglInspiralID = ilwd.get_ilwdchar_class(u"sngl_inspiral", u"event_id")

class SnglInspiralTable(table.Table):
	tableName = "sngl_inspiral"
		"end_time_ns": "int_4s",
		"end_time_gmst": "real_8",
		"impulse_time": "int_4s",
		"impulse_time_ns": "int_4s",
		"template_duration": "real_8",
		"event_duration": "real_8",
		"eff_distance": "real_4",
		"coa_phase": "real_4",
		"mass1": "real_4",
		"mass2": "real_4",
		"mchirp": "real_4",
		"mtotal": "real_4",
		"eta": "real_4",
		"kappa": "real_4",
		"chi": "real_4",
		"tau0": "real_4",
		"tau2": "real_4",
		"tau3": "real_4",
		"tau4": "real_4",
		"tau5": "real_4",
		"ttotal": "real_4",
		"psi0": "real_4",
		"psi3": "real_4",
		"alpha": "real_4",
		"alpha1": "real_4",
		"alpha2": "real_4",
		"alpha3": "real_4",
		"alpha4": "real_4",
		"alpha5": "real_4",
		"alpha6": "real_4",
		"beta": "real_4",
		"f_final": "real_4",
		"chisq": "real_4",
		"chisq_dof": "int_4s",
		"bank_chisq": "real_4",
		"bank_chisq_dof": "int_4s",
		"cont_chisq": "real_4",
		"cont_chisq_dof": "int_4s",
		"sigmasq": "real_8",
		"rsqveto_duration": "real_4",
		"Gamma0": "real_4",
		"Gamma1": "real_4",
		"Gamma2": "real_4",
		"Gamma3": "real_4",
		"Gamma4": "real_4",
		"Gamma5": "real_4",
		"Gamma6": "real_4",
		"Gamma7": "real_4",
		"Gamma8": "real_4",
		"Gamma9": "real_4",
		"spin1x": "real_4",
		"spin1y": "real_4",
		"spin1z": "real_4",
		"spin2x": "real_4",
		"spin2y": "real_4",
		"spin2z": "real_4",
	}
	# FIXME:  lal uses an ID of 0 to indicate "no valid ID has been
	# set", so we start at 1 for safety, but eventually that should be
	# fixed in LAL and then this can be put back to 0 for cleanliness.
	next_id = SnglInspiralID(1)

	def get_column(self,column,fac=250.,index=6.):
		if column == 'reduced_chisq':
			return self.get_reduced_chisq()
		if column == 'reduced_bank_chisq':
			return self.get_reduced_bank_chisq()
		if column == 'reduced_cont_chisq':
			return self.get_reduced_cont_chisq()
		if column == 'snr_over_chi':
			return self.get_snr_over_chi()
		if column == 'effective_snr':
			return self.get_effective_snr(fac=fac)
		if column == 'new_snr':
			return self.get_new_snr(index=index)
		if column == 'lvS5stat':
			return self.get_lvS5stat()
		elif column == 'chirp_eff_distance':
			return self.get_chirp_eff_dist()
		else:

	def get_end(self):
		return [row.get_end() for row in self]

	def get_reduced_chisq(self):
		return self.get_column('chisq') / (2*self.get_column('chisq_dof') - 2)

	def get_reduced_bank_chisq(self):
		return self.get_column('bank_chisq') / self.get_column('bank_chisq_dof')

	def get_reduced_cont_chisq(self):
		return self.get_column('cont_chisq') / self.get_column('cont_chisq_dof')

	def get_effective_snr(self, fac=250.0):
		snr = self.get_column('snr')
		rchisq = self.get_column('reduced_chisq')
		return snr/ (1 + snr**2/fac)**(0.25) / rchisq**(0.25)

	def get_bank_effective_snr(self, fac=250.0):
		rchisq = self.get_column('reduced_bank_chisq')

	def get_cont_effective_snr(self, fac=250.0):
		rchisq = self.get_column('reduced_cont_chisq')

	def get_new_snr(self, index=6.0):
		# kwarg 'index' is assigned to the parameter chisq_index
		# nhigh gives the asymptotic large  rho behaviour of d (ln chisq) / d (ln rho) 
		# for fixed new_snr eg nhigh = 2 -> chisq ~ rho^2 at large rho 
		nhigh = 2.
		newsnr = snr/ (0.5*(1+rchisq**(index/nhigh)))**(1./index)
		numpy.putmask(newsnr, rchisq < 1, snr)
		return newsnr

	def get_bank_new_snr(self, index=6.0):
		nhigh = 2.
		banknewsnr = snr/ (0.5*(1+rchisq**(index/nhigh)))**(1./index)
		numpy.putmask(banknewsnr, rchisq < 1, snr)
		return banknewsnr

	def get_cont_new_snr(self, index=6.0):
		nhigh = 2.
		contnewsnr = snr/ (0.5*(1+rchisq**(index/nhigh)))**(1./index)
		numpy.putmask(contnewsnr, rchisq < 1, snr)
		return contnewsnr

	def get_chirp_eff_dist(self, ref_mass=1.4):
		mchirp = self.get_column('mchirp')
		eff_dist = self.get_column('eff_distance')
		return SnglInspiral.chirp_distance(eff_dist, mchirp, ref_mass)

	def get_snr_over_chi(self):
		return self.get_column('snr')/self.get_column('chisq')**(1./2)

	def get_lvS5stat(self):
		return self.get_column('beta')

	def ifocut(self, ifo, inplace=False):
		"""
		Return a SnglInspiralTable with rows from self having IFO equal
		to the given ifo. If inplace, modify self directly, else create
		a new table and fill it.
		"""
		if inplace:
			iterutils.inplace_filter(lambda row: row.ifo == ifo, self)
			return self
		else:
			ifoTrigs = self.copy()
			ifoTrigs.extend([row for row in self if row.ifo == ifo])
			return ifoTrigs

	def veto(self,seglist):
			time = row.get_end()
			else:
	
		"""
		Return the inverse of what veto returns, i.e., return the triggers
		that lie within a given seglist.
		"""
			else:
	
			else:
	
			else:
	
	def getslide(self,slide_num):
		"""
		Return the triggers with a specific slide number.
		@param slide_num: the slide number to recover (contained in the event_id)
		"""
		slideTrigs = self.copy()
		slideTrigs.extend(row for row in self if row.get_slide_number() == slide_num)
		return slideTrigs

class SnglInspiral(table.Table.RowType):
	__slots__ = tuple(SnglInspiralTable.validcolumns.keys())

	def chirp_distance(dist, mchirp, ref_mass=1.4):
		return dist * (2.**(-1./5) * ref_mass / mchirp)**(5./6)

	#
	# Properties
	#

	end = gpsproperty("end_time", "end_time_ns")

	@property
	def spin1(self):
		if self.spin1x is None and self.spin1y is None and self.spin1z is None:
		return numpy.array((self.spin1x, self.spin1y, self.spin1z), dtype = "double")

	@spin1.setter
	def spin1(self, spin):
		if spin is None:
			self.spin1x, self.spin1y, self.spin1z = None, None, None
		else:
			self.spin1x, self.spin1y, self.spin1z = spin

	@property
	def spin2(self):
		if self.spin2x is None and self.spin2y is None and self.spin2z is None:
		return numpy.array((self.spin2x, self.spin2y, self.spin2z), dtype = "double")

	@spin2.setter
	def spin2(self, spin):
			self.spin2x, self.spin2y, self.spin2z = None, None, None
		else:
			self.spin2x, self.spin2y, self.spin2z = spin

	#
	# simulate tempate_id column
	# FIXME:  add a proper column for this
	#

	@property
	def template_id(self):
		return int(self.Gamma0)

	@template_id.setter
	def template_id(self, template_id):
		self.Gamma0 = float(template_id)

	#
	# Methods
	#

		return float(self.chisq)/ (2*self.chisq_dof - 2)

		return float(self.bank_chisq)/ self.bank_chisq_dof

		return float(self.cont_chisq)/ self.cont_chisq_dof

	def get_effective_snr(self,fac=250.0):
		return self.snr/ (1 + self.snr**2/fac)**(0.25)/ self.get_reduced_chisq()**0.25
	
	def get_bank_effective_snr(self,fac=250.0):
		return self.snr/ (1 + self.snr**2/fac)**(0.25)/ self.get_reduced_bank_chisq()**0.25

	def get_cont_effective_snr(self,fac=250.0):
		return self.snr/ (1 + self.snr**2/fac)**(0.25)/ self.get_reduced_cont_chisq()**0.25

	def get_new_snr(self,index=6.0):
		rchisq = self.get_reduced_chisq()
		nhigh = 2.
		if rchisq > 1.:
			return self.snr/ ((1+rchisq**(index/nhigh))/2)**(1./index)
		else:
			return self.snr

	def get_bank_new_snr(self,index=6.0):
		rchisq = self.get_reduced_bank_chisq()
		nhigh = 2.
		else:

	def get_cont_new_snr(self,index=6.0):
		rchisq = self.get_reduced_cont_chisq()
		nhigh = 2.
		else:

	def get_far(self):
		return self.alpha

	def get_ifar(self):
		if self.alpha < 0.000000001:
			self.alpha = 0.000000001
		return 1./self.alpha

		return self.beta

	# FIXME: how are two inspiral events defined to be the same?
	def __eq__(self, other):
		return not (
			cmp(self.ifo, other.ifo) or
			cmp(self.end, other.end) or
			cmp(self.mass1, other.mass1) or
			cmp(self.mass2, other.mass2) or
			cmp(self.spin1, other.spin1) or
			cmp(self.spin2, other.spin2) or
			cmp(self.search, other.search)
		)

	#
	# Legacy
	#

		return self.end

	def set_end(self, gps):
		self.end = gps

SnglInspiralTable.RowType = SnglInspiral

#
#
#                             coinc_inspiral:table
#
#

class CoincInspiralTable(table.Table):
	tableName = "coinc_inspiral"
		"mass": "real_8",
		"mchirp": "real_8",
		"minimum_duration": "real_8",
		"snr": "real_8",
		"false_alarm_rate": "real_8",
		"combined_far": "real_8"
	}
		"ci_cei_index": ("coinc_event_id",)
	}
	interncolumns = ("coinc_event_id", "ifos")

class CoincInspiral(table.Table.RowType):
	"""
	Example:

	>>> x = CoincInspiral()
	>>> x.end = LIGOTimeGPS(10)
	>>> x.end
	LIGOTimeGPS(10, 0)
	>>> x.end = None
	>>> print(x.end)
	None
	"""
	__slots__ = tuple(CoincInspiralTable.validcolumns.keys())

	def set_ifos(self, ifos):
		self.instruments = ifos

CoincInspiralTable.RowType = CoincInspiral

#
#
#                             sngl_ringdown:table
#
#

SnglRingdownID = ilwd.get_ilwdchar_class(u"sngl_ringdown", u"event_id")

class SnglRingdownTable(table.Table):
	tableName = "sngl_ringdown"
		"start_time_gmst": "real_8",
		"quality": "real_4",
		"phase": "real_4",
		"mass": "real_4",
		"spin": "real_4",
		"epsilon": "real_4",
		"num_clust_trigs": "int_4s",
		"ds2_H1H2": "real_4",
		"ds2_H1L1": "real_4",
		"ds2_H1V1": "real_4",
		"ds2_H2L1": "real_4",
		"ds2_H2V1": "real_4",
		"ds2_L1V1": "real_4",
		"eff_dist": "real_4",
		"sigma_sq": "real_8",
	}
	next_id = SnglRingdownID(0)
	interncolumns = ("process_id", "ifo", "channel")

		return [row.get_start() for row in self]

class SnglRingdown(table.Table.RowType):
	__slots__ = tuple(SnglRingdownTable.validcolumns.keys())

	def get_id_parts(self):
		"""
		Return the three pieces of the int_8s-style event_id.
		"""
		int_event_id = int(self.event_id)
		a = int_event_id // 1000000000
		slidenum = (int_event_id % 1000000000) // 100000
		b = int_event_id % 100000
		return int(a), int(slidenum), int(b)

SnglRingdownTable.RowType = SnglRingdown

#
#
#                             coinc_ringdown:table
#
#

class CoincRingdownTable(table.Table):
	tableName = "coinc_ringdown"
		"frequency": "real_8",
		"quality": "real_8",
		"spin": "real_8",
		"choppedl_snr": "real_8",
		"snr_sq": "real_8",
		"eff_coh_snr": "real_8",
		"null_stat": "real_8",
		"kappa": "real_8",
		"snr_ratio": "real_8",
	}
	# constraints = "PRIMARY KEY (coinc_event_id)"
		"cr_cei_index": ("coinc_event_id",)
	}

class CoincRingdown(table.Table.RowType):
	__slots__ = tuple(CoincRingdownTable.validcolumns.keys())

		self.ifos = ifos_from_instrument_set(ifos)

		return instrument_set_from_ifos(self.ifos)

CoincRingdownTable.RowType = CoincRingdown

#
#
#                             multi_inspiral:table
#
#

MultiInspiralID = ilwd.get_ilwdchar_class(u"multi_inspiral", u"event_id")

class MultiInspiralTable(table.Table):
	tableName = "multi_inspiral"
		"distance": "real_4",
		"eff_dist_h1": "real_4",
		"eff_dist_h2": "real_4",
		"eff_dist_l": "real_4",
		"eff_dist_g": "real_4",
		"eff_dist_t": "real_4",
		"eff_dist_v": "real_4",
		"eff_dist_h1h2": "real_4",
		"snr_dof": "int_4s",
		"trace_snr": "real_4",
		"snr_h1": "real_4",
		"snr_h2": "real_4",
		"snr_l": "real_4",
		"snr_g": "real_4",
		"snr_t": "real_4",
		"snr_v": "real_4",
		"amp_term_1": "real_4",
		"amp_term_2": "real_4",
		"amp_term_3": "real_4",
		"amp_term_4": "real_4",
		"amp_term_5": "real_4",
		"amp_term_6": "real_4",
		"amp_term_7": "real_4",
		"amp_term_8": "real_4",
		"amp_term_9": "real_4",
		"amp_term_10": "real_4",
		"sigmasq_h1": "real_8",
		"sigmasq_h2": "real_8",
		"sigmasq_l": "real_8",
		"sigmasq_g": "real_8",
		"sigmasq_t": "real_8",
		"sigmasq_v": "real_8",
		"chisq_h1": "real_4",
		"chisq_h2": "real_4",
		"chisq_l": "real_4",
		"chisq_g": "real_4",
		"chisq_t": "real_4",
		"chisq_v": "real_4",
		"sngl_chisq_dof": "int_4s",
		"bank_chisq_h1": "real_4",
		"bank_chisq_h2": "real_4",
		"bank_chisq_l": "real_4",
		"bank_chisq_g": "real_4",
		"bank_chisq_t": "real_4",
		"bank_chisq_v": "real_4",
		"sngl_bank_chisq_dof": "int_4s",
		"cont_chisq_h1": "real_4",
		"cont_chisq_h2": "real_4",
		"cont_chisq_l": "real_4",
		"cont_chisq_g": "real_4",
		"cont_chisq_t": "real_4",
		"cont_chisq_v": "real_4",
		"sngl_cont_chisq_dof": "int_4s",
		"ra": "real_4",
		"dec": "real_4",
		"inclination": "real_4",
		"polarization": "real_4",
		"null_statistic": "real_4",
		"null_stat_h1h2": "real_4",
		"null_stat_degen": "real_4",
		"event_id": "ilwd:char",
		"h1quad_re": "real_4",
		"h1quad_im": "real_4",
		"h2quad_re": "real_4",
		"h2quad_im": "real_4",
		"l1quad_re": "real_4",
		"l1quad_im": "real_4",
		"g1quad_re": "real_4",
		"g1quad_im": "real_4",
		"t1quad_re": "real_4",
		"t1quad_im": "real_4",
		"v1quad_re": "real_4",
		"v1quad_im": "real_4",
		"coh_snr_h1h2": "real_4",
		"cohSnrSqLocal": "real_4",
		"autoCorrCohSq": "real_4",
		"crossCorrCohSq": "real_4",
		"autoCorrNullSq": "real_4",
		"crossCorrNullSq": "real_4",
		"ampMetricEigenVal1": "real_8",
		"ampMetricEigenVal2": "real_8",
		"time_slide_id": "ilwd:char"
	}
	next_id = MultiInspiralID(0)
	interncolumns = ("process_id", "ifos", "search")
	instrument_id = {"G1":"g", "H1":"h1", "H2":"h2", "L1":"l", "T1":"t",
	                 "V1":"v"}

	def get_column(self,column):
		if column == 'bestnr':
			return self.get_bestnr()
		elif column == 'new_snr':
			return self.get_new_snr()
		elif column == "null_snr":
			return self.get_null_snr()
		elif column == 'coinc_snr':
			return self.get_coinc_snr()
		elif column == 'reduced_chisq':
		elif column == 'reduced_bank_chisq':
		elif column == 'reduced_cont_chisq':
		elif column == "reduced_chisq_h1":
			return self.get_reduced_sngl_chisq(ifo="hl")
		elif column == "reduced_chisq_h2":
			return self.get_reduced_sngl_chisq(ifo="h2")
		elif column == "reduced_chisq_l":
			return self.get_reduced_sngl_chisq(ifo="l")
		elif column == "reduced_chisq_v":
			return self.get_reduced_sngl_chisq(ifo="v")
		else:

	def get_coinc_snr(self):
		if len(self):
			return (numpy.asarray(list(self.get_sngl_snrs().values()))\
			            **2).sum(axis=0)**(1./2.)
		else:
			return numpy.array([])

	def get_new_snr(self, index=4.0,nhigh = 3.0, column='chisq'):
		# nhigh gives the asymptotic large rho behaviour of
		# d (ln chisq) / d (ln rho) 
		rchisq = self.get_column('reduced_%s' % column)

	def get_null_snr(self):
		"""
		Get the coherent Null SNR for each row in the table.
		"""
		null_snr_sq = self.get_coinc_snr()**2 - self.get_column('snr')**2
		null_snr_sq[null_snr_sq < 0] = 0.
		return null_snr_sq**(1./2.)

	def get_coinc_chisq(self):
		"""@returns the coincident chisq for each row in the table
		"""
			return (numpy.asarray(list(self.get_sngl_chisqs().values()))\
						**2).sum(axis=0)**(1./2.)
		else:

	def get_reduced_coinc_chisq(self):
		"""@returns the coincident chisq per degree of freedom for each
		row in the table
		"""
		dof = float(len(self) and self[0].sngl_chisq_dof or 1)
		return self.get_coinc_chisq()/dof

	def get_null_chisq(self):
		"""@returns the coherent null chisq for each row in the table
		"""
		null_chisq_sq = (self.get_reduced_coinc_chisq() -
		                 self.get_reduced_chisq())
		null_chisq_sq[null_chisq_sq < 0] = 0.
		return null_chisq_sq

	def get_sigmasq(self, instrument):
		"""
		Get the single-detector SNR of the given instrument for each
		row in the table.
		"""
		return self.get_column('sigmasq_%s'
		                       % self.instrument_id[instrument.upper()])

	def get_sigmasqs(self, instruments=None):
		"""
		Return dictionary of single-detector sigmas for each row in the
		table.
		"""
			if not instruments:
				instruments = list(map(str, \
					instrument_set_from_ifos(self[0].ifos)))
			return dict((ifo, self.get_sigmasq(ifo))\
				    for ifo in instruments)
		else:
			return dict()

	def get_sngl_snr(self, instrument):
		"""
		"""
		return self.get_column('snr_%s'\

	def get_sngl_snrs(self, instruments=None):
		"""
		Get the single-detector SNRs for each row in the table.
		"""
		if len(self) and instruments is None:
			instruments = list(map(str, \
			                instrument_set_from_ifos(self[0].ifos)))
		elif instruments is None:
			instruments = []
		return dict((ifo, self.get_sngl_snr(ifo))\
		            for ifo in instruments)

	def get_sngl_chisq(self, instrument):
		"""
		Get the single-detector \chi^2 of the given instrument for each
		"""
		return self.get_column('chisq_%s'\

	def get_sngl_chisqs(self, instruments=None):
		"""
		Get the single-detector \chi^2 for each row in the table.
		"""
		return dict((ifo, self.get_sngl_chisq(ifo))\

	def get_sngl_bank_chisq(self, instrument):
		"""
		"""
		return self.get_column('bank_chisq_%s'\

	def get_sngl_bank_chisqs(self, instruments=None):
		"""
		"""
		return dict((ifo, self.get_sngl_bank_chisq(ifo))\

	def get_sngl_cont_chisq(self, instrument):
		"""
		"""
		return self.get_column('cont_chisq_%s'\

	def get_sngl_cont_chisqs(self, instruments=None):
		"""
		"""
		return dict((ifo, self.get_sngl_cont_chisq(ifo))\

	def get_bestnr(self, index=4.0, nhigh=3.0, null_snr_threshold=4.25,\
		           null_grad_thresh=20., null_grad_val = 1./5.):
		"""
		Get the BestNR statistic for each row in the table
		"""
		return [row.get_bestnr(index=index, nhigh=nhigh,
		                       null_snr_threshold=null_snr_threshold,
		                       null_grad_thresh=null_grad_thresh,
		                       null_grad_val=null_grad_val)
		        for row in self]

	def getstat(self):
		return self.get_column('snr')

			else:

		"""
		"""
			else:

		"""
		"""

		"""@returns the chisq per degree of freedom for each row in
		this table
		"""
		return self.get_column('chisq') / self.get_column('chisq_dof')

		"""@returns the bank chisq per degree of freedom for each row in
		this table
		"""

		"""@returns the auto (continuous) chisq per degree of freedom
		for each row in this table
		"""

	def get_reduced_sngl_chisq(self, instrument):
		"""@returns the single-detector chisq per degree of freedom for
		each row in this table
		"""
		return (self.get_column("chisq_%s"
		                        % self.instrument_id[instrument.upper()]) /
		        self.get_column("sngl_chisq_dof"))

	def get_sngl_new_snr(self, ifo, column="chisq", index=4.0, nhigh = 3.0):
		column = column.lower()
		if column == "chisq":
			rchisq = self.get_reduced_sngl_chisq(ifo)
		elif column == "bank_chisq":
			rchisq = self.get_reduced_sngl_bank_chisq(ifo)
		elif column == "cont_chisq":
			rchisq = self.get_reduced_cont_chisq()
		else:
			rchisq = getattr(self, column) / getattr(self, "%s_dof" % column)

class MultiInspiral(table.Table.RowType):
	__slots__ = tuple(MultiInspiralTable.validcolumns.keys())
	instrument_id = MultiInspiralTable.instrument_id

		return self.chisq / self.chisq_dof

		return self.bank_chisq / self.bank_chisq_dof

		return self.cont_chisq / self.cont_chisq_dof

		return (getattr(self,
		                "chisq_%s" % self.instrument_id[instrument.upper()]) /
		        getattr(self, "sngl_chisq_dof"))

	def get_reduced_sngl_bank_chisq(self, instrument):
		return (getattr(self, ("bank_chisq_%s"
		                       % self.instrument_id[instrument.upper()])) /
				getattr(self, "sngl_bank_chisq_dof"))

	def get_reduced_sngl_cont_chisq(self, instrument):
		return (getattr(self, ("cont_chisq_%s"

		return LIGOTimeGPS(self.end_time, self.end_time_ns)

		"""
		"""

		"""
		Get the coincident SNR for this row.
		"""
		return (numpy.asarray(list(self.get_sngl_snrs().values()))**2)\
		            .sum()**(1./2.)

		"""@returns the coincident chisq for this row
		"""
		return ((numpy.asarray(list(self.get_sngl_chisqs().values()))**2)
		            .sum()**(1./2.))

		"""@returns the coincident chisq per degree of freedom for this row
		"""
		return self.get_coinc_chisq()/self.sngl_chisq_dof

	def get_new_snr(self, index=4.0, nhigh = 3.0, column='chisq'):
			rchisq = self.get_reduced_chisq()
			rchisq = self.get_reduced_bank_chisq()
		else:
			return self.snr /\
                               ((1+rchisq**(index/nhigh))/2)**(1./index)
		else:

			rchisq = self.get_reduced_sngl_cont_chisq(ifo)
			return self.get_sngl_snr(ifo) /\
							   ((1+rchisq**(index/nhigh))/2)**(1./index)
		else:

	def get_sngl_new_snrs(self, column="chisq", index=4.0, nhigh = 3.0):
		"""@returns a dictionary of single-detector newSNRs for this row.
		"""
		return dict((ifo,
		             self.get_sngl_new_snr(ifo, column=column, index=index, nhigh=nhigh)) for
		            ifo in instrument_set_from_ifos(self.ifos))

		"""
		Get the coherent Null SNR for this row.
		"""
		null_snr_sq = (numpy.asarray(list(self.get_sngl_snrs().values()))**2)\
                             .sum() - self.snr**2
		if null_snr_sq < 0:
			return 0
		else:
			return null_snr_sq**(1./2.)

		"""@returns the coherent null chisq for this row
		"""
		if null_chisq_sq < 0:
			return 0
		else:
			return null_chisq_sq

		"""
		Get the single-detector SNR for the given instrument for this
		row
		"""
		return getattr(self, "snr_%s" % self.instrument_id[instrument.upper()])

	def get_sngl_snrs(self):
		"""
		Return a dictionary of single-detector SNRs for this row.
		"""
		return dict((ifo, self.get_sngl_snr(ifo)) for ifo in\
                            instrument_set_from_ifos(self.ifos))

		"""@returns the single-detector chisq for the given instrument
		for this row
		"""
		return getattr(self,
		               "chisq_%s" % self.instrument_id[instrument.upper()])

	def get_sngl_chisqs(self):
		"""@returns a dictionary of single-detector chisqs for this row.
		"""
		return dict((ifo, self.get_sngl_chisq(ifo)) for ifo in

		"""
		"""
		self.ifos = ifos_from_instrument_set(instruments)

		"""
		"""

	def get_slide_number(self):
		"""
		Return the slide-number for this trigger
		"""
		a, slide_number, b = self.get_id_parts()
		if slide_number > 5000:
			slide_number = 5000 - slide_number
		return slide_number

		"""
		Return the BestNR statistic for this row.
		"""
		# weight SNR by chisq
		bestnr = self.get_new_snr(index=index, nhigh=nhigh,
		                          column="chisq")
		if len(self.get_ifos()) < 3:
			return bestnr
		# recontour null SNR threshold for higher SNRs
		if self.snr > null_grad_thresh:
			null_snr_threshold += (self.snr - null_grad_thresh) * null_grad_val
		# weight SNR by null SNR
		if self.get_null_snr() > null_snr_threshold:
			bestnr /= 1 + self.get_null_snr() - null_snr_threshold
		return bestnr

MultiInspiralTable.RowType = MultiInspiral

#
#
#                              sim_inspiral:table
#
#

SimInspiralID = ilwd.get_ilwdchar_class(u"sim_inspiral", u"simulation_id")

class SimInspiralTable(table.Table):
	tableName = "sim_inspiral"
		"waveform": "lstring",
		"geocent_end_time": "int_4s",
		"geocent_end_time_ns": "int_4s",
		"h_end_time": "int_4s",
		"h_end_time_ns": "int_4s",
		"l_end_time": "int_4s",
		"l_end_time_ns": "int_4s",
		"g_end_time": "int_4s",
		"g_end_time_ns": "int_4s",
		"t_end_time": "int_4s",
		"t_end_time_ns": "int_4s",
		"v_end_time": "int_4s",
		"v_end_time_ns": "int_4s",
		"source": "lstring",
		"longitude": "real_4",
		"latitude": "real_4",
		"theta0": "real_4",
		"phi0": "real_4",
		"f_lower": "real_4",
		"eff_dist_h": "real_4",
		"numrel_mode_min": "int_4s",
		"numrel_mode_max": "int_4s",
		"numrel_data": "lstring",
		"amp_order": "int_4s",
		"taper": "lstring",
		"bandpass": "int_4s",
		"simulation_id": "ilwd:char"
	}
	constraints = "PRIMARY KEY (simulation_id)"
	next_id = SimInspiralID(0)
	interncolumns = ("process_id", "waveform", "source")

		if column == 'chirp_dist' or column == 'chirp_distance':
			return self.get_chirp_dist()
		# be strict about formatting of chirp eff distance
		if column[0:14] == 'chirp_eff_dist' and column[14:16] in ['_h','_l','_g','_t','_v'] and len(column) == 16:
			site = column[-1]
			return self.get_chirp_eff_dist(site)
		elif column == 'spin1':
			return self.get_spin_mag(1)
		elif column == 'spin2':
			return self.get_spin_mag(2)
		elif column == 'total_mass' or column == 'mtotal':
			m1=self.getColumnByName('mass1').asarray()
			m2=self.getColumnByName('mass2').asarray()
			return m1+m2
		else:

	def get_chirp_dist(self,ref_mass=1.4):
		dist = self.get_column('distance')
		return SnglInspiral.chirp_distance(dist, mchirp, ref_mass)

	def get_chirp_eff_dist(self,site,ref_mass=1.4):
		eff_dist = self.get_column('eff_dist_' + site)

	def get_spin_mag(self,objectnumber):
		sx = self.get_column('spin' + str(objectnumber) + 'x')
		sy = self.get_column('spin' + str(objectnumber) + 'y')
		sz = self.get_column('spin' + str(objectnumber) + 'z')
		return (sx**2 + sy**2 + sz**2)**(0.5)

	def veto(self,seglist,site=None):
		keep.extend(row for row in self if row.get_end(site) not in seglist)

			else:

		"""
		"""
			else:

	def get_end(self, site=None):
		return numpy.asarray([row.get_end(site=site) for row in self])

class SimInspiral(table.Table.RowType):
	"""
	Example:

	>>> x = SimInspiral()
	>>> x.ra_dec = 0., 0.
	>>> x.ra_dec
	(0.0, 0.0)
	>>> x.ra_dec = None
	>>> print(x.ra_dec)
	None
	>>> x.time_geocent = None
	>>> print(x.time_geocent)
	None
	>>> print(x.end_time_gmst)
	None
	>>> x.time_geocent = LIGOTimeGPS(6e8)
	600000000
	>>> print(numpy.around(x.end_time_gmst, 8))
	-2238.39417156
	"""
	__slots__ = tuple(SimInspiralTable.validcolumns.keys())

	time_geocent = gpsproperty_with_gmst("geocent_end_time", "geocent_end_time_ns", "end_time_gmst")

	@property
	def ra_dec(self):
		if self.longitude is None and self.latitude is None:
		return self.longitude, self.latitude

	@ra_dec.setter
	def ra_dec(self, radec):
		if radec is None:
			self.longitude = self.latitude = None
		else:
			self.longitude, self.latitude = radec

	@property

		else:

	@property

		else:

	def time_at_instrument(self, instrument, offsetvect):
		"""
		Return the "time" of the injection, delay corrected for the
		displacement from the geocentre to the given instrument.

		NOTE:  this method does not account for the rotation of the
		Earth that occurs during the transit of the plane wave from
		the detector to the geocentre.  That is, it is assumed the
		Earth is in the same orientation with respect to the
		celestial sphere when the wave passes through the detector
		as when it passes through the geocentre.  The Earth rotates
		by about 1.5 urad during the 21 ms it takes light to travel
		the radius of the Earth, which corresponds to 10 m of
		displacement at the equator, or 33 light-ns.  Therefore,
		the failure to do a proper retarded time calculation here
		results in errors as large as 33 ns.  This is insignificant
		for burst searches, but be aware that this approximation is
		being made if the return value is used in other contexts.
		"""
		# the offset is subtracted from the time of the injection.
		# injections are done this way so that when the triggers
		# that result from an injection have the offset vector
		# added to their times the triggers will form a coinc
		t_geocent = self.time_geocent - offsetvect[instrument]
		ra, dec = self.ra_dec
		return t_geocent + lal.TimeDelayFromEarthCenter(lal.cached_detector_by_prefix[instrument].location, ra, dec, t_geocent)

	def get_time_geocent(self):
		# FIXME:  delete this method
		warnings.warn("SimInspiral.get_time_geocent() is deprecated.  use SimInspiral.time_geocent instead", DeprecationWarning)
		return self.time_geocent

	def set_time_geocent(self, gps):
		warnings.warn("SimInspiral.set_time_geocent() is deprecated.  use SimInspiral.time_geocent instead", DeprecationWarning)
		self.time_geocent = gps

	def get_ra_dec(self):
		warnings.warn("SimInspiral.get_ra_dec() is deprecated.  use SimInspiral.ra_dec instead", DeprecationWarning)
		return self.ra_dec

	def get_end(self, site = None):
		warnings.warn("SimInspiral.get_end() is deprecated.  use SimInspiral.time_geocent or SimInspiral.time_at_instrument() instead", DeprecationWarning)
		if site is None:
			return self.time_geocent
		else:
			return LIGOTimeGPS(getattr(self, "%s_end_time" % site.lower()), getattr(self, "%s_end_time_ns" % site.lower()))

	def get_eff_dist(self, instrument):
		return getattr(self, "eff_dist_%s" % instrument[0].lower())

	def get_chirp_eff_dist(self, instrument, ref_mass = 1.4):
		return SnglInspiral.chirp_distance(self.get_eff_dist(instrument), self.mchirp, ref_mass)

	def get_spin_mag(self, objectnumber):
		s = getattr(self, "spin%d" % objectnumber)
		return math.sqrt(numpy.dot(s, s))

SimInspiralTable.RowType = SimInspiral

#
#
#                               sim_burst:table
#
#

SimBurstID = ilwd.get_ilwdchar_class(u"sim_burst", u"simulation_id")

class SimBurstTable(table.Table):
	tableName = "sim_burst"
		"ra": "real_8",
		"dec": "real_8",
		"psi": "real_8",
		"time_geocent_gps": "int_4s",
		"time_geocent_gps_ns": "int_4s",
		"time_geocent_gmst": "real_8",
		"duration": "real_8",
		"bandwidth": "real_8",
		"q": "real_8",
		"pol_ellipse_angle": "real_8",
		"pol_ellipse_e": "real_8",
		"amplitude": "real_8",
		"hrss": "real_8",
		"egw_over_rsquared": "real_8",
		"waveform_number": "int_8u",
	}
	next_id = SimBurstID(0)
	interncolumns = ("process_id", "waveform")

class SimBurst(table.Table.RowType):
	"""
	Example:

	>>> x = SimBurst()
	(0.0, 0.0)
	None
	None
	>>> print(x.time_geocent_gmst)
	None
	600000000
	>>> print(numpy.around(x.time_geocent_gmst, 8))
	"""
	__slots__ = tuple(SimBurstTable.validcolumns.keys())

	time_geocent = gpsproperty_with_gmst("time_geocent_gps", "time_geocent_gps_ns", "time_geocent_gmst")

	@property
		if self.ra is None and self.dec is None:
		return self.ra, self.dec

			self.ra = self.dec = None
		else:
			self.ra, self.dec = radec

		"""

		"""

		"""
		Do not use this method:  use .time_at_instrument() if that's what you want, or use .time_geocent if that's what you want.

		Also ... this doesn't return the *end time*, it returns the *PEAK TIME*.  You've been warned.
		"""
		instrument = site + "1"	# ugh ...
		return self.time_at_instrument(instrument, {instrument: 0.0})

SimBurstTable.RowType = SimBurst

#
#
#                              sim_ringdown:table
#
#

SimRingdownID = ilwd.get_ilwdchar_class(u"sim_ringdown", u"simulation_id")

class SimRingdownTable(table.Table):
	tableName = "sim_ringdown"
		"coordinates": "lstring",
		"geocent_start_time": "int_4s",
		"geocent_start_time_ns": "int_4s",
		"h_start_time": "int_4s",
		"h_start_time_ns": "int_4s",
		"l_start_time": "int_4s",
		"l_start_time_ns": "int_4s",
		"v_start_time": "int_4s",
		"v_start_time_ns": "int_4s",
		"hrss_h": "real_4",
		"hrss_l": "real_4",
		"hrss_v": "real_4",
	}
	next_id = SimRingdownID(0)
	interncolumns = ("process_id", "waveform", "coordinates")

class SimRingdown(table.Table.RowType):
	__slots__ = tuple(SimRingdownTable.validcolumns.keys())

	def get_start(self, site = None):
		if not site:
			return LIGOTimeGPS(self.geocent_start_time, self.geocent_start_time_ns)
		else:
			site = site[0].lower()
			return LIGOTimeGPS(getattr(self, site + '_start_time'), getattr(self, site + '_start_time_ns'))

SimRingdownTable.RowType = SimRingdown

#
#
#                               summ_value:table
#
#

SummValueID = ilwd.get_ilwdchar_class(u"summ_value", u"summ_value_id")

class SummValueTable(table.Table):
	tableName = "summ_value"
		"summ_value_id": "ilwd:char",
		"frameset_group": "lstring",
		"segment_def_id": "ilwd:char",
		"value": "real_4",
		"error": "real_4",
		"intvalue": "int_4s",
		"comment": "lstring"
	}
	constraints = "PRIMARY KEY (summ_value_id)"
	next_id = SummValueID(0)
	interncolumns = ("program", "process_id", "ifo", "name", "comment")

class SummValue(table.Table.RowType):
	"""
	Example:

	>>> x = SummValue()
	>>> assert x.ifo == 'H1,L1'
	>>> x.start = LIGOTimeGPS(0)
	>>> x.segment
	>>> x.segment = None
	>>> print(x.segment)
	None
	"""
	__slots__ = tuple(SummValueTable.validcolumns.keys())

	instruments = instrumentsproperty("ifo")

	segment = segmentproperty("start", "end")

SummValueTable.RowType = SummValue

#
#
#                            sim_inst_params:table
#
#

SimInstParamsID = ilwd.get_ilwdchar_class(u"sim_inst_params", u"simulation_id")

class SimInstParamsTable(table.Table):
	tableName = "sim_inst_params"
		"simulation_id": "ilwd:char",
	}
	next_id = SimInstParamsID(0)

class SimInstParams(table.Table.RowType):
	__slots__ = tuple(SimInstParamsTable.validcolumns.keys())

SimInstParamsTable.RowType = SimInstParams

#
#
#                               stochastic:table
#
#

class StochasticTable(table.Table):
	tableName = "stochastic"
		"ifo_one": "lstring",
		"ifo_two": "lstring",
		"channel_one": "lstring",
		"channel_two": "lstring",
		"duration_ns": "int_4s",
		"f_min": "real_8",
		"f_max": "real_8",
		"cc_stat": "real_8",
		"cc_sigma": "real_8"
	}

class Stochastic(table.Table.RowType):
	__slots__ = tuple(StochasticTable.validcolumns.keys())

StochasticTable.RowType = Stochastic

#
#
#                               stochsumm:table
#
#

class StochSummTable(table.Table):
	tableName = "stochsumm"
		"y_opt": "real_8",
		"error": "real_8"
	}

class StochSumm(table.Table.RowType):
	__slots__ = tuple(StochSummTable.validcolumns.keys())

StochSummTable.RowType = StochSumm

#
#
#                            external_trigger:table
#
#

# FIXME: this table is completely different from the official definition.
# Someone *HAS* to sort this out.

class ExtTriggersTable(table.Table):
	tableName = "external_trigger"
		"det_alts": "lstring",
		"det_band": "lstring",
		"det_fluence": "lstring",
		"det_fluence_int": "lstring",
		"det_name": "lstring",
		"det_peak": "lstring",
		"det_peak_int": "lstring",
		"det_snr": "lstring",
		"email_time": "int_4s",
		"event_dec": "real_4",
		"event_dec_err": "real_4",
		"event_epoch": "lstring",
		"event_err_type": "lstring",
		"event_ra": "real_4",
		"event_ra_err": "real_4",
		"event_type": "lstring",
		"event_z": "real_4",
		"event_z_err": "real_4",
		"notice_comments": "lstring",
		"notice_id": "lstring",
		"notice_sequence": "lstring",
		"notice_time": "int_4s",
		"notice_type": "lstring",
		"notice_url": "lstring",
		"obs_fov_dec": "real_4",
		"obs_fov_dec_width": "real_4",
		"obs_fov_ra": "real_4",
		"obs_fov_ra_width": "real_4",
		"obs_loc_ele": "real_4",
		"obs_loc_lat": "real_4",
		"obs_loc_long": "real_4",
		"ligo_fave_lho": "real_4",
		"ligo_fave_llo": "real_4",
		"ligo_delay": "real_4",
		"event_number_gcn": "int_4s",
		"event_number_grb": "lstring",
		"event_status": "int_4s"
	}

class ExtTriggers(table.Table.RowType):
	__slots__ = tuple(ExtTriggersTable.validcolumns.keys())

ExtTriggersTable.RowType = ExtTriggers

#
#
#                                 filter:table
#
#

FilterID = ilwd.get_ilwdchar_class(u"filter", u"filter_id")

class FilterTable(table.Table):
	tableName = "filter"
		"filter_name": "lstring",
		"param_set": "int_4s",
	}
	constraints = "PRIMARY KEY (filter_id)"
	next_id = FilterID(0)

class Filter(table.Table.RowType):
	__slots__ = tuple(FilterTable.validcolumns.keys())

FilterTable.RowType = Filter

#
#
#                                segment:table
#
#

SegmentID = ilwd.get_ilwdchar_class(u"segment", u"segment_id")

class SegmentTable(table.Table):
	tableName = "segment"
		"segment_id": "ilwd:char",
		"segment_def_cdb": "int_4s"
	}
	constraints = "PRIMARY KEY (segment_id)"
	next_id = SegmentID(0)
	interncolumns = ("process_id",)

@total_ordering
class Segment(table.Table.RowType):
	"""
	Example:

	>>> x = Segment()
	None
	>>> print(x.start)
	None
	>>> # non-LIGOTimeGPS times are converted to LIGOTimeGPS
	>>> x.segment = (20, 30.125)
	>>> x.end
	LIGOTimeGPS(30, 125000000)
	>>> # initialization from a tuple or with arguments
	>>> Segment((20, 30)).segment
	segment(LIGOTimeGPS(20, 0), LIGOTimeGPS(30, 0))
	>>> Segment(20, 30).segment
	>>> # use as a segment object in segmentlist operations
	>>> from ligo import segments
	>>> x = segments.segmentlist([Segment(0, 10), Segment(20, 30)])
	>>> abs(x)
	LIGOTimeGPS(20, 0)
	>>> y = segments.segmentlist([Segment(5, 15), Segment(25, 35)])
	>>> abs(x & y)
	>>> abs(x | y)
	LIGOTimeGPS(30, 0)
	>>> 8.0 in x
	True
	>>> 12 in x
	False
	>>> Segment(2, 3) in x
	True
	>>> Segment(2, 12) in x
	False
	>>> segments.segment(2, 3) in x
	True
	>>> segments.segment(2, 12) in x
	False
	>>> # make sure results are segment table row objects
	>>> segments.segmentlist(map(Segment, x & y))	# doctest: +ELLIPSIS
	[<glue.ligolw.lsctables.Segment object at 0x...>, <glue.ligolw.lsctables.Segment object at 0x...>]

	This implementation uses a non-standard extension to encode
	infinite values for boundaries:  the second and nanosecond
	components are both set to 0x7FFFFFFF or 0xFFFFFFFF to indicate
	positive resp. negative infinity.  For this reason, "denormalized"
	LIGOTimeGPS objects (objects whose nanoseconds fields contain
	values exceeding +/-999999999) are disallowed for use with this
	class.

	Example:

	>>> # OK
	>>> x.start = -segments.infinity()
	>>> # also OK
	>>> x.start = float("-inf")
	>>> # infinite boundaries always returned as segments.infinity
	>>> # instances
	>>> x.start
	-infinity
	>>> x.end = float("+inf")
	segment(-infinity, infinity)
	"""
	__slots__ = tuple(SegmentTable.validcolumns.keys())

	def get(self):
		"""
		Return the segment described by this row.
		"""
		return self.segment

	def set(self, segment):
		"""
		Set the segment described by this row.
		"""
		self.segment = segment

	# emulate a ligo.segments.segment object

	def __abs__(self):
		return abs(self.segment)

	def __lt__(self, other):
		return self.segment < other

		return self.segment == other

	def __contains__(self, other):
		return other in self.segment

		return self.segment[i]

	def __init__(self, *args, **kwargs):
		if kwargs:
			super(Segment, self).__init__(**kwargs)
		if args:
			try:
				# first try unpacking arguments
				self.segment = args
			except ValueError:
				# didn't work, try unpacking 0th argument
				self.segment, = args

		return len(self.segment)

	def __nonzero__(self):
		return bool(self.segment)

SegmentTable.RowType = Segment

#
#
#                            segment_definer:table
#
#

SegmentDefID = ilwd.get_ilwdchar_class(u"segment_definer", u"segment_def_id")

class SegmentDefTable(table.Table):
	tableName = "segment_definer"
		"version": "int_4s",
		"insertion_time": "int_4s"
	}
	constraints = "PRIMARY KEY (segment_def_id)"
	next_id = SegmentDefID(0)

class SegmentDef(table.Table.RowType):
	"""
	Example:

	>>> x = SegmentDef()
	>>> assert x.instruments =={'H1', 'L1'}
	"""
	__slots__ = tuple(SegmentDefTable.validcolumns.keys())

		"""
		"""

		"""
		"""

SegmentDefTable.RowType = SegmentDef

#
#
#                            segment_summary:table
#
#

SegmentSumID = ilwd.get_ilwdchar_class(u"segment_summary", u"segment_sum_id")

class SegmentSumTable(table.Table):
	tableName = "segment_summary"
		"segment_sum_id": "ilwd:char",
	}
	constraints = "PRIMARY KEY (segment_sum_id)"
	next_id = SegmentSumID(0)
	interncolumns = ("process_id","segment_def_id")

	def get(self, segment_def_id = None):
		"""
		the segments carrying the given segment_def_id.  If
		segment_def_id is None then all segments are returned.

		"""
		if segment_def_id is None:
			return segments.segmentlist(row.segment for row in self)
		return segments.segmentlist(row.segment for row in self if row.segment_def_id == segment_def_id)

class SegmentSum(Segment):
	__slots__ = tuple(SegmentSumTable.validcolumns.keys())

SegmentSumTable.RowType = SegmentSum

#
#
#                               time_slide:table
#
#

TimeSlideID = ilwd.get_ilwdchar_class(u"time_slide", u"time_slide_id")

class TimeSlideTable(table.Table):
	tableName = "time_slide"
		"instrument": "lstring",
		"offset": "real_8"
	}
	constraints = "PRIMARY KEY (time_slide_id, instrument)"
	next_id = TimeSlideID(0)
	interncolumns = ("process_id", "time_slide_id", "instrument")

	def as_dict(self):
		"""
		Return a ditionary mapping time slide IDs to offset
		dictionaries.
		"""
		# FIXME:  deferred to work around non-LDG environments
		from lalburst.offsetvector import offsetvector
		d = {}
			if row.time_slide_id not in d:
				d[row.time_slide_id] = offsetvector()
			if row.instrument in d[row.time_slide_id]:
				raise KeyError("'%s': duplicate instrument '%s'" % (row.time_slide_id, row.instrument))
			d[row.time_slide_id][row.instrument] = row.offset
		return d

	def append_offsetvector(self, offsetvect, process):
		"""
		Append rows describing an instrument --> offset mapping to
		this table.  offsetvect is a dictionary mapping instrument
		to offset.  process should be the row in the process table
		on which the new time_slide table rows will be blamed (or
		any object with a process_id attribute).  The return value
		is the time_slide_id assigned to the new rows.
		"""
		time_slide_id = self.get_next_id()
		for instrument, offset in offsetvect.items():
			row = self.RowType()
			row.process_id = process.process_id
			row.time_slide_id = time_slide_id
			row.instrument = instrument
			row.offset = offset
			self.append(row)
		return time_slide_id

	def get_time_slide_id(self, offsetvect, create_new = None, superset_ok = False, nonunique_ok = False):
		"""
		Return the time_slide_id corresponding to the offset vector
		described by offsetvect, a dictionary of instrument/offset
		pairs.

		If the optional create_new argument is None (the default),
		then the table must contain a matching offset vector.  The
		return value is the ID of that vector.  If the table does
		not contain a matching offset vector then KeyError is
		raised.

		If the optional create_new argument is set to a Process
		object (or any other object with a process_id attribute),
		then if the table does not contain a matching offset vector
		a new one will be added to the table and marked as having
		been created by the given process.  The return value is the
		ID of the (possibly newly created) matching offset vector.

		If the optional superset_ok argument is False (the default)
		then an offset vector in the table is considered to "match"
		the requested offset vector only if they contain the exact
		same set of instruments.  If the superset_ok argument is
		True, then an offset vector in the table is considered to
		match the requested offset vector as long as it provides
		the same offsets for the same instruments as the requested
		vector, even if it provides offsets for other instruments
		as well.

		More than one offset vector in the table might match the
		requested vector.  If the optional nonunique_ok argument is
		False (the default), then KeyError will be raised if more
		than one offset vector in the table is found to match the
		requested vector.  If the optional nonunique_ok is True
		then the return value is the ID of one of the matching
		offset vectors selected at random.
		"""
		# look for matching offset vectors
		if superset_ok:
			ids = [id for id, slide in self.as_dict().items() if offsetvect == dict((instrument, offset) for instrument, offset in slide.items() if instrument in offsetvect)]
		else:
			ids = [id for id, slide in self.as_dict().items() if offsetvect == slide]
		if len(ids) > 1:
			# found more than one
			if nonunique_ok:
				# and that's OK
				return ids[0]
			# and that's not OK
			raise KeyError("%s not unique" % repr(offsetvect))
		if len(ids) == 1:
			# found one
			return ids[0]
		# offset vector not found in table
		if create_new is None:
			raise KeyError("%s not found" % repr(offsetvect))
		# that's OK, create new vector, return its ID
		return self.append_offsetvector(offsetvect, create_new)

class TimeSlide(table.Table.RowType):
	__slots__ = tuple(TimeSlideTable.validcolumns.keys())

TimeSlideTable.RowType = TimeSlide

#
#
#                             coinc_definer:table
#
#

CoincDefID = ilwd.get_ilwdchar_class(u"coinc_definer", u"coinc_def_id")

class CoincDefTable(table.Table):
	tableName = "coinc_definer"
		"coinc_def_id": "ilwd:char",
		"search_coinc_type": "int_4u",
		"description": "lstring"
	}
	constraints = "PRIMARY KEY (coinc_def_id)"
	next_id = CoincDefID(0)
		"cd_ssct_index": ("search", "search_coinc_type")
	}

	def get_coinc_def_id(self, search, search_coinc_type, create_new = True, description = None):
		"""
		Return the coinc_def_id for the row in the table whose
		search string and search_coinc_type integer have the values
		given.  If a matching row is not found, the default
		behaviour is to create a new row and return the ID assigned
		to the new row.  If, instead, create_new is False then
		KeyError is raised when a matching row is not found.  The
		optional description parameter can be used to set the
		description string assigned to the new row if one is
		created, otherwise the new row is left with no description.
		"""
		rows = [row for row in self if (row.search, row.search_coinc_type) == (search, search_coinc_type)]
		if len(rows) > 1:
			raise ValueError("(search, search coincidence type) = ('%s', %d) is not unique" % (search, search_coinc_type))
		if len(rows) > 0:
			return rows[0].coinc_def_id

		# coinc type not found in table
		if not create_new:
			raise KeyError((search, search_coinc_type))
		row.coinc_def_id = self.get_next_id()
		row.search_coinc_type = search_coinc_type
		row.description = description

		return row.coinc_def_id

class CoincDef(table.Table.RowType):
	__slots__ = tuple(CoincDefTable.validcolumns.keys())

CoincDefTable.RowType = CoincDef

#
#
#                              coinc_event:table
#
#

CoincID = ilwd.get_ilwdchar_class(u"coinc_event", u"coinc_event_id")

class CoincTable(table.Table):
	tableName = "coinc_event"
		"nevents": "int_4u",
		"likelihood": "real_8"
	}
	constraints = "PRIMARY KEY (coinc_event_id)"
	next_id = CoincID(0)
	interncolumns = ("process_id", "coinc_def_id", "time_slide_id", "instruments")
		"ce_cdi_index": ("coinc_def_id",),
		"ce_tsi_index": ("time_slide_id",)
	}

class Coinc(table.Table.RowType):
	__slots__ = tuple(CoincTable.validcolumns.keys())

	insts = instrumentsproperty("instruments")

		return self.insts

		self.insts = instruments

CoincTable.RowType = Coinc

#
#
#                            coinc_event_map:table
#
#

class CoincMapTable(table.Table):
	tableName = "coinc_event_map"
		"table_name": "char_v",
	}
	interncolumns = ("table_name",)
		"cem_tn_ei_index": ("table_name", "event_id"),
		"cem_cei_index": ("coinc_event_id",)
	}

class CoincMap(table.Table.RowType):
	__slots__ = tuple(CoincMapTable.validcolumns.keys())

CoincMapTable.RowType = CoincMap

#
#
#                                dq_list Table
#
#

DQSpecListID = ilwd.get_ilwdchar_class(u"dq_list", u"dq_list_id")
DQSpecListRowID = ilwd.get_ilwdchar_class(u"dq_list", u"dq_list_row_id")

class DQSpecListTable(table.Table):
	tableName = "dq_list"
		"dq_list_id": "ilwd:char",
		"dq_list_row_id": "ilwd:char",
		"flag": "lstring",
		"low_window": "real_8",
		"high_window": "real_8"
	}
	constraints = "PRIMARY KEY (dq_list_id, dq_list_row_id)"
	next_id = DQSpecListID(0)

class DQSpec(table.Table.RowType):
	__slots__ = tuple(DQSpecListTable.validcolumns.keys())

	def apply_to_segmentlist(self, seglist):
		"""
		Apply our low and high windows to the segments in a
		segmentlist.
		"""
		for i, seg in enumerate(seglist):
			seglist[i] = seg.__class__(seg[0] - self.low_window, seg[1] + self.high_window)

DQSpecListTable.RowType = DQSpec

#
#
#                               ligolw_mon:table
#
#

LIGOLWMonID = ilwd.get_ilwdchar_class(u"ligolw_mon", u"event_id")

class LIGOLWMonTable(table.Table):
	tableName = "ligolw_mon"
		"time": "int_4s",
		"time_ns": "int_4s",
		"confidence": "real_8",
	}
	next_id = LIGOLWMonID(0)

class LIGOLWMon(table.Table.RowType):
	__slots__ = tuple(LIGOLWMonTable.validcolumns.keys())

	def get_time(self):
		return LIGOTimeGPS(self.time, self.time_ns)

	def set_time(self, gps):
		self.time, self.time_ns = gps.seconds, gps.nanoseconds

LIGOLWMonTable.RowType = LIGOLWMon

#
#
#                            veto_definer:table
#
#

class VetoDefTable(table.Table):
	tableName = "veto_definer"
		"category": "int_4s",
		"start_pad": "int_4s",
		"end_pad": "int_4s",
	}
	interncolumns = ("process_id","ifo")

class VetoDef(table.Table.RowType):
	__slots__ = tuple(VetoDefTable.validcolumns.keys())

VetoDefTable.RowType = VetoDef

#
#
#                               summ_mime:table
#
#

SummMimeID = ilwd.get_ilwdchar_class(u"summ_mime", u"summ_mime_id")

class SummMimeTable(table.Table):
	tableName = "summ_mime"
		"origin": "lstring",
		"filename": "lstring",
		"submitter": "lstring",
		"descrip": "lstring",
		"mimedata": "blob",
		"mimedata_length": "int_4s",
		"mimetype": "lstring",
		"summ_mime_id": "ilwd:char"
	}
	constraints = "PRIMARY KEY (summ_mime_id)"
	next_id = SummMimeID(0)

class SummMime(table.Table.RowType):
	__slots__ = tuple(SummMimeTable.validcolumns.keys())

		self.end_time, self.end_time_ns = gps.seconds, gps.nanoseconds

SummMimeTable.RowType = SummMime

#
#
#                            time_slide_segment_map:table
#
#

class TimeSlideSegmentMapTable(table.Table):
	tableName = "time_slide_segment_map"
	}

class TimeSlideSegmentMap(table.Table.RowType):
	__slots__ = tuple(TimeSlideSegmentMapTable.validcolumns.keys())

TimeSlideSegmentMapTable.RowType = TimeSlideSegmentMap

#
#
#                                Table Metadata
#
#

#
# Table name ---> table type mapping.
#

TableByName = {
	CoincDefTable.tableName: CoincDefTable,
	CoincInspiralTable.tableName: CoincInspiralTable,
	CoincMapTable.tableName: CoincMapTable,
	CoincRingdownTable.tableName: CoincRingdownTable,
	CoincTable.tableName: CoincTable,
	DQSpecListTable.tableName: DQSpecListTable,
	ExperimentMapTable.tableName: ExperimentMapTable,
	ExperimentSummaryTable.tableName: ExperimentSummaryTable,
	ExperimentTable.tableName: ExperimentTable,
	ExtTriggersTable.tableName: ExtTriggersTable,
	FilterTable.tableName: FilterTable,
	GDSTriggerTable.tableName: GDSTriggerTable,
	LfnTable.tableName: LfnTable,
	LIGOLWMonTable.tableName: LIGOLWMonTable,
	MultiBurstTable.tableName: MultiBurstTable,
	MultiInspiralTable.tableName: MultiInspiralTable,
	ProcessParamsTable.tableName: ProcessParamsTable,
	ProcessTable.tableName: ProcessTable,
	SearchSummaryTable.tableName: SearchSummaryTable,
	SearchSummVarsTable.tableName: SearchSummVarsTable,
	SegmentDefTable.tableName: SegmentDefTable,
	SegmentSumTable.tableName: SegmentSumTable,
	SegmentTable.tableName: SegmentTable,
	SimBurstTable.tableName: SimBurstTable,
	SimInspiralTable.tableName: SimInspiralTable,
	SimInstParamsTable.tableName: SimInstParamsTable,
	SimRingdownTable.tableName: SimRingdownTable,
	SnglBurstTable.tableName: SnglBurstTable,
	SnglInspiralTable.tableName: SnglInspiralTable,
	SnglRingdownTable.tableName: SnglRingdownTable,
	StochasticTable.tableName: StochasticTable,
	StochSummTable.tableName: StochSummTable,
	SummValueTable.tableName: SummValueTable,
	SummMimeTable.tableName: SummMimeTable,
	TimeSlideSegmentMapTable.tableName: TimeSlideSegmentMapTable,
	TimeSlideTable.tableName: TimeSlideTable,
	VetoDefTable.tableName: VetoDefTable
}

def reset_next_ids(classes):
	"""
	For each class in the list, if the .next_id attribute is not None
	(meaning the table has an ID generator associated with it), set
	.next_id to 0.  This has the effect of reseting the ID generators,
	and is useful in applications that process multiple documents and
	add new rows to tables in those documents.  Calling this function
	between documents prevents new row IDs from growing continuously
	from document to document.  There is no need to do this, it's
	purpose is merely aesthetic, but it can be confusing to open a
	document and find process ID 300 in the process table and wonder
	what happened to the other 299 processes.

	Example:

	>>> reset_next_ids(TableByName.values())
	"""
	for cls in classes:
		cls.reset_next_id()

#
#
#
#

#
#

	"""
	glue.ligolw.LIGOLWContentHandler, to cause it to use the Table
	classes defined in this module when parsing XML documents.

	Example:

	>>> class MyContentHandler(ligolw.LIGOLWContentHandler):
	...	pass
	...
	>>> use_in(MyContentHandler)
	<class 'glue.ligolw.lsctables.MyContentHandler'>
	"""
	ContentHandler = table.use_in(ContentHandler)

	def startTable(self, parent, attrs, __orig_startTable = ContentHandler.startTable):
		name = table.Table.TableName(attrs[u"Name"])
		if name in TableByName:
			return TableByName[name](attrs)
		return __orig_startTable(self, parent, attrs)

# Copyright (C) 2013 Ian Harry
#
#
#
"""
This module contains functions to filter injections with only useful templates.

This module implements a set of checks to test for each segment and template
combination whether injections contained within the segment are sufficiently
"similar" to the template to require a matched-filter. There are a few ways of
testing the "similarity" of templates and injections.

* A chirp time threshold rejects templates if chirp time difference is large
* A coarse match threshold rejects templates if a coarse overlap is small

"""

from pycbc import DYN_RANGE_FAC
from pycbc.filter import match
from pycbc.pnutils import nearest_larger_binary_number
from pycbc.pnutils import mass1_mass2_to_tau0_tau3
from pycbc.types import FrequencySeries, zeros

_injfilterrejector_group_help = \
    ("Options that, if injections are present in "
     "this run, are responsible for performing pre-checks between injections "
     "in the data being filtered and the current search template to determine "
     "if the template has any chance of actually detecting the injection. "
     "The parameters of this test are given by the various options below. "
     "The --injection-filter-rejector-chirp-time-window and "
     "--injection-filter-rejector-match-threshold options need to be provided "
     "if those tests are desired. Other options will take default values "
     "unless overriden. More details on these options follow.")

_injfilterer_cthresh_help = \
    ("If this value is not None and an "
     "injection file is given then we will calculate the difference in "
     "chirp time (tau_0) between the template and each injection in the "
     "analysis segment. If the difference is greate than this threshold for "
     "all injections then filtering is not performed. By default this will "
     "be None.")
_injfilterer_mthresh_help = \
     "injection file is provided then we will calculate a 'coarse match' "
     "between the template and each injection in the analysis segment. If the "
     "match is less than this threshold for all injections then filtering is "
     "not performed. Parameters for the 'coarse match' follow. By default "
     "this value will be None.")
_injfilterer_deltaf_help = \
    ("If injections are present and a match threshold is "
     "provided, this option specifies the frequency spacing that will be used "
     "for injections, templates and PSD when computing the 'coarse match'. "
     "Templates will be generated directly with this spacing. The PSD and "
     "injections will be resampled.")
_injfilterer_fmax_help = \
     "provided, this option specifies the maximum frequency that will be used "
     "Templates will be generated directly with this max frequency. The PSD "
     "and injections' frequency series will be truncated.")
_injfilterer_buffer_help = \
    ("If injections are present and either a match "
     "threshold or a chirp-time window is given, we will determine if "
     "injections are 'in' the specified analysis chunk by using the end "
     "times. If this value is non-zero the analysis chunk is extended on both "
     "sides by this amount before determining if injections are within the "
     "given window.")
_injfilterer_flower_help = \
     "threshold or a chirp-time window is given, this value is used to set "
     "the lower frequency for determine chirp times or for calculating "
     "matches. If this value is None the lower frequency used for the full "
     "matched-filter is used. Otherwise this value is used.")

def insert_injfilterrejector_option_group(parser):
    """Add options for injfilterrejector to executable."""
    injfilterrejector_group = \
        parser.add_argument_group(_injfilterrejector_group_help)
    curr_arg = "--injection-filter-rejector-chirp-time-window"
    injfilterrejector_group.add_argument(curr_arg, type=float, default=None,
                                         help=_injfilterer_cthresh_help)
    curr_arg = "--injection-filter-rejector-match-threshold"
                                         help=_injfilterer_mthresh_help)
    curr_arg = "--injection-filter-rejector-coarsematch-deltaf"
    injfilterrejector_group.add_argument(curr_arg, type=float, default=1.,
                                         help=_injfilterer_deltaf_help)
    curr_arg = "--injection-filter-rejector-coarsematch-fmax"
    injfilterrejector_group.add_argument(curr_arg, type=float, default=256.,
                                         help=_injfilterer_fmax_help)
    curr_arg = "--injection-filter-rejector-seg-buffer"
    injfilterrejector_group.add_argument(curr_arg, type=int, default=10,
                                         help=_injfilterer_buffer_help)
    curr_arg = "--injection-filter-rejector-f-lower"
    injfilterrejector_group.add_argument(curr_arg, type=int, default=None,
                                         help=_injfilterer_flower_help)

def insert_injfilterrejector_option_group_multi_ifo(parser):
    injfilterrejector_group.add_argument(
        curr_arg, type=float, default=None, nargs='+', metavar='IFO:VALUE',
        action=MultiDetOptionAction, help=_injfilterer_cthresh_help)
        action=MultiDetOptionAction, help=_injfilterer_mthresh_help)
        curr_arg, type=float, default=1., nargs='+', metavar='IFO:VALUE',
        action=MultiDetOptionAction, help=_injfilterer_deltaf_help)
        curr_arg, type=float, default=256., nargs='+', metavar='IFO:VALUE',
        action=MultiDetOptionAction, help=_injfilterer_fmax_help)
        curr_arg, type=int, default=10, nargs='+', metavar='IFO:VALUE',
        action=MultiDetOptionAction, help=_injfilterer_buffer_help)
        curr_arg, type=int, default=None, help=_injfilterer_flower_help,
        metavar='IFO:VALUE', action=MultiDetOptionAction, nargs='+')

class InjFilterRejector(object):
    """Class for holding parameters for using injection/template pre-filtering.

    This class is responsible for identifying where a matched-filter operation
    between templates and data is unncessary because the injections contained
    in the data will not match well with the given template.
    """

    def __init__(self, injection_file, chirp_time_window,
                 match_threshold, f_lower, coarsematch_deltaf=1.,
                 coarsematch_fmax=256, seg_buffer=10):
        """Initialise InjFilterRejector instance."""
        # Determine if InjFilterRejector is to be enabled
        if injection_file is None or injection_file == 'False' or\
                (chirp_time_window is None and match_threshold is None):
            self.enabled = False
            return
        else:
            self.enabled = True

        # Store parameters
        self.chirp_time_window = chirp_time_window
        self.match_threshold = match_threshold
        self.coarsematch_deltaf = coarsematch_deltaf
        self.coarsematch_fmax = coarsematch_fmax
        self.seg_buffer = seg_buffer
        self.f_lower = f_lower
        assert(self.f_lower is not None)

        # Variables for storing arrays (reduced injections, memory
        # for templates, reduced PSDs ...)
        self.short_injections = {}
        self._short_template_mem = None
        self._short_psd_storage = {}
        self._short_template_id = None

    def from_cli(cls, opt):
        """Create an InjFilterRejector instance from command-line options."""
        injection_file = opt.injection_file
        chirp_time_window = \
            opt.injection_filter_rejector_chirp_time_window
        match_threshold = opt.injection_filter_rejector_match_threshold
        coarsematch_deltaf = opt.injection_filter_rejector_coarsematch_deltaf
        coarsematch_fmax = opt.injection_filter_rejector_coarsematch_fmax
        seg_buffer = opt.injection_filter_rejector_seg_buffer
        if opt.injection_filter_rejector_f_lower is not None:
            f_lower = opt.injection_filter_rejector_f_lower
        else:
            # NOTE: Uses main low-frequency cutoff as default option. This may
            #       need some editing if using this in multi_inspiral, which I
            #       leave for future work, or if this is being used in another
            #       code which doesn't have --low-frequency-cutoff
            f_lower = opt.low_frequency_cutoff
        return cls(injection_file, chirp_time_window, match_threshold,
                   f_lower, coarsematch_deltaf=coarsematch_deltaf,
                   coarsematch_fmax=coarsematch_fmax,
                   seg_buffer=seg_buffer)

    def from_cli_single_ifo(cls, opt, ifo):
        injection_file = opt.injection_file[ifo]
            opt.injection_filter_rejector_chirp_time_window[ifo]
        match_threshold = opt.injection_filter_rejector_match_threshold[ifo]
        coarsematch_deltaf = \
            opt.injection_filter_rejector_coarsematch_deltaf[ifo]
        coarsematch_fmax = opt.injection_filter_rejector_coarsematch_fmax[ifo]
        seg_buffer = opt.injection_filter_rejector_seg_buffer[ifo]
        if opt.injection_filter_rejector_f_lower[ifo] is not None:
            f_lower = opt.injection_filter_rejector_f_lower[ifo]
        else:
        return cls(injection_file, chirp_time_window,
                   match_threshold, f_lower,
                   coarsematch_deltaf, coarsematch_fmax,

    def from_cli_multi_ifos(cls, opt, ifos):
        inj_filter_rejectors = {}
            inj_filter_rejectors[ifo] = cls.from_cli_single_ifo(opt, ifo)
        return inj_filter_rejectors

    def generate_short_inj_from_inj(self, inj_waveform, simulation_id):
        """Generate and a store a truncated representation of inj_waveform."""
        if not self.enabled or not self.match_threshold:
            # Do nothing!
            return
        if simulation_id in self.short_injections:
            err_msg = "An injection with simulation id "
            err_msg += str(simulation_id)
            err_msg += " has already been added. This suggests "
            err_msg += "that your injection file contains injections with "
            err_msg += "duplicate simulation_ids. This is not allowed."
        curr_length = len(inj_waveform)
        new_length = int(nearest_larger_binary_number(curr_length))
        # Don't want length less than 1/delta_f
        while new_length * inj_waveform.delta_t < 1./self.coarsematch_deltaf:
            new_length = new_length * 2
        inj_waveform.resize(new_length)
        inj_tilde = inj_waveform.to_frequencyseries()
        # Dynamic range is important here!
        inj_tilde_np = inj_tilde.numpy() * DYN_RANGE_FAC
        delta_f = inj_tilde.get_delta_f()
        new_freq_len = int(self.coarsematch_fmax / delta_f + 1)
        # This shouldn't be a problem if injections are generated at
        # 16384 Hz ... It is only a problem of injection sample rate
        # gives a lower Nyquist than the trunc_f_max. If this error is
        # ever raised one could consider zero-padding the injection.
        assert(new_freq_len <= len(inj_tilde))
        df_ratio = int(self.coarsematch_deltaf/delta_f)
        inj_tilde_np = inj_tilde_np[:new_freq_len:df_ratio]
        new_inj = FrequencySeries(inj_tilde_np, dtype=np.complex64,
                                  delta_f=self.coarsematch_deltaf)
        self.short_injections[simulation_id] = new_inj

    def template_segment_checker(self, bank, t_num, segment, start_time):
        """Test if injections in segment are worth filtering with template.

        Using the current template, current segment, and injections within that
        segment. Test if the injections and sufficiently "similar" to any of
        the injections to justify actually performing a matched-filter call.
        Ther are two parts to this test: First we check if the chirp time of
        the template is within a provided window of any of the injections. If
        not then stop here, it is not worth filtering this template, segment
        combination for this injection set. If this check passes we compute a
        match between a coarse representation of the template and a coarse
        representation of each of the injections. If that match is above a
        user-provided value for any of the injections then filtering can
        proceed. This is currently only available if using frequency-domain
        templates.

        FIXME

        Returns
        --------
        FIXME
        """
        if not self.enabled:
            # If disabled, always filter (ie. return True)
            return True

        # Get times covered by segment analyze
        sample_rate = 2. * (len(segment) - 1) * segment.delta_f
        cum_ind = segment.cumulative_index
        diff = segment.analyze.stop - segment.analyze.start
        seg_start_time = cum_ind / sample_rate + start_time
        seg_end_time = (cum_ind + diff) / sample_rate + start_time
        # And add buffer
        seg_start_time = seg_start_time - self.seg_buffer
        seg_end_time = seg_end_time + self.seg_buffer

        # Chirp time test
        if self.chirp_time_window is not None:
            m1 = bank.table[t_num]['mass1']
            m2 = bank.table[t_num]['mass2']
            tau0_temp, _ = mass1_mass2_to_tau0_tau3(m1, m2, self.f_lower)
            for inj in self.injection_params.table:
                if isinstance(inj, np.record):
                    # hdf format file
                    end_time = inj['tc']
                else:
                    # must be an xml file originally
                    end_time = inj.geocent_end_time + \
                        1E-9 * inj.geocent_end_time_ns

                if not(seg_start_time <= end_time <= seg_end_time):
                    continue
                tau0_inj, _ = \
                    mass1_mass2_to_tau0_tau3(inj.mass1, inj.mass2,
                                             self.f_lower)
                tau_diff = abs(tau0_temp - tau0_inj)
                if tau_diff <= self.chirp_time_window:
                    break
            else:
                # Get's here if all injections are outside chirp-time window
                return False

        # Coarse match test
        if self.match_threshold:
            if self._short_template_mem is None:
                # Set the memory for the short templates
                wav_len = 1 + int(self.coarsematch_fmax /
                                  self.coarsematch_deltaf)
                self._short_template_mem = zeros(wav_len, dtype=np.complex64)

            # Set the current short PSD to red_psd
            try:
                red_psd = self._short_psd_storage[id(segment.psd)]
                # PSD doesn't exist yet, so make it!
                curr_psd = segment.psd.numpy()
                step_size = int(self.coarsematch_deltaf / segment.psd.delta_f)
                max_idx = int(self.coarsematch_fmax / segment.psd.delta_f) + 1
                red_psd_data = curr_psd[:max_idx:step_size]
                red_psd = FrequencySeries(red_psd_data, #copy=False,
                                          delta_f=self.coarsematch_deltaf)
                self._short_psd_storage[id(curr_psd)] = red_psd

            # Set htilde to be the current short template
            if not t_num == self._short_template_id:
                # Set the memory for the short templates if unset
                if self._short_template_mem is None:
                    wav_len = 1 + int(self.coarsematch_fmax /
                                      self.coarsematch_deltaf)
                    self._short_template_mem = zeros(wav_len,
                                                     dtype=np.complex64)
                # Generate short waveform
                htilde = bank.generate_with_delta_f_and_max_freq(
                    t_num, self.coarsematch_fmax, self.coarsematch_deltaf,
                    low_frequency_cutoff=bank.table[t_num].f_lower,
                    cached_mem=self._short_template_mem)
                self._short_template_id = t_num
                self._short_template_wav = htilde
            else:
                htilde = self._short_template_wav

            for ii, inj in enumerate(self.injection_params.table):
                    sim_id = ii
                else:
                    sim_id = inj.simulation_id

                if not(seg_start_time < end_time < seg_end_time):
                    continue
                curr_inj = self.short_injections[sim_id]
                o, _ = match(htilde, curr_inj, psd=red_psd,
                             low_frequency_cutoff=self.f_lower)
                if o > self.match_threshold:
                    break
            else:
                # Get's here if all injections are outside match threshold

# convenience classes for accessing hdf5 trigger files
# the 'get_column()' method is implemented parallel to
# the existing pylal.SnglInspiralUtils functions

import inspect

from itertools import chain
from six.moves import cPickle as pickle

from io import BytesIO
from lal import LIGOTimeGPS, YRJUL_SI

from glue.ligolw import ligolw
from glue.ligolw import lsctables
from glue.ligolw.utils import process as ligolw_process

from pycbc import version as pycbc_version
from pycbc.tmpltbank import return_search_summary
from pycbc.tmpltbank import return_empty_sngl
from pycbc import events, conversions, pnutils
from pycbc.events import ranking, veto

class HFile(h5py.File):
    """ Low level extensions to the capabilities of reading an hdf5 File
    """
    def select(self, fcn, *args, **kwds):
        """ Return arrays from an hdf5 file that satisfy the given function

        fcn : a function
            A function that accepts the same number of argument as keys given
            and returns a boolean array of the same length.

        args : strings
            A variable number of strings that are keys into the hdf5. These must
            refer to arrays of equal length.

        chunksize : {1e6, int}, optional
            Number of elements to read and process at a time.

        return_indices : bool, optional
            If True, also return the indices of elements passing the function.

        Returns
        -------
        values : np.ndarrays
            A variable number of arrays depending on the number of keys into
            the hdf5 file that are given. If return_indices is True, the first
            element is an array of indices of elements passing the function.

        >>> f = HFile(filename)
        >>> snr = f.select(lambda snr: snr > 6, 'H1/snr')
        """

        # get references to each array
        refs = {}
        data = {}
            refs[arg] = self[arg]
            data[arg] = []

        return_indices = kwds.get('return_indices', False)
        indices = np.array([], dtype=np.uint64)

        # To conserve memory read the array in chunks
        chunksize = kwds.get('chunksize', int(1e6))
        size = len(refs[arg])

        i = 0
        while i < size:
            r = i + chunksize if i + chunksize < size else size

            #Read each chunks worth of data and find where it passes the function
            partial = [refs[arg][i:r] for arg in args]
            keep = fcn(*partial)
            if return_indices:
                indices = np.concatenate([indices, np.flatnonzero(keep) + i])

            #store only the results that pass the function
            for arg, part in zip(args, partial):
                data[arg].append(part[keep])

            i += chunksize

        # Combine the partial results into full arrays
        if len(args) == 1:
            res = np.concatenate(data[args[0]])
                return indices.astype(np.uint64), res
            else:
                return res
        else:
            res = tuple(np.concatenate(data[arg]) for arg in args)
                return (indices.astype(np.uint64),) + res
            else:
                return res

class DictArray(object):
    """ Utility for organizing sets of arrays of equal length.

    Manages a dictionary of arrays of equal length. This can also
    be instantiated with a set of hdf5 files and the key values. The full
    data is always in memory and all operations create new instances of the
    DictArray.
    """
    def __init__(self, data=None, files=None, groups=None):
        """ Create a DictArray

        data: dict, optional
            Dictionary of equal length numpy arrays
        files: list of filenames, optional
            List of hdf5 file filenames. Incompatibile with the `data` option.
        groups: list of strings
            List of keys into each file. Required by the files option.
        """
        # Check that input fits with how the DictArray is set up
        if data and files:
            raise RuntimeError('DictArray can only have data or files as '
                               'input, not both.')
        if files and not groups:
            raise RuntimeError('If files are given then need groups.')

        self.groups = groups
        if files:
            self.data = {}
            for g in groups:
                self.data[g] = []

            for f in files:
                d = HFile(f)
                for g in groups:
                    if g in d:
                        self.data[g].append(d[g][:])
                d.close()

            for k in self.data:
                if not len(self.data[k]) == 0:
                    self.data[k] = np.concatenate(self.data[k])

        for k in self.data:
            setattr(self, k, self.data[k])

    def _return(self, data):
        return self.__class__(data=data)

        return len(self.data[tuple(self.data.keys())[0]])

    def __add__(self, other):
        data = {}
            try:
                data[k] = np.concatenate([self.data[k], other.data[k]])
                logging.info('%s does not exist in other data' % k)
        return self._return(data=data)

    def select(self, idx):
        """ Return a new DictArray containing only the indexed values
        """
        data = {}
            data[k] = self.data[k][idx]

    def remove(self, idx):
        """ Return a new DictArray that does not contain the indexed values
        """
        data = {}
            data[k] = np.delete(self.data[k], np.array(idx, dtype=int))

    def save(self, outname):
        f = HFile(outname, "w")
        for k in self.attrs:
            f.attrs[k] = self.attrs[k]

            f.create_dataset(k, data=self.data[k],
                      compression='gzip',
                      compression_opts=9,
                      shuffle=True)
        f.close()

class StatmapData(DictArray):
    def __init__(self, data=None, seg=None, attrs=None, files=None,
                 groups=('stat', 'time1', 'time2', 'trigger_id1',
                         'trigger_id2', 'template_id', 'decimation_factor',
                         'timeslide_id')):
        super(StatmapData, self).__init__(data=data, files=files,
                                          groups=groups)

        if data:
            self.seg=seg
            self.attrs=attrs
        elif files:
            f = HFile(files[0], "r")
            self.seg = f['segments']
            self.attrs = f.attrs

        return self.__class__(data=data, attrs=self.attrs, seg=self.seg)

    def cluster(self, window):
        """ Cluster the dict array, assuming it has the relevant Coinc colums,
        time1, time2, stat, and timeslide_id
        """
        # If no events, do nothing
        if len(self.time1) == 0 or len(self.time2) == 0:
            return self
        from pycbc.events import cluster_coincs
        interval = self.attrs['timeslide_interval']
        cid = cluster_coincs(self.stat, self.time1, self.time2,
                                 self.timeslide_id, interval, window)
        return self.select(cid)

        super(StatmapData, self).save(outname)
        with HFile(outname, "w") as f:
            for key in self.seg.keys():
                f['segments/%s/start' % key] = self.seg[key]['start'][:]
                f['segments/%s/end' % key] = self.seg[key]['end'][:]

class MultiifoStatmapData(StatmapData):
    def __init__(self, data=None, seg=None, attrs=None,
                       files=None, ifos=None):
        groups = ['decimation_factor', 'stat', 'template_id', 'timeslide_id']
            groups += ['%s/time' % ifo]
            groups += ['%s/trigger_id' % ifo]

        super(MultiifoStatmapData, self).__init__(data=data, files=files,
                                                  groups=groups, attrs=attrs,
                                                  seg=seg)

        ifolist = self.attrs['ifos'].split(' ')
        return self.__class__(data=data, attrs=self.attrs, seg=self.seg,
                              ifos=ifolist)

        """
        pivot_ifo = self.attrs['pivot']
        fixed_ifo = self.attrs['fixed']
        if len(self.data['%s/time' % pivot_ifo]) == 0 or len(self.data['%s/time' % fixed_ifo]) == 0:
        cid = cluster_coincs(self.stat,
                             self.data['%s/time' % pivot_ifo],
                             self.data['%s/time' % fixed_ifo],
                             self.timeslide_id,
                             interval,
                             window)

class FileData(object):

    def __init__(self, fname, group=None, columnlist=None, filter_func=None):
        """
        group : string
            Name of group to be read from the file
        columnlist : list of strings
            Names of columns to be read; if None, use all existing columns
        filter_func : string
            String should evaluate to a Boolean expression using attributes
            of the class instance derived from columns: ex. 'self.snr < 6.5'
        """
        if not fname: raise RuntimeError("Didn't get a file!")
        self.fname = fname
        self.h5file = HFile(fname, "r")
            if len(self.h5file.keys()) == 1:
                group, = self.h5file.keys()
            else:
                raise RuntimeError("Didn't get a group!")
        self.group_key = group
        self.group = self.h5file[group]
        self.columns = columnlist if columnlist is not None \
                       else list(self.group.keys())
        self.filter_func = filter_func
        self._mask = None

    def close(self):
        self.h5file.close()

    @property
    def mask(self):
        """
        Create a mask implementing the requested filter on the datasets

        Returns
        -------
        array of Boolean
            True for dataset indices to be returned by the get_column method
        """
        if self.filter_func is None:
            raise RuntimeError("Can't get a mask without a filter function!")
        else:
            # only evaluate if no previous calculation was done
            if self._mask is None:
                # get required columns into the namespace as numpy arrays
                for column in self.columns:
                    if column in self.filter_func:
                        setattr(self, column, self.group[column][:])
                self._mask = eval(self.filter_func)
            return self._mask

    def get_column(self, col):
        """
        col : string
            Name of the dataset to be returned

        Returns
        -------
        numpy array
            Values from the dataset, filtered if requested
        """
        # catch corner case with an empty file (group with no datasets)
        if not len(self.group.keys()):
            return np.array([])
        vals = self.group[col]
        if self.filter_func:
            return vals[self.mask]
        else:
            return vals[:]

class DataFromFiles(object):

    def __init__(self, filelist, group=None, columnlist=None, filter_func=None):
        self.files = filelist
        self.group = group
        self.columns = columnlist

        """
        Loop over files getting the requested dataset values from each

        Returns
        -------
            Values from the dataset, filtered if requested and
            concatenated in order of file list
        """
        logging.info('getting %s' % col)
        vals = []
        for f in self.files:
            d = FileData(f, group=self.group, columnlist=self.columns,
                         filter_func=self.filter_func)
            vals.append(d.get_column(col))
            # Close each file since h5py has an upper limit on the number of
            # open file objects (approx. 1000)
            d.close()
        logging.info('- got %i values' % sum(len(v) for v in vals))
        return np.concatenate(vals)

class SingleDetTriggers(object):
    """
    Provides easy access to the parameters of single-detector CBC triggers.
    """
    # FIXME: Some of these are optional and should be kwargs.
    def __init__(self, trig_file, bank_file, veto_file,
                 segment_name, filter_func, detector, premask=None):
        logging.info('Loading triggers')
        self.trigs_f = HFile(trig_file, 'r')
        self.trigs = self.trigs_f[detector]
        self.ifo = detector  # convenience attributes
        self.detector = detector
        if bank_file:
            logging.info('Loading bank')
            self.bank = HFile(bank_file, 'r')
        else:
            logging.info('No bank file given')
            # empty dict in place of non-existent hdf file
            self.bank = {}

        if premask is not None:
            self.mask = premask
        else:
            self.mask = np.ones(len(self.trigs['end_time']), dtype=bool)

        if veto_file:
            logging.info('Applying veto segments')
            # veto_mask is an array of indices into the trigger arrays
            # giving the surviving triggers
            logging.info('%i triggers before vetoes', self.mask.sum())
            self.veto_mask, _ = events.veto.indices_outside_segments(
                self.end_time, [veto_file],
                ifo=detector, segment_name=segment_name)

            idx = np.flatnonzero(self.mask)[self.veto_mask]
            self.mask[:] = False
            self.mask[idx] = True
            logging.info('%i triggers remain after vetoes',
                          len(self.veto_mask))

        # FIXME this should use the hfile select interface to avoid
        # memory and processing limitations.
        if filter_func:
            # get required columns into the namespace with dummy attribute
            # names to avoid confusion with other class properties
            logging.info('Setting up filter function')
            for c in self.trigs.keys():
                if c in filter_func:
                    setattr(self, '_'+c, self.trigs[c][:])
            for c in self.bank.keys():
                    # get template parameters corresponding to triggers
                    setattr(self, '_'+c,
                          np.array(self.bank[c])[self.trigs['template_id'][:]])

            self.filter_mask = eval(filter_func.replace('self.', 'self._'))
            # remove the dummy attributes
            for c in chain(self.trigs.keys(), self.bank.keys()):
                if c in filter_func: delattr(self, '_'+c)

            self.mask = self.mask & self.filter_mask
            logging.info('%i triggers remain after cut on %s',
                         sum(self.mask), filter_func)

        # Is key in the TRIGGER_MERGE file?
        try:
            return self.get_column(key)
            pass

        # Is key in the bank file?
        try:
            self.checkbank(key)
            return self.bank[key][:][self.template_id]
        except (RuntimeError, KeyError) as exc:
            err_msg = "Cannot find {} in input files".format(key)
            raise_from(ValueError(err_msg), exc)

    def checkbank(self, param):
        if self.bank == {}:
            return RuntimeError("Can't get %s values without a bank file"
                                                                       % param)

    def trig_dict(self):
        """Returns dict of the masked trigger valuse """
        mtrigs = {}
        for k in self.trigs:
            if len(self.trigs[k]) == len(self.trigs['end_time']):
                if self.mask is not None:
                    mtrigs[k] = self.trigs[k][self.mask]
                else:
                    mtrigs[k] = self.trigs[k][:]
        return mtrigs

    def get_param_names(cls):
        """Returns a list of plottable CBC parameter variables"""
        return [m[0] for m in inspect.getmembers(cls) \
            if type(m[1]) == property]

    def apply_mask(self, logic_mask):
        """Apply a boolean array to the set of triggers"""
        if hasattr(self.mask, 'dtype') and (self.mask.dtype == 'bool'):
            orig_indices = self.mask.nonzero()[0][logic_mask]
            self.mask[orig_indices] = True
        else:
            self.mask = list(np.array(self.mask)[logic_mask])

    def mask_to_n_loudest_clustered_events(self, rank_method,
                                           n_loudest=10,
                                           cluster_window=10):
        """Edits the mask property of the class to point to the N loudest
        single detector events as ranked by ranking statistic. Events are
        clustered so that no more than 1 event within +/- cluster-window will
        be considered."""

        # If this becomes memory intensive we can optimize
        stat = rank_method.rank_stat_single((self.ifo, self.trig_dict()))

        times = self.end_time
        index = stat.argsort()[::-1]
        new_times = []
        new_index = []
        for curr_idx in index:
            curr_time = times[curr_idx]
            for time in new_times:
                if abs(curr_time - time) < cluster_window:
                    break
            else:
                # Only get here if no other triggers within cluster window
                new_index.append(curr_idx)
                new_times.append(curr_time)
            if len(new_index) >= n_loudest:
                break

        index = np.array(new_index)
        index.sort()
        self.stat = stat[index]
        if hasattr(self.mask, 'dtype') and self.mask.dtype == 'bool':
            orig_indices = np.flatnonzero(self.mask)[index]
            self.mask = list(orig_indices)
        elif isinstance(self.mask, list):
            self.mask = list(np.array(self.mask)[index])

    @property
    def template_id(self):
        return self.get_column('template_id')

    @property
    def mass1(self):
        self.checkbank('mass1')
        return self.bank['mass1'][:][self.template_id]

    @property
    def mass2(self):
        self.checkbank('mass2')
        return self.bank['mass2'][:][self.template_id]

    @property
    def spin1z(self):
        self.checkbank('spin1z')
        return self.bank['spin1z'][:][self.template_id]

    @property
    def spin2z(self):
        self.checkbank('spin2z')
        return self.bank['spin2z'][:][self.template_id]

    @property
    def spin2x(self):
        self.checkbank('spin2x')
        return self.bank['spin2x'][:][self.template_id]

    @property
    def spin2y(self):
        self.checkbank('spin2y')
        return self.bank['spin2y'][:][self.template_id]

    @property
    def spin1x(self):
        self.checkbank('spin1x')
        return self.bank['spin1x'][:][self.template_id]

    @property
    def spin1y(self):
        self.checkbank('spin1y')
        return self.bank['spin1y'][:][self.template_id]

    @property
    def inclination(self):
        self.checkbank('inclination')
        return self.bank['inclination'][:][self.template_id]

    @property
    def f_lower(self):
        self.checkbank('f_lower')
        return self.bank['f_lower'][:][self.template_id]

    @property
    def approximant(self):
        self.checkbank('approximant')
        return self.bank['approximant'][:][self.template_id]

    @property
    def mtotal(self):
        return self.mass1 + self.mass2

    @property
    def mchirp(self):
        return conversions.mchirp_from_mass1_mass2(self.mass1, self.mass2)

    @property
    def eta(self):
        return conversions.eta_from_mass1_mass2(self.mass1, self.mass2)

    @property
    def effective_spin(self):
        # FIXME assumes aligned spins
        return conversions.chi_eff(self.mass1, self.mass2,
                                   self.spin1z, self.spin2z)

    # IMPROVEME: would like to have a way to access all get_freq and/or
    # other pnutils.* names rather than hard-coding each one
    # - eg make this part of a fancy interface to the bank file ?
    @property
    def f_seobnrv2_peak(self):
        return pnutils.get_freq('fSEOBNRv2Peak', self.mass1, self.mass2,
                                self.spin1z, self.spin2z)

    @property
    def f_seobnrv4_peak(self):
        return pnutils.get_freq('fSEOBNRv4Peak', self.mass1, self.mass2,

    @property
    def end_time(self):
        return self.get_column('end_time')

    @property
    def template_duration(self):
        return self.get_column('template_duration')

    @property
    def snr(self):
        return self.get_column('snr')

    @property
    def sgchisq(self):
        return self.get_column('sg_chisq')

    @property
    def u_vals(self):
        return self.get_column('u_vals')

    @property
    def rchisq(self):
        return self.get_column('chisq') \
            / (self.get_column('chisq_dof') * 2 - 2)

    @property
    def psd_var_val(self):
        return self.get_column('psd_var_val')

    @property
    def newsnr(self):
        return ranking.newsnr(self.snr, self.rchisq)

    @property
    def newsnr_sgveto(self):
        return ranking.newsnr_sgveto(self.snr, self.rchisq, self.sgchisq)

    @property
    def newsnr_sgveto_psdvar(self):
        return ranking.newsnr_sgveto_psdvar(self.snr, self.rchisq,
                                           self.sgchisq, self.psd_var_val)

    @property
    def newsnr_sgveto_psdvar_threshold(self):
        return ranking.newsnr_sgveto_psdvar_threshold(self.snr, self.rchisq,

    def get_ranking(self, rank_name, **kwargs):
        return ranking.get_sngls_ranking_from_trigs(self, rank_name, **kwargs)

    def get_column(self, cname):
        # Fiducial value that seems to work, not extensively tuned.
        MFRAC = 0.3

        # If the mask accesses few enough elements then directly use it
        # This can be slower than reading in all the elements if most of them
        # will be read.
        if self.mask is not None and (isinstance(self.mask, list) or \
                (len(self.mask.nonzero()[0]) < (len(self.mask) * MFRAC))):
            return self.trigs[cname][self.mask]

        # We have a lot of elements to read so we resort to readin the entire
        # array before masking.
        elif self.mask is not None:
            return self.trigs[cname][:][self.mask]
        else:
            return self.trigs[cname][:]

class ForegroundTriggers(object):
    # FIXME: A lot of this is hardcoded to expect two ifos
    def __init__(self, coinc_file, bank_file, sngl_files=None, n_loudest=None,
                     group='foreground'):
        self.coinc_file = FileData(coinc_file, group=group)
        if 'ifos' in self.coinc_file.h5file.attrs:
            self.ifos = self.coinc_file.h5file.attrs['ifos'].split(' ')
        else:
            self.ifos = [self.coinc_file.h5file.attrs['detector_1'],
                         self.coinc_file.h5file.attrs['detector_2']]
        self.sngl_files = {}
        if sngl_files is not None:
            for sngl_file in sngl_files:
                curr_dat = FileData(sngl_file)
                curr_ifo = curr_dat.group_key
                self.sngl_files[curr_ifo] = curr_dat

        if not all([ifo in self.sngl_files.keys() for ifo in self.ifos]):
            print("sngl_files: {}".format(sngl_files))
            print("self.ifos: {}".format(self.ifos))
            raise RuntimeError("IFOs in statmap file not all represented "
                               "by single-detector trigger files.")
        if not sorted(self.sngl_files.keys()) == sorted(self.ifos):
            logging.warning("WARNING: Single-detector trigger files "
                            "given for IFOs not in the statmap file")

        self.bank_file = HFile(bank_file, "r")
        self.n_loudest = n_loudest

        self._sort_arr = None
        self._template_id = None
        self._trig_ids = None

    @property
    def sort_arr(self):
        if self._sort_arr is None:
            ifar = self.coinc_file.get_column('ifar')
            sorting = ifar.argsort()[::-1]
            if self.n_loudest:
                sorting = sorting[:self.n_loudest]
            self._sort_arr = sorting
        return self._sort_arr

    @property
        if self._template_id is None:
            template_id = self.get_coincfile_array('template_id')
            self._template_id = template_id
        return self._template_id

    @property
    def trig_id(self):
        if self._trig_ids is not None:
            return self._trig_ids
        self._trig_ids = {}

        try:  # New style multi-ifo file
            ifos = self.coinc_file.h5file.attrs['ifos'].split(' ')
            for ifo in ifos:
                trigid = self.get_coincfile_array(ifo + '/trigger_id')
                self._trig_ids[ifo] = trigid
        except KeyError:  # Old style two-ifo file
            ifo1 = self.coinc_file.h5file.attrs['detector_1']
            ifo2 = self.coinc_file.h5file.attrs['detector_2']
            trigid1 = self.get_coincfile_array('trigger_id1')
            trigid2 = self.get_coincfile_array('trigger_id2')
            self._trig_ids[ifo1] = trigid1
            self._trig_ids[ifo2] = trigid2
        return self._trig_ids

    def get_coincfile_array(self, variable):
        return self.coinc_file.get_column(variable)[self.sort_arr]

    def get_bankfile_array(self, variable):
        try:
            return self.bank_file[variable][:][self.template_id]
            if len(self.template_id) == 0:
                return np.array([])
            raise

    def get_snglfile_array_dict(self, variable):
        return_dict = {}
            try:
                tid = self.trig_id[ifo]
                lgc = tid == -1
                # Put in *some* value for the invalid points to avoid failure
                # Make sure this doesn't change the cached internal array!
                tid = np.copy(tid)
                tid[lgc] = 0
                # If small number of points don't read the full file
                if len(tid) < 1000:
                    curr = []
                    hdf_dataset = self.sngl_files[ifo].group[variable]
                    for idx in tid:
                        curr.append(hdf_dataset[idx])
                    curr = np.array(curr)
                else:
                    curr = self.sngl_files[ifo].get_column(variable)[tid]
                if len(self.trig_id[ifo]) == 0:
                    curr = np.array([])
                    lgc = curr == 0
                else:
                    raise
            return_dict[ifo] = (curr, np.logical_not(lgc))
        return return_dict

    def get_end_time(self):
        try:  # First try new-style format
            ref_times = None
                times = self.get_coincfile_array('{}/time'.format(ifo))
                if ref_times is None:
                    ref_times = times
                else:
                    ref_times[ref_times < 0] = times[ref_times < 0]
        except KeyError:  # Else fall back on old two-det format
            ref_times = self.get_coincfile_array('time1')
        return ref_times

    def get_ifos(self):
        ifo_list = []
        n_trigs = len(self.get_coincfile_array('template_id'))
                ifo_trigs = np.where(self.get_coincfile_array(ifo+'/time') < 0,
                                     '-', ifo)
                ifo_list.append(ifo_trigs)
            ifo_list = [list(trig[trig != '-']) \
                        for trig in iter(np.array(ifo_list).T)]
            # Currently assumes two-det is Hanford and Livingston
            ifo_list = [['H1', 'L1'] for i in range(n_trigs)]
        return ifo_list

    def to_coinc_xml_object(self, file_name):
        outdoc = ligolw.Document()
        outdoc.appendChild(ligolw.LIGO_LW())

        ifos = list(self.sngl_files.keys())
        proc_id = ligolw_process.register_to_xmldoc(outdoc, 'pycbc',
                     {}, ifos=ifos, comment='', version=pycbc_version.git_hash,
                     cvs_repository='pycbc/'+pycbc_version.git_branch,
                     cvs_entry_time=pycbc_version.date).process_id

        search_summ_table = lsctables.New(lsctables.SearchSummaryTable)
        coinc_h5file = self.coinc_file.h5file
        try:
            start_time = coinc_h5file['segments']['coinc']['start'][:].min()
            end_time = coinc_h5file['segments']['coinc']['end'][:].max()
            start_times = []
            end_times = []
            for ifo_comb in coinc_h5file['segments']:
                if ifo_comb == 'foreground_veto':
                    continue
                seg_group = coinc_h5file['segments'][ifo_comb]
                start_times.append(seg_group['start'][:].min())
                end_times.append(seg_group['end'][:].max())
            start_time = min(start_times)
            end_time = max(end_times)
        num_trigs = len(self.sort_arr)
        search_summary = return_search_summary(start_time, end_time,
                                               num_trigs, ifos)
        search_summ_table.append(search_summary)
        outdoc.childNodes[0].appendChild(search_summ_table)

        sngl_inspiral_table = lsctables.New(lsctables.SnglInspiralTable)
        coinc_def_table = lsctables.New(lsctables.CoincDefTable)
        coinc_event_table = lsctables.New(lsctables.CoincTable)
        coinc_inspiral_table = lsctables.New(lsctables.CoincInspiralTable)
        coinc_event_map_table = lsctables.New(lsctables.CoincMapTable)
        time_slide_table = lsctables.New(lsctables.TimeSlideTable)

        # Set up time_slide table
        time_slide_id = lsctables.TimeSlideID(0)
            time_slide_row = lsctables.TimeSlide()
            time_slide_row.instrument = ifo
            time_slide_row.time_slide_id = time_slide_id
            time_slide_row.offset = 0
            time_slide_row.process_id = proc_id
            time_slide_table.append(time_slide_row)

        # Set up coinc_definer table
        coinc_def_id = lsctables.CoincDefID(0)
        coinc_def_row = lsctables.CoincDef()
        coinc_def_row.search = "inspiral"
        coinc_def_row.description = \
            "sngl_inspiral<-->sngl_inspiral coincidences"
        coinc_def_row.coinc_def_id = coinc_def_id
        coinc_def_row.search_coinc_type = 0
        coinc_def_table.append(coinc_def_row)

        bank_col_names = ['mass1', 'mass2', 'spin1z', 'spin2z']
        bank_col_vals = {}
        for name in bank_col_names:
            bank_col_vals[name] = self.get_bankfile_array(name)

        coinc_event_names = ['ifar', 'time', 'fap', 'stat']
        coinc_event_vals = {}
        for name in coinc_event_names:
            if name == 'time':
                coinc_event_vals[name] = self.get_end_time()
            else:
                coinc_event_vals[name] = self.get_coincfile_array(name)

        sngl_col_names = ['snr', 'chisq', 'chisq_dof', 'bank_chisq',
                          'bank_chisq_dof', 'cont_chisq', 'cont_chisq_dof',
                          'end_time', 'template_duration', 'coa_phase',
                          'sigmasq']
        sngl_col_vals = {}
        for name in sngl_col_names:
            sngl_col_vals[name] = self.get_snglfile_array_dict(name)

        sngl_event_count = 0
        for idx in range(len(self.sort_arr)):
            # Set up IDs and mapping values
            coinc_id = lsctables.CoincID(idx)

            # Set up sngls
            sngl_mchirps = []
            sngl_mtots = []
            net_snrsq = 0
            triggered_ifos = []
                # If this ifo is not participating in this coincidence then
                # ignore it and move on.
                if not sngl_col_vals['snr'][ifo][1][idx]:
                    continue
                triggered_ifos += [ifo]
                event_id = lsctables.SnglInspiralID(sngl_event_count)
                sngl_event_count += 1
                sngl = return_empty_sngl()
                sngl.event_id = event_id
                sngl.ifo = ifo
                net_snrsq += sngl_col_vals['snr'][ifo][0][idx]**2
                for name in sngl_col_names:
                    val = sngl_col_vals[name][ifo][0][idx]
                    if name == 'end_time':
                        sngl.set_end(LIGOTimeGPS(val))
                    else:
                        setattr(sngl, name, val)
                for name in bank_col_names:
                    val = bank_col_vals[name][idx]
                    setattr(sngl, name, val)
                sngl.mtotal, sngl.eta = pnutils.mass1_mass2_to_mtotal_eta(
                        sngl.mass1, sngl.mass2)
                sngl.mchirp, _ = pnutils.mass1_mass2_to_mchirp_eta(
                sngl.eff_distance = (sngl.sigmasq)**0.5 / sngl.snr
                # If exact match is not used, then take mean
                # masses from the single triggers
                sngl_mchirps += [sngl.mchirp]
                sngl_mtots += [sngl.mtotal]

                sngl_inspiral_table.append(sngl)

                # Set up coinc_map entry
                coinc_map_row = lsctables.CoincMap()
                coinc_map_row.table_name = 'sngl_inspiral'
                coinc_map_row.coinc_event_id = coinc_id
                coinc_map_row.event_id = event_id
                coinc_event_map_table.append(coinc_map_row)

            sngl_combined_mchirp = np.mean(sngl_mchirps)
            sngl_combined_mtot = np.mean(sngl_mtots)

            # Set up coinc inspiral and coinc event tables
            coinc_event_row = lsctables.Coinc()
            coinc_inspiral_row = lsctables.CoincInspiral()
            coinc_event_row.coinc_def_id = coinc_def_id
            coinc_event_row.nevents = len(triggered_ifos)
            coinc_event_row.instruments = ','.join(triggered_ifos)
            coinc_inspiral_row.set_ifos(triggered_ifos)
            coinc_event_row.time_slide_id = time_slide_id
            coinc_event_row.process_id = proc_id
            coinc_event_row.coinc_event_id = coinc_id
            coinc_inspiral_row.coinc_event_id = coinc_id
            coinc_inspiral_row.mchirp = sngl_combined_mchirp
            coinc_inspiral_row.mass = sngl_combined_mtot
            coinc_inspiral_row.set_end(
                LIGOTimeGPS(coinc_event_vals['time'][idx])
            )
            coinc_inspiral_row.snr = net_snrsq**0.5
            coinc_inspiral_row.false_alarm_rate = coinc_event_vals['fap'][idx]
            coinc_inspiral_row.combined_far = 1./coinc_event_vals['ifar'][idx]
            # Transform to Hz
            coinc_inspiral_row.combined_far = \
                                    coinc_inspiral_row.combined_far / YRJUL_SI
            coinc_event_row.likelihood = coinc_event_vals['stat'][idx]
            coinc_inspiral_row.minimum_duration = 0.
            coinc_event_table.append(coinc_event_row)
            coinc_inspiral_table.append(coinc_inspiral_row)

        outdoc.childNodes[0].appendChild(coinc_def_table)
        outdoc.childNodes[0].appendChild(coinc_event_table)
        outdoc.childNodes[0].appendChild(coinc_event_map_table)
        outdoc.childNodes[0].appendChild(time_slide_table)
        outdoc.childNodes[0].appendChild(coinc_inspiral_table)
        outdoc.childNodes[0].appendChild(sngl_inspiral_table)

        ligolw_utils.write_filename(outdoc, file_name)

class ReadByTemplate(object):
    # default assignment to {} is OK for a variable used only in __init__
    def __init__(self, filename, bank=None, segment_name=None, veto_files=None,
                 gating_veto_windows={}):
        self.filename = filename
        self.file = h5py.File(filename, 'r')
        self.ifo = tuple(self.file.keys())[0]
        self.valid = None
        self.bank = h5py.File(bank, 'r') if bank else {}

        # Determine the segments which define the boundaries of valid times
        # to use triggers
        key = '%s/search/' % self.ifo
        s, e = self.file[key + 'start_time'][:], self.file[key + 'end_time'][:]
        self.segs = veto.start_end_to_segments(s, e).coalesce()
        if segment_name is None:
            segment_name = []
        if veto_files is None:
            veto_files = []
        for vfile, name in zip(veto_files, segment_name):
            veto_segs = veto.select_segments_by_definer(vfile, ifo=self.ifo,
                                                        segment_name=name)
            self.segs = (self.segs - veto_segs).coalesce()
        if self.ifo in gating_veto_windows:
            gating_veto = gating_veto_windows[self.ifo].split(',')
            gveto_before = float(gating_veto[0])
            gveto_after = float(gating_veto[1])
            if gveto_before > 0 or gveto_after < 0:
                raise ValueError("Gating veto window values must be negative "
                                 "before gates and positive after gates.")
            if not (gveto_before == 0 and gveto_after == 0):
                gate_times = np.unique(
                    self.file[self.ifo + '/gating/auto/time'][:])
                gating_veto_segs = veto.start_end_to_segments(gate_times + gveto_before,
                                                              gate_times + gveto_after).coalesce()
                self.segs = (self.segs - gating_veto_segs).coalesce()
        self.valid = veto.segments_to_start_end(self.segs)

    def get_data(self, col, num):
        """ Get a column of data for template with id 'num'

        col: str
            Name of column to read
        num: int
            The template id to read triggers for

        Returns
        -------
        data: numpy.ndarray
            The requested column of data
        """
        ref = self.file['%s/%s_template' % (self.ifo, col)][num]
        return self.file['%s/%s' % (self.ifo, col)][ref]

    def set_template(self, num):
        """ Set the active template to read from

        Parameters        ----------
        num: int

        Returns
        -------
        trigger_id: numpy.ndarray
            The indices of this templates triggers
        """
        self.template_num = num
        times = self.get_data('end_time', num)

        # Determine which of these template's triggers are kept after
        # applying vetoes
        if self.valid:
            self.keep = veto.indices_within_times(times, self.valid[0],
                                                  self.valid[1])
#            logging.info('applying vetoes')
        else:
            self.keep = np.arange(0, len(times))

        if self.bank != {}:
            self.param = {}
            if 'parameters' in self.bank.attrs:
                for col in self.bank.attrs['parameters']:
                    self.param[col] = self.bank[col][self.template_num]
            else:
                for col in self.bank:

        # Calculate the trigger id by adding the relative offset in self.keep
        # to the absolute beginning index of this templates triggers stored
        # in 'template_boundaries'
        trigger_id = self.keep + \
                         self.file['%s/template_boundaries' % self.ifo][num]
        return trigger_id

    def __getitem__(self, col):
        """ Return the column of data for current active template after
        applying vetoes

        col: str

        Returns
        -------
        """
        if self.template_num is None:
            raise ValueError('You must call set_template to first pick the '
                             'template to read data from')
        data = self.get_data(col, self.template_num)
        data = data[self.keep] if self.valid else data

chisq_choices = ['traditional', 'cont', 'bank', 'max_cont_trad', 'sg',
                 'max_bank_cont', 'max_bank_trad', 'max_bank_cont_trad']

def get_chisq_from_file_choice(hdfile, chisq_choice):
    f = hdfile
    if chisq_choice in ['traditional','max_cont_trad', 'max_bank_trad',
                             'max_bank_cont_trad']:
        trad_chisq = f['chisq'][:]
        # We now need to handle the case where chisq is not actually calculated
        # 0 is used as a sentinel value
        trad_chisq_dof = f['chisq_dof'][:]
        trad_chisq /= (trad_chisq_dof * 2 - 2)
    if chisq_choice in ['cont', 'max_cont_trad', 'max_bank_cont',
        cont_chisq = f['cont_chisq'][:]
        cont_chisq_dof = f['cont_chisq_dof'][:]
        cont_chisq /= cont_chisq_dof
    if chisq_choice in ['bank', 'max_bank_cont', 'max_bank_trad',
        bank_chisq = f['bank_chisq'][:]
        bank_chisq_dof = f['bank_chisq_dof'][:]
        bank_chisq /= bank_chisq_dof
    if chisq_choice == 'sg':
        chisq = f['sg_chisq'][:]
    elif chisq_choice == 'traditional':
        chisq = trad_chisq
    elif chisq_choice == 'cont':
        chisq = cont_chisq
    elif chisq_choice == 'bank':
        chisq = bank_chisq
    elif chisq_choice == 'max_cont_trad':
        chisq = np.maximum(trad_chisq, cont_chisq)
    elif chisq_choice == 'max_bank_cont':
        chisq = np.maximum(bank_chisq, cont_chisq)
    elif chisq_choice == 'max_bank_trad':
        chisq = np.maximum(bank_chisq, trad_chisq)
    elif chisq_choice == 'max_bank_cont_trad':
        chisq = np.maximum(np.maximum(bank_chisq, cont_chisq), trad_chisq)
    else:
        err_msg = "Do not recognize --chisq-choice %s" % chisq_choice
    return chisq

def save_dict_to_hdf5(dic, filename):
    """
    dic:
        python dictionary to be converted to hdf5 format
    filename:
        desired name of hdf5 file
    """
    with h5py.File(filename, 'w') as h5file:
        recursively_save_dict_contents_to_group(h5file, '/', dic)

def recursively_save_dict_contents_to_group(h5file, path, dic):
    """
    h5file:
        h5py file to be written to
    path:
        path within h5py file to saved dictionary
    dic:
    """
    for key, item in dic.items():
        if isinstance(item, (np.ndarray, np.int64, np.float64, str, int, float,
                             bytes, tuple, list)):
            h5file[path + str(key)] = item
        elif isinstance(item, dict):
            recursively_save_dict_contents_to_group(h5file, path + key + '/', item)
        else:
            raise ValueError('Cannot save %s type' % type(item))

def load_hdf5_to_dict(h5file, path):
    """
    h5file:
        h5py file to be loaded as a dictionary
    path:
        path within h5py file to load: '/' for the whole h5py file

    Returns
    -------
    dic:
        dictionary with hdf5 file group content
    """
    dic = {}
    for key, item in h5file[path].items():
        if isinstance(item, h5py.Dataset):
            dic[key] = item[()]
        elif isinstance(item, h5py.Group):
            dic[key] = load_hdf5_to_dict(h5file, path + key + '/')
        else:
            raise ValueError('Cannot load %s type' % type(item))
    return dic

def combine_and_copy(f, files, group):
    """ Combine the same column from multiple files and save to a third"""
    # ensure that the files input is stable for iteration order
    assert isinstance(files, (list, tuple))
    f[group] = np.concatenate([fi[group][:] if group in fi else \
                                   np.array([], dtype=np.uint32) for fi in files])

def name_all_datasets(files):
    datasets = []
    for fi in files:
        datasets += get_all_subkeys(fi, '/')
    return set(datasets)

def get_all_subkeys(grp, key):
    subkey_list = []
    subkey_start = key
    if key == '':
        grpk = grp
    else:
        grpk = grp[key]
    for sk in grpk.keys():
        path = subkey_start + '/' + sk
        if isinstance(grp[path], h5py.Dataset):
            subkey_list.append(path.lstrip('/'))
        else:
            subkey_list += get_all_subkeys(grp, path)
    # returns an empty list if there is no dataset or subgroup within the group
    return subkey_list

#
#
#                          Checkpointing utilities
#
#

def dump_state(state, fp, path=None, dsetname='state',
               protocol=pickle.HIGHEST_PROTOCOL):
    """Dumps the given state to an hdf5 file handler.

    The state is stored as a raw binary array to ``{path}/{dsetname}`` in the
    given hdf5 file handler. If a dataset with the same name and path is
    already in the file, the dataset will be resized and overwritten with the
    new state data.

    state : any picklable object
        The sampler state to dump to file. Can be the object returned by
        any of the samplers' `.state` attribute (a dictionary of dictionaries),
        or any picklable object.
    fp : h5py.File
        An open hdf5 file handler. Must have write capability enabled.
    path : str, optional
        The path (group name) to store the state dataset to. Default (None)
        will result in the array being stored to the top level.
    dsetname : str, optional
        The name of the dataset to store the binary array to. Default is
        ``state``.
    protocol : int, optional
        The protocol version to use for pickling. See the :py:mod:`pickle`
        module for more details.
    """
    memfp = BytesIO()
    pickle.dump(state, memfp, protocol=protocol)
    dump_pickle_to_hdf(memfp, fp, path=path, dsetname=dsetname)

def dump_pickle_to_hdf(memfp, fp, path=None, dsetname='state'):
    """Dumps pickled data to an hdf5 file object.

    memfp : file object
        Bytes stream of pickled data.
    """
    memfp.seek(0)
    bdata = np.frombuffer(memfp.read(), dtype='S1')
    if path is not None:
        dsetname = path + '/' + dsetname
    if dsetname not in fp:
        fp.create_dataset(dsetname, shape=bdata.shape, maxshape=(None,),
                          dtype=bdata.dtype)
    elif bdata.size != fp[dsetname].shape[0]:
        fp[dsetname].resize((bdata.size,))
    fp[dsetname][:] = bdata

def load_state(fp, path=None, dsetname='state'):
    """Loads a sampler state from the given hdf5 file object.

    The sampler state is expected to be stored as a raw bytes array which can
    be loaded by pickle.

        An open hdf5 file handler.
        The path (group name) that the state data is stored to. Default (None)
        is to read from the top level.
        The name of the dataset that the state data is stored to. Default is
    """
        fp = fp[path]
    bdata = fp[dsetname][()].tobytes()
    return pickle.load(BytesIO(bdata))

import os
import lal
from six import u as unicode
from glue.ligolw import param as ligolw_param
from pycbc import pnutils
from pycbc.results import ifo_color
from pycbc.results import source_color
from pycbc.mchirp_area import calc_probabilities

#FIXME Legacy build PSD xml helpers, delete me when we move away entirely from
# xml formats
def _build_series(series, dim_names, comment, delta_name, delta_unit):
    from glue.ligolw import array as ligolw_array
    Attributes = ligolw.sax.xmlreader.AttributesImpl
    elem = ligolw.LIGO_LW(
            Attributes({u"Name": unicode(series.__class__.__name__)}))
    if comment is not None:
        elem.appendChild(ligolw.Comment()).pcdata = comment
    elem.appendChild(ligolw.Time.from_gps(series.epoch, u"epoch"))
    elem.appendChild(ligolw_param.Param.from_pyvalue(u"f0", series.f0,
                                                     unit=u"s^-1"))
    delta = getattr(series, delta_name)
    if numpy.iscomplexobj(series.data.data):
        data = numpy.row_stack((numpy.arange(len(series.data.data)) * delta,
                             series.data.data.real, series.data.data.imag))
    else:
                                series.data.data))
    a = ligolw_array.Array.build(series.name, data, dim_names=dim_names)
    a.Unit = str(series.sampleUnits)
    dim0 = a.getElementsByTagName(ligolw.Dim.tagName)[0]
    dim0.Unit = delta_unit
    dim0.Start = series.f0
    dim0.Scale = delta
    elem.appendChild(a)
    return elem

def snr_series_to_xml(snr_series, document, sngl_inspiral_id):
    """Save an SNR time series into an XML document, in a format compatible
    with BAYESTAR.
    """
    snr_lal = snr_series.lal()
    snr_lal.name = 'snr'
    snr_lal.sampleUnits = ''
    snr_xml = _build_series(snr_lal, (u'Time', u'Time,Real,Imaginary'), None,
                            'deltaT', 's')
    snr_node = document.childNodes[-1].appendChild(snr_xml)
    eid_param = ligolw_param.Param.build(u'event_id', u'ilwd:char',
                                         sngl_inspiral_id)
    snr_node.appendChild(eid_param)

def make_psd_xmldoc(psddict, xmldoc=None):
    """Add a set of PSDs to a LIGOLW XML document. If the document is not
    given, a new one is created first.
    """
    xmldoc = ligolw.Document() if xmldoc is None else xmldoc.childNodes[0]

    # the PSDs must be children of a LIGO_LW with name "psd"
    root_name = u"psd"
    lw = xmldoc.appendChild(
        ligolw.LIGO_LW(Attributes({u"Name": root_name})))

    for instrument, psd in psddict.items():
        xmlseries = _build_series(psd, (u"Frequency,Real", u"Frequency"),
                                  None, 'deltaF', 's^-1')
        fs = lw.appendChild(xmlseries)
        fs.appendChild(ligolw_param.Param.from_pyvalue(u"instrument",
                                                       instrument))
    return xmldoc

class SingleCoincForGraceDB(object):
    """Create xml files and submit them to gracedb from PyCBC Live"""
    def __init__(self, ifos, coinc_results, **kwargs):
        """Initialize a ligolw xml representation of a zerolag trigger
        for upload from pycbc live to gracedb.

            A list of the ifos participating in this trigger.
        coinc_results: dict of values
            A dictionary of values. The format is defined in
            pycbc/events/coinc.py and matches the on disk representation
            in the hdf file for this time.
        psds: dict of FrequencySeries
            Dictionary providing PSD estimates for all involved detectors.
        low_frequency_cutoff: float
            Minimum valid frequency for the PSD estimates.
        high_frequency_cutoff: float, optional
            Maximum frequency considered for the PSD estimates. Default None.
        followup_data: dict of dicts, optional
            Dictionary providing SNR time series for each detector,
            to be used in sky localization with BAYESTAR. The format should
            be `followup_data['H1']['snr_series']`. More detectors can be
            present than given in `ifos`. If so, the extra detectors will only
            be used for sky localization.
        channel_names: dict of strings, optional
            Strain channel names for each detector.
            Will be recorded in the sngl_inspiral table.
        mc_area_args: dict of dicts, optional
            Dictionary providing arguments to be used in source probability
            estimation with pycbc/mchirp_area.py
        """
        self.template_id = coinc_results['foreground/%s/template_id' % ifos[0]]
        self.coinc_results = coinc_results

        # remember if this should be marked as HWINJ
        self.is_hardware_injection = ('HWINJ' in coinc_results
                                      and coinc_results['HWINJ'])

        # Check if we need to apply a time offset (this may be permerger)
        self.time_offset = 0
        rtoff = 'foreground/{}/time_offset'.format(ifos[0])
        if rtoff in coinc_results:
            self.time_offset = coinc_results[rtoff]

        if 'followup_data' in kwargs:
            fud = kwargs['followup_data']
            assert len({fud[ifo]['snr_series'].delta_t for ifo in fud}) == 1, \
                    "delta_t for all ifos do not match"
            self.snr_series = {ifo: fud[ifo]['snr_series'] for ifo in fud}
            usable_ifos = fud.keys()
            followup_ifos = list(set(usable_ifos) - set(ifos))

            for ifo in self.snr_series:
                self.snr_series[ifo].start_time += self.time_offset
        else:
            self.snr_series = None
            usable_ifos = ifos
            followup_ifos = []

        # Set up the bare structure of the xml document

        proc_id = ligolw_process.register_to_xmldoc(
            outdoc, 'pycbc', {}, ifos=usable_ifos, comment='',
            version=pycbc_version.version,
            cvs_repository='pycbc/'+pycbc_version.git_branch,
            cvs_entry_time=pycbc_version.date).process_id

        coinc_def_row.description = "sngl_inspiral<-->sngl_inspiral coincs"

        # Set up coinc inspiral and coinc event tables
        coinc_id = lsctables.CoincID(0)
        coinc_event_row = lsctables.Coinc()
        coinc_event_row.coinc_def_id = coinc_def_id
        coinc_event_row.nevents = len(usable_ifos)
        coinc_event_row.instruments = ','.join(usable_ifos)
        coinc_event_row.time_slide_id = lsctables.TimeSlideID(0)
        coinc_event_row.process_id = proc_id
        coinc_event_row.coinc_event_id = coinc_id
        coinc_event_row.likelihood = 0.
        coinc_event_table.append(coinc_event_row)

        # Set up sngls

        sngl_populated = None
        network_snrsq = 0
        for sngl_id, ifo in enumerate(usable_ifos):
            sngl = return_empty_sngl(nones=True)
            sngl.event_id = lsctables.SnglInspiralID(sngl_id)
            sngl.process_id = proc_id
            sngl.ifo = ifo
            names = [n.split('/')[-1] for n in coinc_results
                     if 'foreground/%s' % ifo in n]
            for name in names:
                val = coinc_results['foreground/%s/%s' % (ifo, name)]
                if name == 'end_time':
                    val += self.time_offset
                    sngl.set_end(lal.LIGOTimeGPS(val))
                else:
                    try:
                    except AttributeError:
                        pass
            if sngl.mass1 and sngl.mass2:
                sngl_populated = sngl
            if sngl.snr:
                network_snrsq += sngl.snr ** 2.0
            if 'channel_names' in kwargs and ifo in kwargs['channel_names']:
                sngl.channel = kwargs['channel_names'][ifo]
            sngl_inspiral_table.append(sngl)

            # Set up coinc_map entry
            coinc_map_row = lsctables.CoincMap()
            coinc_map_row.table_name = 'sngl_inspiral'
            coinc_map_row.coinc_event_id = coinc_id
            coinc_map_row.event_id = sngl.event_id
            coinc_event_map_table.append(coinc_map_row)

            if self.snr_series is not None:
                snr_series_to_xml(self.snr_series[ifo], outdoc, sngl.event_id)

        # set merger time to the average of the ifo peaks
        self.merger_time = numpy.mean(
                    [coinc_results['foreground/{}/end_time'.format(ifo)]
                     for ifo in ifos]) + self.time_offset

        # for subthreshold detectors, respect BAYESTAR's assumptions and checks
        bayestar_check_fields = ('mass1 mass2 mtotal mchirp eta spin1x '
                                 'spin1y spin1z spin2x spin2y spin2z').split()
        for sngl in sngl_inspiral_table:
            if sngl.ifo in followup_ifos:
                for bcf in bayestar_check_fields:
                    setattr(sngl, bcf, getattr(sngl_populated, bcf))
                sngl.set_end(lal.LIGOTimeGPS(self.merger_time))

        # Set up the coinc inspiral table
        coinc_inspiral_row = lsctables.CoincInspiral()
        # This seems to be used as FAP, which should not be in gracedb
        coinc_inspiral_row.false_alarm_rate = 0
        coinc_inspiral_row.minimum_duration = 0.
        coinc_inspiral_row.set_ifos(usable_ifos)
        coinc_inspiral_row.coinc_event_id = coinc_id
        coinc_inspiral_row.mchirp = sngl_populated.mchirp
        coinc_inspiral_row.mass = sngl_populated.mtotal
        coinc_inspiral_row.end_time = sngl_populated.end_time
        coinc_inspiral_row.end_time_ns = sngl_populated.end_time_ns
        coinc_inspiral_row.snr = network_snrsq ** 0.5
        far = 1.0 / (lal.YRJUL_SI * coinc_results['foreground/ifar'])
        coinc_inspiral_row.combined_far = far
        coinc_inspiral_table.append(coinc_inspiral_row)

        # append the PSDs
        self.psds = kwargs['psds']
        psds_lal = {}
        for ifo in self.psds:
            psd = self.psds[ifo]
            kmin = int(kwargs['low_frequency_cutoff'] / psd.delta_f)
            fseries = lal.CreateREAL8FrequencySeries(
                "psd", psd.epoch, kwargs['low_frequency_cutoff'], psd.delta_f,
                lal.StrainUnit**2 / lal.HertzUnit, len(psd) - kmin)
            fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC ** 2.0
            psds_lal[ifo] = fseries
        make_psd_xmldoc(psds_lal, outdoc)

        # source probabilities estimation
        if 'mc_area_args' in kwargs:
            eff_distances = [sngl.eff_distance for sngl in sngl_inspiral_table]
            probabilities = calc_probabilities(coinc_inspiral_row.mchirp,
                                               coinc_inspiral_row.snr,
                                               min(eff_distances),
                                               kwargs['mc_area_args'])
            self.probabilities = probabilities
        else:
            self.probabilities = None

        self.outdoc = outdoc
        self.time = sngl_populated.get_end()

    def save(self, filename):
        """Write this trigger to gracedb compatible xml format

        filename: str
            Name of file to write to disk.
        """
        gz = filename.endswith('.gz')
        ligolw_utils.write_filename(self.outdoc, filename, gz=gz)

        # save source probabilities in a json file
        if self.probabilities is not None:
            prob_fname = filename.replace('.xml.gz', '_probs.json')
            with open(prob_fname, 'w') as prob_outfile:
                json.dump(self.probabilities, prob_outfile)
            logging.info('Source probabilities file saved as %s', prob_fname)

    def upload(self, fname, gracedb_server=None, testing=True,
               extra_strings=None, search='AllSky'):
        """Upload this trigger to gracedb

        fname: str
            The name to give the xml file associated with this trigger
        gracedb_server: string, optional
            URL to the GraceDB web API service for uploading the event.
            If omitted, the default will be used.
        testing: bool
            Switch to determine if the upload should be sent to gracedb as a
            test trigger (True) or a production trigger (False).
        search: str
            String going into the "search" field of the GraceDB event.
        """
        from ligo.gracedb.rest import GraceDb
        import matplotlib
        matplotlib.use('Agg')
        import pylab as pl

        # first of all, make sure the event is saved on disk
        # as GraceDB operations can fail later
        self.save(fname)

        if self.snr_series is not None:
            if fname.endswith('.xml.gz'):
                snr_series_fname = fname.replace('.xml.gz', '.hdf')
            else:
                snr_series_fname = fname.replace('.xml', '.hdf')
            snr_series_plot_fname = snr_series_fname.replace('.hdf',
                                                             '_snr.png')
            psd_series_plot_fname = snr_series_fname.replace('.hdf',
                                                             '_psd.png')
            pl.figure()
            ref_time = int(self.merger_time)
            for ifo in sorted(self.snr_series):
                curr_snrs = self.snr_series[ifo]
                curr_snrs.save(snr_series_fname, group='%s/snr' % ifo)
                pl.plot(curr_snrs.sample_times - ref_time, abs(curr_snrs),
                        c=ifo_color(ifo), label=ifo)
                if ifo in self.ifos:
                    base = 'foreground/{}/'.format(ifo)
                    snr = self.coinc_results[base + 'snr']
                    mt = (self.coinc_results[base + 'end_time']
                          + self.time_offset)
                    pl.plot([mt - ref_time], [snr], c=ifo_color(ifo),
                            marker='x')

            pl.legend()
            pl.xlabel('GPS time from {:d} (s)'.format(ref_time))
            pl.ylabel('SNR')
            pl.savefig(snr_series_plot_fname)
            pl.close()

                # Undo dynamic range factor
                curr_psd = self.psds[ifo].astype(numpy.float64)
                curr_psd /= pycbc.DYN_RANGE_FAC ** 2.0
                curr_psd.save(snr_series_fname, group='%s/psd' % ifo)
                # Can't plot log(0) so start from point 1
                pl.loglog(curr_psd.sample_frequencies[1:],
                          curr_psd[1:]**0.5, c=ifo_color(ifo), label=ifo)
            pl.xlim([10, 1300])
            pl.ylim([3E-24, 1E-20])
            pl.xlabel('Frequency (Hz)')
            pl.ylabel('ASD')
            pl.savefig(psd_series_plot_fname)

            prob_fname = fname.replace('.xml.gz', '_probs.json')
            prob_plot_fname = prob_fname.replace('.json', '.png')

            prob_plot = {k: v for (k, v) in self.probabilities.items()
                         if v != 0.0}
            labels, sizes = zip(*prob_plot.items())
            colors = [source_color(label) for label in labels]
            fig, ax = pl.subplots()
            ax.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%',
                   textprops={'fontsize': 15})
            ax.axis('equal')
            fig.savefig(prob_plot_fname)

        gid = None
        try:
            # try connecting to GraceDB
            gracedb = GraceDb(gracedb_server) \
                    if gracedb_server is not None else GraceDb()

            # create GraceDB event
            group = 'Test' if testing else 'CBC'
            r = gracedb.createEvent(group, "pycbc", fname, search).json()
            gid = r["graceid"]
            logging.info("Uploaded event %s", gid)

            if self.is_hardware_injection:
                gracedb.writeLabel(gid, 'INJ')
                logging.info("Tagging event %s as an injection", gid)

            # upload PSDs. Note that the PSDs are already stored in the
            # original event file and we just upload a copy of that same file
            # here. This keeps things as they were in O2 and can be removed
            # after updating the follow-up infrastructure
            psd_fname = 'psd.xml.gz' if fname.endswith('.gz') else 'psd.xml'
            gracedb.writeLog(gid, "PyCBC PSD estimate from the time of event",
                             psd_fname, open(fname, "rb").read(), "psd")
            logging.info("Uploaded PSDs for event %s", gid)

            # add info for tracking code version
            gracedb_tag_with_version(gracedb, gid)

            extra_strings = [] if extra_strings is None else extra_strings
            for text in extra_strings:
                gracedb.writeLog(gid, text, tag_name=['analyst_comments'])

            # upload SNR series in HDF format and plots
                gracedb.writeLog(gid, 'SNR timeseries HDF file upload',
                                 filename=snr_series_fname)
                gracedb.writeLog(gid, 'SNR timeseries plot upload',
                                 filename=snr_series_plot_fname,
                                 tag_name=['background'],
                                 displayName=['SNR timeseries'])
                gracedb.writeLog(gid, 'PSD plot upload',
                                 filename=psd_series_plot_fname,
                                 tag_name=['psd'], displayName=['PSDs'])

            # upload source probabilities in json format and plot
            if self.probabilities is not None:
                gracedb.writeLog(gid, 'source probabilities JSON file upload',
                                 filename=prob_fname, tag_name=['em_follow'])
                logging.info('Uploaded source probabilities for event %s', gid)
                gracedb.writeLog(gid, 'source probabilities plot upload',
                                 filename=prob_plot_fname,
                                 tag_name=['em_follow'])
                logging.info('Uploaded source probabilities pie chart for '
                             'event %s', gid)

        except Exception as exc:
            logging.error('Something failed during the upload/annotation of '
                          'event %s on GraceDB. The event may not have been '
                          'uploaded!', fname)
            logging.error(str(exc))

        return gid

def gracedb_tag_with_version(gracedb, event_id):
    """Add a GraceDB log entry reporting PyCBC's version and install location.
    """
    version_str = 'Using PyCBC version {}{} at {}'
    version_str = version_str.format(
            pycbc_version.version,
            ' (release)' if pycbc_version.release else '',
            os.path.dirname(pycbc.__file__))
    gracedb.writeLog(event_id, version_str)

__all__ = ['SingleCoincForGraceDB', 'make_psd_xmldoc', 'snr_series_to_xml',
           'gracedb_tag_with_version']

# Copyright (C) 2015  Collin Capano
#
#

#
#
#                           Preamble
#
#
"""
This modules provides definitions of, and helper functions for, FieldArray.
FieldArray are wrappers of numpy recarrays with additional functionality
useful for storing and retrieving data created by a search for gravitationa
waves.
"""

import os, sys, types, re, copy, numpy, inspect
import six
from glue.ligolw import types as ligolw_types
from pycbc import coordinates, conversions, cosmology

# what functions are given to the eval in FieldArray's __getitem__:
_numpy_function_lib = {_x: _y for _x,_y in numpy.__dict__.items()
                       if isinstance(_y, (numpy.ufunc, float))}

#
#
#                           Data type mappings
#
#
# add ligolw_types to numpy typeDict
numpy.typeDict.update(ligolw_types.ToNumPyType)

# Annoyingly, numpy has no way to store NaNs in an integer field to indicate
# the equivalent of None. This can be problematic for fields that store ids:
# if an array has an id field with value 0, it isn't clear if this is because
# the id is the first element, or if no id was set. To clear up the ambiguity,
# we define here an integer to indicate 'id not set'.
ID_NOT_SET = -1
EMPTY_OBJECT = None
VIRTUALFIELD_DTYPE = 'VIRTUAL'

def set_default_empty(array):
    if array.dtype.names is None:
        # scalar dtype, just set
        if array.dtype.str[1] == 'i':
            # integer, set to ID_NOT_SET
            array[:] = ID_NOT_SET
        elif array.dtype.str[1] == 'O':
            # object, set to EMPTY_OBJECT
            array[:] = EMPTY_OBJECT
    else:
        for name in array.dtype.names:
            set_default_empty(array[name])

def default_empty(shape, dtype):
    """Numpy's empty array can have random values in it. To prevent that, we
    define here a default emtpy array. This default empty is a numpy.zeros
    array, except that objects are set to None, and all ints to ID_NOT_SET.
    """
    default = numpy.zeros(shape, dtype=dtype)
    set_default_empty(default)
    return default

# set default data types
_default_types_status = {
    'default_strlen': 50,
    'ilwd_as_int': True,
    'lstring_as_obj': False
}

def lstring_as_obj(true_or_false=None):
    """Toggles whether lstrings should be treated as strings or as objects.
    When FieldArrays is first loaded, the default is True.

    true_or_false : {None|bool}
        Pass True to map lstrings to objects; False otherwise. If None
        provided, just returns the current state.

    Return
    ------
    current_stat : bool
        The current state of lstring_as_obj.

    Examples
    --------
    >>> from pycbc.io import FieldArray
    >>> FieldArray.lstring_as_obj()
        True
    >>> FieldArray.FieldArray.from_arrays([numpy.zeros(10)], dtype=[('foo', 'lstring')])
    FieldArray([(0.0,), (0.0,), (0.0,), (0.0,), (0.0,), (0.0,), (0.0,), (0.0,),
           (0.0,), (0.0,)],
          dtype=[('foo', 'O')])
    >>> FieldArray.lstring_as_obj(False)
        False
    FieldArray([('0.0',), ('0.0',), ('0.0',), ('0.0',), ('0.0',), ('0.0',),
           ('0.0',), ('0.0',), ('0.0',), ('0.0',)],
          dtype=[('foo', 'S50')])
    """
    if true_or_false is not None:
        _default_types_status['lstring_as_obj'] = true_or_false
        # update the typeDict
        numpy.typeDict[u'lstring'] = numpy.object_ \
            if _default_types_status['lstring_as_obj'] \
            else 'S%i' % _default_types_status['default_strlen']
    return _default_types_status['lstring_as_obj']

def ilwd_as_int(true_or_false=None):
    """Similar to lstring_as_obj, sets whether or not ilwd:chars should be
    treated as strings or as ints. Default is True.
    """
        _default_types_status['ilwd_as_int'] = true_or_false
        numpy.typeDict[u'ilwd:char'] = int \
            if _default_types_status['ilwd_as_int'] \
            else 'S%i' % default_strlen
    return _default_types_status['ilwd_as_int']

def default_strlen(strlen=None):
    """Sets the default string length for lstring and ilwd:char, if they are
    treated as strings. Default is 50.
    """
    if strlen is not None:
        _default_types_status['default_strlen'] = strlen
        # update the typeDicts as needed
        lstring_as_obj(_default_types_status['lstring_as_obj'])
        ilwd_as_int(_default_types_status['ilwd_as_int'])
    return _default_types_status['default_strlen']

# set the defaults
lstring_as_obj(True)
ilwd_as_int(True)

#
#
#
#

#
#   Argument syntax parsing
#
# this parser will pull out sufields as separate identifiers from their parent
# field; e.g., foo.bar --> ['foo', 'bar']
_pyparser = re.compile(r'(?P<identifier>[\w_][\w\d_]*)')
# this parser treats subfields as one identifier with their parent field;
# e.g., foo.bar --> ['foo.bar']
_fieldparser = re.compile(r'(?P<identifier>[\w_][.\w\d_]*)')
def get_vars_from_arg(arg):
    """Given a python string, gets the names of any identifiers use in it.
    For example, if ``arg = '3*narf/foo.bar'``, this will return
    ``set(['narf', 'foo', 'bar'])``.
    """
    return set(_pyparser.findall(arg))

def get_fields_from_arg(arg):
    """Given a python string, gets FieldArray field names used in it. This
    differs from get_vars_from_arg in that any identifier with a '.' in it
    will be treated as one identifier. For example, if
    ``arg = '3*narf/foo.bar'``, this will return ``set(['narf', 'foo.bar'])``.
    """
    return set(_fieldparser.findall(arg))

# this parser looks for fields inside a class method function. This is done by
# looking for variables that start with self.{x} or self["{x}"]; e.g.,
# self.a.b*3 + self.c, self['a.b']*3 + self.c, self.a.b*3 + self["c"], all
# return set('a.b', 'c').
_instfieldparser = re.compile(
    r'''self(?:\.|(?:\[['"]))(?P<identifier>[\w_][.\w\d_]*)''')
def get_instance_fields_from_arg(arg):
    """Given a python string definining a method function on an instance of an
    FieldArray, returns the field names used in it. This differs from
    get_fields_from_arg in that it looks for variables that start with 'self'.
    """
    return set(_instfieldparser.findall(arg))

def get_needed_fieldnames(arr, names):
    """Given a FieldArray-like array and a list of names, determines what
    fields are needed from the array so that using the names does not result
    in an error.

    arr : instance of a FieldArray or similar
        The array from which to determine what fields to get.
    names : (list of) strings
        A list of the names that are desired. The names may be either a field,
        a virtualfield, a property, a method of ``arr``, or any function of
        these. If a virtualfield/property or a method, the source code of that
        property/method will be analyzed to pull out what fields are used in
        it.

    Returns
    -------
    set
        The set of the fields needed to evaluate the names.
    """
    fieldnames = set([])
    # we'll need the class that the array is an instance of to evaluate some
    # things
    cls = arr.__class__
    if isinstance(names, string_types):
        names = [names]
    # parse names for variables, incase some of them are functions of fields
    parsed_names = set([])
    for name in names:
        parsed_names.update(get_fields_from_arg(name))
    # only include things that are in the array's namespace
    names = list(parsed_names & (set(dir(arr)) | set(arr.fieldnames)))
        if name in arr.fieldnames:
            # is a field, just add the name
            fieldnames.update([name])
        else:
            # the name is either a virtualfield, a method, or some other
            # property; we need to evaluate the source code to figure out what
            # fields we need
            try:
                # the underlying functions of properties need to be retrieved
                # using their fget attribute
                func = getattr(cls, name).fget
                # no fget attribute, assume is an instance method
                func = getattr(arr, name)
            # evaluate the source code of the function
            try:
                sourcecode = inspect.getsource(func)
            except TypeError:
                # not a function, just pass
                continue
            # evaluate the source code for the fields
            possible_fields = get_instance_fields_from_arg(sourcecode)
            # some of the variables returned by possible fields may themselves
            # be methods/properties that depend on other fields. For instance,
            # mchirp relies on eta and mtotal, which each use mass1 and mass2;
            # we therefore need to anayze each of the possible fields
            fieldnames.update(get_needed_fieldnames(arr, possible_fields))
    return fieldnames

def get_dtype_descr(dtype):
    """Numpy's ``dtype.descr`` will return empty void fields if a dtype has
    offsets specified. This function tries to fix that by not including
    fields that have no names and are void types.
    """
    dts = []
    for dt in dtype.descr:
        if (dt[0] == '' and dt[1][1] == 'V'):
            continue

        # Downstream codes (numpy, etc) can't handle metadata in dtype
        if isinstance(dt[1], tuple):
            dt = (dt[0], dt[1][0])

        dts.append(dt)
    return dts

def combine_fields(dtypes):
    """Combines the fields in the list of given dtypes into a single dtype.

    dtypes : (list of) numpy.dtype(s)
        Either a numpy.dtype, or a list of numpy.dtypes.

    Returns
    -------
    numpy.dtype
        A new dtype combining the fields in the list of dtypes.
    """
    if not isinstance(dtypes, list):
        dtypes = [dtypes]
    # Note: incase any of the dtypes have offsets, we won't include any fields
    # that have no names and are void
    new_dt = numpy.dtype([dt for dtype in dtypes \
        for dt in get_dtype_descr(dtype)])
    return new_dt

def _ensure_array_list(arrays):
    """Ensures that every element in a list is an instance of a numpy array."""
    # Note: the isinstance test is needed below so that instances of FieldArray
    # are not converted to numpy arrays
    return [numpy.array(arr, ndmin=1) if not isinstance(arr, numpy.ndarray)
            else arr for arr in arrays]

def merge_arrays(merge_list, names=None, flatten=True, outtype=None):
    """Merges the given arrays into a single array. The arrays must all have
    the same shape. If one or more of the given arrays has multiple fields,
    all of the fields will be included as separate fields in the new array.

    merge_list : list of arrays
        The list of arrays to merge.
    names : {None | sequence of strings}
        Optional, the names of the fields in the output array. If flatten is
        True, must be the same length as the total number of fields in
        merge_list.  Otherise, must be the same length as the number of
        arrays in merge_list.  If None provided, and flatten is True, names
        used will be the same as the name of the fields in the given arrays.
        If the datatype has no name, or flatten is False, the new field will
        be `fi` where i is the index of the array in arrays.
    flatten : bool
        Make all of the fields in the given arrays separate fields in the
        new array. Otherwise, each array will be added as a field. If an
        array has fields, they will be subfields in the output array. Default
        is True.
    outtype : {None | class}
        Cast the new array to the given type. Default is to return a
        numpy structured array.

    Returns
    -------
    new array : {numpy.ndarray | outtype}
        A new array with all of the fields in all of the arrays merged into
        a single array.
    """
    # make sure everything in merge_list is an array
    merge_list = _ensure_array_list(merge_list)
    if not all(merge_list[0].shape == arr.shape for arr in merge_list):
        raise ValueError("all of the arrays in merge_list must have the " +
            "same shape")
    if flatten:
        new_dt = combine_fields([arr.dtype for arr in merge_list])
    else:
        new_dt = numpy.dtype([('f%i' %ii, arr.dtype.descr) \
            for ii,arr in enumerate(merge_list)])
    new_arr = merge_list[0].__class__(merge_list[0].shape, dtype=new_dt)
    # ii is a counter to keep track of which fields from the new array
    # go with which arrays in merge list
    ii = 0
    for arr in merge_list:
        if arr.dtype.names is None:
            new_arr[new_dt.names[ii]] = arr
            ii += 1
        else:
            for field in arr.dtype.names:
                new_arr[field] = arr[field]
                ii += 1
    # set the names if desired
    if names is not None:
        new_arr.dtype.names = names
    # ditto the outtype
    if outtype is not None:
        new_arr = new_arr.view(type=outtype)
    return new_arr

def add_fields(input_array, arrays, names=None, assubarray=False):
    """Adds the given array(s) as new field(s) to the given input array.
    Returns a new instance of the input_array with the new fields added.

    input_array : instance of a numpy.ndarray or numpy recarray
        The array to to add the fields to.
    arrays : (list of) numpy array(s)
        The arrays to add. If adding multiple arrays, must be a list;
        if adding a single array, can just be that array.
        Optional, the name(s) of the new fields in the output array. If
        adding multiple fields, must be a list of strings with the same
        length as the list of arrays. If None provided, names used will
        be the same as the name of the datatype in the given arrays.
        If the datatype has no name, the new field will be ``'fi'`` where
        i is the index of the array in arrays.
    assubarray : bool
        Add the list of arrays as a single subarray field. If True, and names
        provided, names should be a string or a length-1 sequence. Default is
        False, in which case each array will be added as a separate field.

    Returns
    -------
    new_array : new instance of `input_array`
        A copy of the `input_array` with the desired fields added.
    """
    if not isinstance(arrays, list):
        arrays = [arrays]
    # ensure that all arrays in arrays are arrays
    arrays = _ensure_array_list(arrays)
    # set the names
        if isinstance(names, string_types):
            names = [names]
        # check if any names are subarray names; if so, we have to add them
        # separately
        subarray_names = [name for name in names if len(name.split('.')) > 1]
    else:
        subarray_names = []
    if any(subarray_names):
        subarrays = [arrays[ii] for ii,name in enumerate(names) \
            if name in subarray_names]
        # group together by subarray
        groups = {}
        for name,arr in zip(subarray_names, subarrays):
            key = name.split('.')[0]
            subkey = '.'.join(name.split('.')[1:])
            try:
                groups[key].append((subkey, arr))
                groups[key] = [(subkey, arr)]
        # now cycle over the groups, adding all of the fields in each group
        # as a subarray
        for group_name in groups:
            # we'll create a dictionary out of the subarray field names ->
            # subarrays
            thisdict = dict(groups[group_name])
            # check if the input array has this field; if so, remove it, then
            # add it back with the other new arrays
            if group_name in input_array.fieldnames:
                # get the data
                new_subarray = input_array[group_name]
                # add the new fields to the subarray
                new_subarray = add_fields(new_subarray, thisdict.values(),
                    thisdict.keys())
                # remove the original from the input array
                input_array = input_array.without_fields(group_name)
            else:
                new_subarray = thisdict.values()
            # add the new subarray to input_array as a subarray
            input_array = add_fields(input_array, new_subarray,
                names=group_name, assubarray=True)
            # set the subarray names
            input_array[group_name].dtype.names = thisdict.keys()
        # remove the subarray names from names
        keep_idx = [ii for ii,name in enumerate(names) \
            if name not in subarray_names]
        names = [names[ii] for ii in keep_idx]
        # if there's nothing left, just return
        if names == []:
            return input_array
        # also remove the subarray arrays
        arrays = [arrays[ii] for ii in keep_idx]
    if assubarray:
        # merge all of the arrays into a single array
        if len(arrays) > 1:
            arrays = [merge_arrays(arrays, flatten=True)]
        # now merge all the fields as a single subarray
        merged_arr = numpy.empty(len(arrays[0]),
            dtype=[('f0', arrays[0].dtype.descr)])
        merged_arr['f0'] = arrays[0]
        arrays = [merged_arr]
    merge_list = [input_array] + arrays
        names = list(input_array.dtype.names) + names
    # merge into a single array
    return merge_arrays(merge_list, names=names, flatten=True,
        outtype=type(input_array))

#
#
#                           Base FieldArray definitions
#
#

# We'll include functions in various pycbc modules in FieldArray's function
# library. All modules used must have an __all__ list defined.
_modules_for_functionlib = [conversions, coordinates, cosmology]
_fieldarray_functionlib = {_funcname : getattr(_mod, _funcname)
                              for _mod in _modules_for_functionlib
                              for _funcname in getattr(_mod, '__all__')}

class FieldArray(numpy.recarray):
    """
    Subclass of numpy.recarray that adds additional functionality.

    Initialization is done the same way as numpy.recarray, with the addition
    that a "name" attribute can be passed to name the output array. When you
    initialize an array it creates a new zeroed array. This is similar to
    numpy.recarray, except that ``numpy.recarray(shape)`` will create an empty
    array, whereas here the default is to zero all of the elements (see
    ``default_zero`` for definition of zero for different data types). If you
    prefer an empty array, set ``zero=False`` when initializing.

    You cannot pass an array or sequence as input as you do with numpy.array.
    To initialize an FieldArray from an already existing arrays, use the
    ``FieldArray.from_arrays`` class method. To initialize from a list of
    tuples, use ``FieldArray.from_records``. See the docstring for those methods
    for details. For more information on initalizing an empty array, see
    ``numpy.recarray`` help.

    Some additional features:

    * **Arbitrary functions**:

      You can retrive functions on fields in the same manner that you access
      individual fields. For example, if you have a FieldArray ``x`` with
      fields ``a`` and ``b``, you can access each field with
      ``x['a'], x['b']``.  You can also do ``x['a*b/(a+b)**2.']``,
      ``x[cos(a)*sin(b)]``, etc. Boolean operations are also possible, e.g.,
      ``x['(a < 3) & (b < 2)']``. Syntax for functions is python. Any numpy
      ufunc, as well as all functions listed in the functionlib attribute, may
      be used. Note that while fields may be accessed as attributes (e.g,
      field ``a`` can be accessed via ``x['a']`` or ``x.a``), functions on
      multiple fields may not (``x.a+b`` does not work, for obvious reasons).

    * **Subfields and '.' indexing**:
      Structured arrays, which are the base class for recarrays and, by
      inheritance, FieldArray, allows for fields to themselves have fields. For
      example, an array ``x`` may have fields ``a`` and ``b``, with ``b`` having
      subfields ``c`` and ``d``. You can access subfields using other index
      notation or attribute notation. So, the subfields ``d`` may be retrieved
      via ``x['b']['d']``, ``x.b.d``, ``x['b'].d`` or ``x['b.d']``. Likewise,
      functions can be carried out on the subfields, as they can on fields. If
      ``d`` is a float field, we could get the log of it via ``x['log(b.d)']``.
      There is no limit to the number of subfields. So, ``c`` could also have
      subfield ``c0``, which would be accessed via ``x.c.c0``, or any of the
      other methods.

    .. warning::
       Record arrays also allow you to set values of a field using attribute
       notation. However, this can lead to unexpected results if you
       accidently misspell the attribute. For example, if ``x`` has field
       ``foo``, and you misspell this when setting, e.g., you try to do
       ``x.fooo = numpy.arange(x.size)``, ``foo`` will not be set, nor will
       you get an error. Instead, the attribute ``fooo`` will be added to
       ``x``. If you tried to do this using index notation, however ---
       ``x['fooo'] = numpy.arange(x.size)`` --- you will
       get an ``AttributeError`` as you might expect. For this reason, it is
       recommended that you always use index notation when *setting* values;
       you can use either index or attribute notation when *retrieving*
       values.

    * **Properties and methods as fields**:
      If a propety or instance method is defined for a class that inherits from
      FieldArray, those can be accessed in the same way as fields are. For
      example, define ``Foo`` as:

    .. code-block:: python

        class Foo(FieldArray):
            @property
            def bar(self):
                return self['a']**2.

            def narf(self, y):
                return self['a'] + y

    Then if we have an instance: ``foo = Foo(100, dtype=[('a', float)])``.
    The ``bar`` and ``narf`` attributes may be accessed via field notation:
    ``foo.bar``, ``foo['bar']``, ``foo.narf(10)`` and ``foo['narf(10)']``.

    * **Virtual fields**:
      Virtual fields are methods wrapped as properties that operate on one or
      more fields, thus returning an array of values. To outside code virtual
      fields look the same as fields, and can be called similarily. Internally,
      no additional data is stored; the operation is performed on the fly when
      the virtual field is called. Virtual fields can be added to an array
      instance with the add_virtualfields method. Alternatively, virtual fields
      can be defined by sub-classing FieldArray:

            _virtualfields = ['bar']
            @property

    The fields property returns the names of both fields and virtual fields.

    .. note::

        It can happen that a field, virtual field, or function in the
        functionlib have that same name. In that case, precedence is: field,
        virtual field, function. For example, if a function called 'foo' is in
        the function library, and a virtual field is added call 'foo', then
        `a['foo']` will return the virtual field rather than the function.
        Likewise, if the array is initialized with a field called `foo`, or a
        field with that name is added, `a['foo']` will return that field
        rather than the virtual field and/or the function.

    shape : {int | tuple}
        The shape of the new array.
    name : {None | str}
        Optional, what to name the new array. The array's ``name`` attribute
        is set to this.

    For details on other keyword arguments, see ``numpy.recarray`` help.

    name : str
        Instance attribute. The name of the array.

    Examples
    --------
    .. note:: For some predefined arrays with default fields, see the other
        array classes defined below.

    Create an empty array with four rows and two fields called `foo` and
    `bar`, both of which are floats:

    >>> x = FieldArray(4, dtype=[('foo', float), ('bar', float)])

    Set/retrieve a fields using index or attribute syntax:

    >>> x['foo'] = [1.,2.,3.,4.]
    >>> x['bar'] = [5.,6.,7.,8.]
    >>> x
    FieldArray([(1.0, 5.0), (2.0, 6.0), (3.0, 7.0), (4.0, 8.0)],
          dtype=[('foo', '<f8'), ('bar', '<f8')])
    >>> x.foo
        array([ 1.,  2.,  3.,  4.])
    >>> x['bar']
        array([ 5.,  6.,  7.,  8.])

    Get the names of the fields:

    >>> x.fieldnames
        ('foo', 'bar')

    Rename the fields to `a` and `b`:

    >>> x.dtype.names = ['a', 'b']
        ('a', 'b')

    Retrieve a function of the fields as if it were a field:

    >>> x['sin(a/b)']
    array([ 0.19866933,  0.3271947 ,  0.41557185,  0.47942554])

    Add a virtual field:

    >>> def c(self):
    ...     return self['a'] + self['b']
    ...
    >>> x = x.add_virtualfields('c', c)
    >>> x.fields
    ('a', 'b', 'c')
    >>> x['c']
    array([  6.,   8.,  10.,  12.])

    Create an array with subfields:

    >>> x = FieldArray(4, dtype=[('foo', [('cat', float), ('hat', int)]), ('bar', float)])
        ['foo.cat', 'foo.hat', 'bar']

    Load from a list of arrays (in this case, from an hdf5 file):

    >>> bankhdf = h5py.File('bank/H1L1-BANK2HDF-1117400416-928800.hdf')
    >>> bankhdf.keys()
        [u'mass1', u'mass2', u'spin1z', u'spin2z', u'template_hash']
    >>> templates = FieldArray.from_arrays(bankhdf.values(), names=bankhdf.keys())
    >>> templates.fieldnames
        ('mass1', 'mass2', 'spin1z', 'spin2z', 'template_hash')
    >>> templates.mass1
    array([ 1.71731389,  1.10231435,  2.99999857, ...,  1.67488706,
            1.00531888,  2.11106491], dtype=float32)

    Sort by a field without having to worry about also sorting the other
    fields:

    >>> templates[['mass1', 'mass2']]
    array([(1.7173138856887817, 1.2124452590942383),
           (1.1023143529891968, 1.0074082612991333),
           (2.9999985694885254, 1.0578444004058838), ...,
           (1.6748870611190796, 1.1758257150650024),
           (1.0053188800811768, 1.0020891427993774),
           (2.111064910888672, 1.0143394470214844)],
          dtype=[('mass1', '<f4'), ('mass2', '<f4')])
    >>> templates.sort(order='mass1')
    array([(1.000025987625122, 1.0000133514404297),
           (1.0002814531326294, 1.0002814531326294),
           (1.0005437135696411, 1.0005437135696411), ...,
           (2.999999523162842, 1.371169090270996),
           (2.999999523162842, 1.4072519540786743), (3.0, 1.4617927074432373)],

    Convert a LIGOLW xml table:

    >>> type(sim_table)
        glue.ligolw.lsctables.SimInspiralTable
    >>> sim_array = FieldArray.from_ligolw_table(sim_table)
    >>> sim_array.mass1
    array([ 2.27440691,  1.85058105,  1.61507106, ...,  2.0504961 ,
            2.33554196,  2.02732205], dtype=float32)
    >>> sim_array.waveform
    array([u'SpinTaylorT2', u'SpinTaylorT2', u'SpinTaylorT2', ...,
           u'SpinTaylorT2', u'SpinTaylorT2', u'SpinTaylorT2'], dtype=object)

    >>> sim_array = FieldArray.from_ligolw_table(sim_table, columns=['simulation_id', 'mass1', 'mass2'])
    >>> sim_array
    FieldArray([(0, 2.274406909942627, 2.6340370178222656),
           (1, 1.8505810499191284, 2.8336880207061768),
           (2, 1.6150710582733154, 2.2336490154266357), ...,
           (11607, 2.0504961013793945, 2.6019821166992188),
           (11608, 2.3355419635772705, 1.2164380550384521),
           (11609, 2.0273220539093018, 2.2453839778900146)],
          dtype=[('simulation_id', '<i8'), ('mass1', '<f4'), ('mass2', '<f4')])

    Add a field to the array:

    >>> optimal_snrs = numpy.random.uniform(4.,40., size=len(sim_array))
    >>> sim_array = sim_array.add_fields(optimal_snrs, 'optimal_snrs')
    >>> sim_array.fieldnames
        ('simulation_id', 'mass1', 'mass2', 'optimal_snrs')

    Notes
    -----
    Input arrays with variable-length strings in one or more fields can be
    tricky to deal with. Numpy arrays are designed to use fixed-length
    datasets, so that quick memory access can be achieved. To deal with
    variable-length strings, there are two options: 1. set the data type to
    object, or 2. set the data type to a string with a fixed length larger
    than the longest string in the input array.

    The first option, using objects, essentially causes the array to store a
    pointer to the string.  This is the most flexible option, as it allows
    strings in the array to be updated to any length. However, operations on
    object fields are slower, as numpy cannot take advantage of its fast
    memory striding abilities (see `this question/answer on stackoverflow
    <http://stackoverflow.com/a/14639568/1366472>`_ for details). Also,
    numpy's support of object arrays is more limited.  In particular, prior
    to version 1.9.2, you cannot create a view of an array that changes the
    dtype if the array has any fields that are objects, even if the view does
    not affect the object fields. (This has since been relaxed.)

    The second option, using strings of a fixed length, solves the issues
    with object fields. However, if you try to change one of the strings
    after the array is created, the string will be truncated at whatever
    string length is used. Additionally, if you choose too large of a string
    length, you can substantially increase the memory overhead for large
    arrays.

    """
    _virtualfields = []
    _functionlib = _fieldarray_functionlib
    __persistent_attributes__ = ['name', '_virtualfields', '_functionlib']

    def __new__(cls, shape, name=None, zero=True, **kwargs):
        """Initializes a new empty array.
        """
        obj = super(FieldArray, cls).__new__(cls, shape, **kwargs).view(
            type=cls)
        obj.name = name
        obj.__persistent_attributes__ = [a
            for a in cls.__persistent_attributes__]
        obj._functionlib = {f: func for f,func in cls._functionlib.items()}
        obj._virtualfields = [f for f in cls._virtualfields]
        # zero out the array if desired
        if zero:
            default = default_empty(1, dtype=obj.dtype)
            obj[:] = default
        return obj

    def __array_finalize__(self, obj):
        """Default values are set here.

        See <https://docs.scipy.org/doc/numpy/user/basics.subclassing.html> for
        details.
        """
        if obj is None:
            return
        # copy persistent attributes
        try:
            obj.__copy_attributes__(self)
            pass
        # numpy has some issues with dtype field names that are unicode,
        # so we'll force them to strings here
        if six.PY2:
            if self.dtype.names is not None and \
                    any(isinstance(name, unicode) for name in obj.dtype.names):
                self.dtype.names = map(str, self.dtype.names)
            # We don't want to do this in python3 because a str *is* unicode,
            # but maybe numpy in python3 is more lenient of this. Let's just
            # wait and see if this becomes a problem in python3

    def __copy_attributes__(self, other, default=None):
        """Copies the values of all of the attributes listed in
        `self.__persistent_attributes__` to other.
        """
        [setattr(other, attr, copy.deepcopy(getattr(self, attr, default))) \
            for attr in self.__persistent_attributes__]

    def __getattribute__(self, attr, no_fallback=False):
        """Allows fields to be accessed as attributes.
        """
        # first try to get the attribute
        try:
            return numpy.ndarray.__getattribute__(self, attr)
        except AttributeError as e:
            # don't try getitem, which might get back here
            if no_fallback:
                raise(e)

            # might be a field, try to retrive it using getitem
            if attr in self.fields:
                return self.__getitem__(attr)
            # otherwise, unrecognized
            raise AttributeError(e)

    def __setitem__(self, item, values):
        """Wrap's recarray's setitem to allow attribute-like indexing when
        setting values.
        """
        if type(item) is int and type(values) is numpy.ndarray:
            # numpy >=1.14 only accepts tuples
            values = tuple(values)
        try:
            return super(FieldArray, self).__setitem__(item, values)
            # we'll get a ValueError if a subarray is being referenced using
            # '.'; so we'll try to parse it out here
            fields = item.split('.')
            if len(fields) > 1:
                for field in fields[:-1]:
                    self = self[field]
                item = fields[-1]
            # now try again

    def __getbaseitem__(self, item):
        """Gets an item assuming item is either an index or a fieldname.
        """
        # We cast to a ndarray to avoid calling array_finalize, which can be
        # slow
        out = self.view(numpy.ndarray)[item]
        # if there are no fields, then we can just return
        if out.dtype.fields is None:
            return out
        # if there are fields, but only a single entry, we'd just get a
        # record by casting to self, so just cast immediately to recarray
        elif out.ndim == 0:
            return out.view(numpy.recarray)
        # otherwise, cast back to an instance of self
        else:
            return out.view(type(self))

    def __getsubitem__(self, item):
        """Gets a subfield using `field.subfield` notation.
        """
        try:
            return self.__getbaseitem__(item)
        except ValueError as err:
            subitems = item.split('.')
            if len(subitems) > 1:
                return self.__getbaseitem__(subitems[0]
                    ).__getsubitem__('.'.join(subitems[1:]))
            else:
                raise ValueError(err)

    def __getitem__(self, item):
        """Wraps recarray's  `__getitem__` so that math functions on fields and
        attributes can be retrieved. Any function in numpy's library may be
        used.
        """
        try:
            return self.__getsubitem__(item)
            #
            #   arg isn't a simple argument of row, so we'll have to eval it
            #
            if not hasattr(self, '_code_cache'):
                self._code_cache = {}

            if item not in self._code_cache:
                code = compile(item, '<string>', 'eval')

                # get the function library
                item_dict = dict(_numpy_function_lib.items())
                item_dict.update(self._functionlib)

                # parse to get possible fields
                itemvars_raw = get_fields_from_arg(item)

                itemvars = []
                for it in itemvars_raw:
                    try:
                        float(it)
                        is_num = True
                    except ValueError:
                        is_num = False

                    if not is_num:
                        itemvars.append(it)

                self._code_cache[item] = (code, itemvars, item_dict)

            code, itemvars, item_dict = self._code_cache[item]
            for it in itemvars:
                if it in self.fieldnames:
                    # pull out the fields: note, by getting the parent fields
                    # we also get the sub fields name
                    item_dict[it] = self.__getbaseitem__(it)
                elif (it in self.__dict__) or (it in self._virtualfields):
                    # pull out any needed attributes
                    item_dict[it] = self.__getattribute__(it, no_fallback=True)
                else:
                    # add any aliases
                    aliases = self.aliases
                    if it in aliases:
                        item_dict[it] = self.__getbaseitem__(aliases[it])

            return eval(code, {"__builtins__": None}, item_dict)

    def __contains__(self, field):
        """Returns True if the given field name is in self's fields."""
        return field in self.fields

    def sort(self, axis=-1, kind='quicksort', order=None):
        """Sort an array, in-place.

        This function extends the standard numpy record array in-place sort
        to allow the basic use of Field array virtual fields. Only a single
        field is currently supported when referencing a virtual field.

        axis : int, optional
            Axis along which to sort. Default is -1, which means sort along the
            last axis.
        kind : {'quicksort', 'mergesort', 'heapsort'}, optional
            Sorting algorithm. Default is 'quicksort'.
        order : list, optional
            When `a` is an array with fields defined, this argument specifies
            which fields to compare first, second, etc.  Not all fields need be
            specified.
        """
        try:
            numpy.recarray.sort(self, axis=axis, kind=kind, order=order)
            if isinstance(order, list):
                raise ValueError("Cannot process more than one order field")
            self[:] = self[numpy.argsort(self[order])]

    def addattr(self, attrname, value=None, persistent=True):
        """Adds an attribute to self. If persistent is True, the attribute will
        be made a persistent attribute. Persistent attributes are copied
        whenever a view or copy of this array is created. Otherwise, new views
        or copies of this will not have the attribute.
        """
        setattr(self, attrname, value)
        # add as persistent
        if persistent and attrname not in self.__persistent_attributes__:
            self.__persistent_attributes__.append(attrname)

    def add_methods(self, names, methods):
        """Adds the given method(s) as instance method(s) of self. The
        method(s) must take `self` as a first argument.
        """
            methods = [methods]
        for name,method in zip(names, methods):
            setattr(self, name, types.MethodType(method, self))

    def add_properties(self, names, methods):
        """Returns a view of self with the given methods added as properties.

        From: <http://stackoverflow.com/a/2954373/1366472>.
        """
        cls = type(self)
        cls = type(cls.__name__, (cls,), dict(cls.__dict__))
            setattr(cls, name, property(method))
        return self.view(type=cls)

    def add_virtualfields(self, names, methods):
        """Returns a view of this array with the given methods added as virtual
        fields. Specifically, the given methods are added using add_properties
        and their names are added to the list of virtual fields. Virtual fields
        are properties that are assumed to operate on one or more of self's
        fields, thus returning an array of values.
        """
        out = self.add_properties(names, methods)
        if out._virtualfields is None:
            out._virtualfields = []
        out._virtualfields.extend(names)
        return out

    def add_functions(self, names, functions):
        """Adds the given functions to the function library.

        Functions are added to this instance of the array; all copies of
        and slices of this array will also have the new functions included.

        names : (list of) string(s)
            Name or list of names of the functions.
        functions : (list of) function(s)
            The function(s) to call.
        """
            functions = [functions]
        if len(functions) != len(names):
            raise ValueError("number of provided names must be same as number "
                             "of functions")
        self._functionlib.update(dict(zip(names, functions)))

    def del_functions(self, names):
        """Removes the specified function names from the function library.

        Functions are removed from this instance of the array; all copies
        and slices of this array will also have the functions removed.

            Name or list of names of the functions to remove.
        """
        for name in names:
            self._functionlib.pop(name)

    def from_arrays(cls, arrays, name=None, **kwargs):
        """Creates a new instance of self from the given (list of) array(s).
        This is done by calling numpy.rec.fromarrays on the given arrays with
        the given kwargs. The type of the returned array is cast to this
        class, and the name (if provided) is set.

        arrays : (list of) numpy array(s)
            A list of the arrays to create the FieldArray from.
        name : {None|str}
            What the output array should be named.

        For other keyword parameters, see the numpy.rec.fromarrays help.

        Returns
        -------
        array : instance of this class
            An array that is an instance of this class in which the field
            data is from the given array(s).
        """
        obj = numpy.rec.fromarrays(arrays, **kwargs).view(type=cls)
        return obj

    def from_records(cls, records, name=None, **kwargs):
        """Creates a new instance of self from the given (list of) record(s).

        A "record" is a tuple in which each element is the value of one field
        in the resulting record array. This is done by calling
        `numpy.rec.fromrecords` on the given records with the given kwargs.
        The type of the returned array is cast to this class, and the name
        (if provided) is set.

        records : (list of) tuple(s)
            A list of the tuples to create the FieldArray from.

        Other Parameters
        ----------------
        For other keyword parameters, see the `numpy.rec.fromrecords` help.

        Returns
        -------
            data is from the given record(s).
        """
        obj = numpy.rec.fromrecords(records, **kwargs).view(
            type=cls)
        return obj

    def from_kwargs(cls, **kwargs):
        """Creates a new instance of self from the given keyword arguments.
        Each argument will correspond to a field in the returned array, with
        the name of the field given by the keyword, and the value(s) whatever
        the keyword was set to. Each keyword may be set to a single value or
        a list of values. The number of values that each argument is set to
        must be the same; this will be the size of the returned array.

        Examples
        --------
        Create an array with fields 'mass1' and 'mass2':
        >>> a = FieldArray.from_kwargs(mass1=[1.1, 3.], mass2=[2., 3.])
        >>> a.fieldnames
        ('mass1', 'mass2')
        >>> a.mass1, a.mass2
        (array([ 1.1,  3. ]), array([ 2.,  3.]))

        Create an array with only a single element in it:
        >>> a = FieldArray.from_kwargs(mass1=1.1, mass2=2.)
        (array([ 1.1]), array([ 2.]))
        """
        arrays = []
        names = []
        for p,vals in kwargs.items():
            if not isinstance(vals, numpy.ndarray):
                if not isinstance(vals, list):
                    vals = [vals]
                vals = numpy.array(vals)
            arrays.append(vals)
            names.append(p)
        return cls.from_arrays(arrays, names=names)

    def from_ligolw_table(cls, table, columns=None, cast_to_dtypes=None):
        """Converts the given ligolw table into an FieldArray. The `tableName`
        attribute is copied to the array's `name`.

        table : LIGOLw table instance
            The table to convert.
        columns : {None|list}
            Optionally specify a list of columns to retrieve. All of the
            columns must be in the table's validcolumns attribute. If None
            provided, all the columns in the table will be converted.
        dtype : {None | dict}
            Override the columns' dtypes using the given dictionary. The
            dictionary should be keyed by the column names, with the values
            a tuple that can be understood by numpy.dtype. For example, to
            cast a ligolw column called "foo" to a field called "bar" with
            type float, cast_to_dtypes would be: ``{"foo": ("bar", float)}``.

        Returns
        -------
        array : FieldArray
            The input table as an FieldArray.
        """
        name = table.tableName.split(':')[0]
        if columns is None:
            # get all the columns
            columns = table.validcolumns
        else:
            # note: this will raise a KeyError if one or more columns is
            # not in the table's validcolumns
            new_columns = {}
            for col in columns:
                new_columns[col] = table.validcolumns[col]
            columns = new_columns
        if cast_to_dtypes is not None:
            dtype = [cast_to_dtypes[col] for col in columns]
        else:
            dtype = list(columns.items())
        # get the values
        if _default_types_status['ilwd_as_int']:
            input_array = \
                [tuple(getattr(row, col) if dt != 'ilwd:char'
                       else int(getattr(row, col))
                       for col,dt in columns.items())
                 for row in table]
        else:
                [tuple(getattr(row, col) for col in columns) for row in table]
        # return the values as an instance of cls
        return cls.from_records(input_array, dtype=dtype,
            name=name)

    def to_array(self, fields=None, axis=0):
        """Returns an `numpy.ndarray` of self in which the fields are included
        as an extra dimension.

        fields : {None, (list of) strings}
            The fields to get. All of the fields must have the same datatype.
            If None, will try to return all of the fields.
        axis : {0, int}
            Which dimension to put the fields in in the returned array. For
            example, if `self` has shape `(l,m,n)` and `k` fields, the
            returned array will have shape `(k,l,m,n)` if `axis=0`, `(l,k,m,n)`
            if `axis=1`, etc. Setting `axis=-1` will put the fields in the
            last dimension. Default is 0.

        Returns
        -------
            The desired fields as a numpy array.
        """
        if fields is None:
            fields = self.fieldnames
        return numpy.stack([self[f] for f in fields], axis=axis)

    @property
    def fieldnames(self):
        """Returns a tuple listing the field names in self. Equivalent to
        `array.dtype.names`, where `array` is self.
        """
        return self.dtype.names

    @property
    def virtualfields(self):
        """Returns a tuple listing the names of virtual fields in self.
        """
        if self._virtualfields is None:
            vfs = tuple()
        else:
            vfs = tuple(self._virtualfields)
        return vfs

    @property
    def functionlib(self):
        """Returns the library of functions that are available when calling
        items.
        """
        return self._functionlib

    @property
    def fields(self):
        """Returns a tuple listing the names of fields and virtual fields in
        self."""
        return tuple(list(self.fieldnames) + list(self.virtualfields))

    @property
    def aliases(self):
        """Returns a dictionary of the aliases, or "titles", of the field names
        in self. An alias can be specified by passing a tuple in the name
        part of the dtype. For example, if an array is created with
        ``dtype=[(('foo', 'bar'), float)]``, the array will have a field
        called `bar` that has alias `foo` that can be accessed using
        either `arr['foo']` or `arr['bar']`. Note that the first string
        in the dtype is the alias, the second the name. This function returns
        a dictionary in which the aliases are the keys and the names are the
        values. Only fields that have aliases are returned.
        """
        return dict(c[0] for c in self.dtype.descr if isinstance(c[0], tuple))

    def add_fields(self, arrays, names=None, assubarray=False):
        """
        Adds the given arrays as new fields to self.  Returns a new instance
        with the new fields added. Note: this array does not change; the
        returned array is a new copy.

            The arrays to add. If adding multiple arrays, must be a list;
            if adding a single array, can just be that array.
        names : (list of) strings
            Optional, the name(s) of the new fields in the output array. If
            adding multiple fields, must be a list of strings with the same
            length as the list of arrays. If None provided, names used will
            be the same as the name of the datatype in the given arrays.
            If the datatype has no name, the new field will be ``'fi'`` where
            i is the index of the array in arrays.
        assubarray : bool
            Add the list of arrays as a single subarray field. If True, and
            names provided, names should be a string or a length-1 sequence.
            Default is False, in which case each array will be added as a
            separate field.

        Returns
        -------
        new_array : new instance of this array
            A copy of this array with the desired fields added.
        """
        newself = add_fields(self, arrays, names=names, assubarray=assubarray)
        self.__copy_attributes__(newself)
        return newself

    def parse_boolargs(self, args):
        """Returns an array populated by given values, with the indices of
        those values dependent on given boolen tests on self.

        The given `args` should be a list of tuples, with the first element the
        return value and the second argument a string that evaluates to either
        True or False for each element in self.

        Each boolean argument is evaluated on elements for which every prior
        boolean argument was False. For example, if array `foo` has a field
        `bar`, and `args = [(1, 'bar < 10'), (2, 'bar < 20'), (3, 'bar < 30')]`,
        then the returned array will have `1`s at the indices for
        which `foo.bar < 10`, `2`s where `foo.bar < 20 and not foo.bar < 10`,
        and `3`s where `foo.bar < 30 and not (foo.bar < 10 or foo.bar < 20)`.

        The last argument in the list may have "else", an empty string, None,
        or simply list a return value. In any of these cases, any element not
        yet populated will be assigned the last return value.

        args : {(list of) tuples, value}
            One or more return values and boolean argument determining where
            they should go.

        Returns
        -------
        return_values : array
            An array with length equal to self, with values populated with the
            return values.
        leftover_indices : array
            An array of indices that evaluated to False for all arguments.
            These indices will not have been popluated with any value,
            defaulting to whatever numpy uses for a zero for the return
            values' dtype. If there are no leftovers, an empty array is
            returned.

        Examples
        --------
        Given the following array:

        >>> arr = FieldArray(5, dtype=[('mtotal', float)])
        >>> arr['mtotal'] = numpy.array([3., 5., 2., 1., 4.])

        Return `"TaylorF2"` for all elements with `mtotal < 4` (note that the
        elements 1 and 4 are leftover):

        >>> arr.parse_boolargs(('TaylorF2', 'mtotal<4'))
            (array(['TaylorF2', '', 'TaylorF2', 'TaylorF2', ''],
            dtype='|S8'),
            array([1, 4]))

        Return `"TaylorF2"` for all elements with `mtotal < 4`,
        `"SEOBNR_ROM_DoubleSpin"` otherwise:

        >>> arr.parse_boolargs([('TaylorF2', 'mtotal<4'), ('SEOBNRv2_ROM_DoubleSpin', 'else')])
            (array(['TaylorF2', 'SEOBNRv2_ROM_DoubleSpin', 'TaylorF2', 'TaylorF2',
            'SEOBNRv2_ROM_DoubleSpin'],
            dtype='|S23'),
            array([], dtype=int64))

        The following will also return the same:

        >>> arr.parse_boolargs([('TaylorF2', 'mtotal<4'), ('SEOBNRv2_ROM_DoubleSpin',)])
        >>> arr.parse_boolargs([('TaylorF2', 'mtotal<4'), ('SEOBNRv2_ROM_DoubleSpin', '')])
        >>> arr.parse_boolargs([('TaylorF2', 'mtotal<4'), 'SEOBNRv2_ROM_DoubleSpin'])

        Return `"TaylorF2"` for all elements with `mtotal < 3`, `"IMRPhenomD"`
        for all elements with `3 <= mtotal < 4`, `"SEOBNRv2_ROM_DoubleSpin"`
        otherwise:

        >>> arr.parse_boolargs([('TaylorF2', 'mtotal<3'), ('IMRPhenomD', 'mtotal<4'), 'SEOBNRv2_ROM_DoubleSpin'])
            (array(['IMRPhenomD', 'SEOBNRv2_ROM_DoubleSpin', 'TaylorF2', 'TaylorF2',

        Just return `"TaylorF2"` for all elements:

        >>> arr.parse_boolargs('TaylorF2')
            (array(['TaylorF2', 'TaylorF2', 'TaylorF2', 'TaylorF2', 'TaylorF2'],

        """
        if not isinstance(args, list):
            args = [args]
        # format the arguments
        return_vals = []
        bool_args = []
            if not isinstance(arg, tuple):
                return_val = arg
                bool_arg = None
            elif len(arg) == 1:
                return_val = arg[0]
            elif len(arg) == 2:
                return_val, bool_arg = arg
            else:
                raise ValueError("argument not formatted correctly")
            return_vals.append(return_val)
            bool_args.append(bool_arg)
        # get the output dtype
        outdtype = numpy.array(return_vals).dtype
        out = numpy.zeros(self.size, dtype=outdtype)
        mask = numpy.zeros(self.size, dtype=bool)
        leftovers = numpy.ones(self.size, dtype=bool)
        for ii,(boolarg,val) in enumerate(zip(bool_args, return_vals)):
            if boolarg is None or boolarg == '' or boolarg.lower() == 'else':
                if ii+1 != len(bool_args):
                    raise ValueError("only the last item may not provide "
                        "any boolean arguments")
                mask = leftovers
            else:
                mask = leftovers & self[boolarg]
            out[mask] = val
            leftovers &= ~mask
        return out, numpy.where(leftovers)[0]

    def append(self, other):
        """Appends another array to this array.

        The returned array will have all of the class methods and virutal
        fields of this array, including any that were added using `add_method`
        or `add_virtualfield`. If this array and other array have one or more
        string fields, the dtype for those fields are updated to a string
        length that can encompass the longest string in both arrays.

        .. note::
            Increasing the length of strings only works for fields, not
            sub-fields.

        other : array
            The array to append values from. It must have the same fields and
            dtype as this array, modulo the length of strings. If the other
            array does not have the same dtype, a TypeError is raised.

        Returns
        -------
        array
            An array with others values appended to this array's values. The
            returned array is an instance of the same class as this array,
            including all methods and virtual fields.
        """
        try:
            return numpy.append(self, other).view(type=self.__class__)
            # see if the dtype error was due to string fields having different
            # lengths; if so, we'll make the joint field the larger of the
            # two
            str_fields = [name for name in self.fieldnames
                          if _isstring(self.dtype[name])]
            # get the larger of the two
            new_strlens = dict(
                [[name,
                  max(self.dtype[name].itemsize, other.dtype[name].itemsize)]
                 for name in str_fields]
            )
            # cast both to the new string lengths
            new_dt = []
            for dt in self.dtype.descr:
                name = dt[0]
                if name in new_strlens:
                    dt = (name, self.dtype[name].type, new_strlens[name])
                new_dt.append(dt)
            new_dt = numpy.dtype(new_dt)
            return numpy.append(
                self.astype(new_dt),
                other.astype(new_dt)
                ).view(type=self.__class__)

    def parse_parameters(cls, parameters, possible_fields):
        """Parses a list of parameters to get the list of fields needed in
        order to evaluate those parameters.

        parameters : (list of) string(s)
            The list of desired parameters. These can be (functions of) fields
            or virtual fields.
        possible_fields : (list of) string(s)
            The list of possible fields.

        Returns
        -------
        list :
            The list of names of the fields that are needed in order to
            evaluate the given parameters.
        """
        if isinstance(possible_fields, string_types):
            possible_fields = [possible_fields]
        possible_fields = list(map(str, possible_fields))
        # we'll just use float as the dtype, as we just need this for names
        arr = cls(1, dtype=list(zip(possible_fields,
                                len(possible_fields)*[float])))
        # try to perserve order
        return list(get_needed_fieldnames(arr, parameters))

def _isstring(dtype):
    """Given a numpy dtype, determines whether it is a string. Returns True
    if the dtype is string or unicode.
    """
    return dtype.type == numpy.unicode_ or dtype.type == numpy.string_

def aliases_from_fields(fields):
    """Given a dictionary of fields, will return a dictionary mapping the
    aliases to the names.
    """
    return dict(c for c in fields if isinstance(c, tuple))

def fields_from_names(fields, names=None):
    """Given a dictionary of fields and a list of names, will return a
    dictionary consisting of the fields specified by names. Names can be
    either the names of fields, or their aliases.
    """

    if names is None:
        return fields
    aliases_to_names = aliases_from_fields(fields)
    names_to_aliases = dict(zip(aliases_to_names.values(),
        aliases_to_names.keys()))
    outfields = {}
        try:
            outfields[name] = fields[name]
            if name in aliases_to_names:
                key = (name, aliases_to_names[name])
            elif name in names_to_aliases:
                key = (names_to_aliases[name], name)
            else:
                raise KeyError('default fields has no field %s' % name)
            outfields[key] = fields[key]
    return outfields

#
#
#                           FieldArray with default fields
#
#

class _FieldArrayWithDefaults(FieldArray):
    """
    Subclasses FieldArray, adding class attribute ``_staticfields``, and
    class method ``default_fields``. The ``_staticfields`` should be a
    dictionary that defines some field names and corresponding dtype. The
    ``default_fields`` method returns a dictionary of the static fields
    and any default virtualfields that were added. A field array can then
    be initialized in one of 3 ways:

     1. With just a shape. In this case, the returned array will have all
     of the default fields.

     2. With a shape and a list of names, given by the ``names`` keyword
     argument. The names may be default fields, virtual fields, a method or
     property of the class, or any python function of these things. If a
     virtual field, method, or property is in the names, the needed underlying
     fields will be included in the return array. For example, if the class
     has a virtual field called 'mchirp', which is a function of fields called
     'mass1' and 'mass2', then 'mchirp' or any function of 'mchirp' may be
     included in the list of names (e.g., names=['mchirp**(5/6)']). If so, the
     returned array will have fields 'mass1' and 'mass2' even if these were
     not specified in names, so that 'mchirp' may be used without error.
     names must be names of either default fields or virtualfields, else a
     KeyError is raised.

     3. With a shape and a dtype. Any field specified by the dtype will be
     used. The fields need not be in the list of default fields, and/or the
     dtype can be different than that specified by the default fields.

    If additional fields are desired beyond the default fields, these can
    be specified using the ``additional_fields`` keyword argument; these should
    be provided in the same way as ``dtype``; i.e, as a list of (name, dtype)
    tuples.

    This class does not define any static fields, and ``default_fields`` just
    returns an empty dictionary. This class is mostly meant to be subclassed
    by other classes, so they can add their own defaults.
    """

    _staticfields = {}
    def default_fields(cls, include_virtual=True, **kwargs):
        """The default fields and their dtypes. By default, this returns
        whatever the class's ``_staticfields`` and ``_virtualfields`` is set
        to as a dictionary of fieldname, dtype (the dtype of virtualfields is
        given by VIRTUALFIELD_DTYPE). This function should be overridden by
        subclasses to add dynamic fields; i.e., fields that require some input
        parameters at initialization. Keyword arguments can be passed to this
        to set such dynamic fields.
        """
        output = cls._staticfields.copy()
        if include_virtual:
            output.update({name: VIRTUALFIELD_DTYPE
                           for name in cls._virtualfields})
        return output

    def __new__(cls, shape, name=None, additional_fields=None,
                field_kwargs=None, **kwargs):
        """The ``additional_fields`` should be specified in the same way as
        ``dtype`` is normally given to FieldArray. The ``field_kwargs`` are
        passed to the class's default_fields method as keyword arguments.
        """
        if field_kwargs is None:
            field_kwargs = {}
        if 'names' in kwargs and 'dtype' in kwargs:
            raise ValueError("Please provide names or dtype, not both")
        default_fields = cls.default_fields(include_virtual=False,
            **field_kwargs)
        if 'names' in kwargs:
            names = kwargs.pop('names')
            if isinstance(names, string_types):
                names = [names]
            # evaluate the names to figure out what base fields are needed
            # to do this, we'll create a small default instance of self (since
            # no names are specified in the following initialization, this
            # block of code is skipped)
            arr = cls(1, field_kwargs=field_kwargs)
            # try to perserve order
            sortdict = dict([[nm, ii] for ii,nm in enumerate(names)])
            names = list(get_needed_fieldnames(arr, names))
            names.sort(key=lambda x: sortdict[x] if x in sortdict
                else len(names))
            # add the fields as the dtype argument for initializing
            kwargs['dtype'] = [(fld, default_fields[fld]) for fld in names]
        if 'dtype' not in kwargs:
            kwargs['dtype'] = list(default_fields.items())
        # add the additional fields
        if additional_fields is not None:
            if not isinstance(additional_fields, list):
                additional_fields = [additional_fields]
            if not isinstance(kwargs['dtype'], list):
                kwargs['dtype'] = [kwargs['dtype']]
            kwargs['dtype'] += additional_fields
        return super(_FieldArrayWithDefaults, cls).__new__(cls, shape,
            name=name, **kwargs)

    def add_default_fields(self, names, **kwargs):
        """
        Adds one or more empty default fields to self.

            The names of the fields to add. Must be a field in self's default
            fields.

        Other keyword args are any arguments passed to self's default fields.

        Returns
        -------
        new array : instance of this array
            A copy of this array with the field added.
        """
        default_fields = self.default_fields(include_virtual=False, **kwargs)
        # parse out any virtual fields
        arr = self.__class__(1, field_kwargs=kwargs)
        sortdict = dict([[nm, ii] for ii,nm in enumerate(names)])
        names = list(get_needed_fieldnames(arr, names))
        names.sort(key=lambda x: sortdict[x] if x in sortdict
            else len(names))
        fields = [(name, default_fields[name]) for name in names]
        arrays = []
        names = []
        for name,dt in fields:
            arrays.append(default_empty(self.size, dtype=[(name, dt)]))
            names.append(name)
        return self.add_fields(arrays, names)

    def parse_parameters(cls, parameters, possible_fields=None):

        possible_fields : {None, dict}
            Specify the list of possible fields. Must be a dictionary given
            the names, and dtype of each possible field. If None, will use this
            class's `_staticfields`.

        Returns
        -------
        list :
        """
        if possible_fields is not None:
            # make sure field names are strings and not unicode
            possible_fields = dict([[f, dt]
                for f,dt in possible_fields.items()])
            class ModifiedArray(cls):
                _staticfields = possible_fields
            cls = ModifiedArray
        return cls(1, names=parameters).fieldnames

#
#
#                           WaveformArray
#
#

class WaveformArray(_FieldArrayWithDefaults):
    """
    A FieldArray with some default fields and properties commonly used
    by CBC waveforms. This may be initialized in one of 3 ways:

    1. With just the size of the array. In this case, the returned array will
    have all of the default field names. Example:

    >>> warr = WaveformArray(10)
    >>> warr.fieldnames
        ('distance',
         'spin2x',
         'mass1',
         'mass2',
         'lambda1',
         'polarization',
         'spin2y',
         'spin2z',
         'spin1y',
         'spin1x',
         'spin1z',
         'inclination',
         'coa_phase',
         'dec',
         'tc',
         'lambda2',
         'ra')

    2. With some subset of the default field names. Example:

    >>> warr = WaveformArray(10, names=['mass1', 'mass2'])

    The list of names may include virtual fields, and methods, as well as
    functions of these. If one or more virtual fields or methods are specified,
    the source code is analyzed to pull out whatever underlying fields are
    needed. Example:

    >>> warr = WaveformArray(10, names=['mchirp**(5/6)', 'chi_eff', 'cos(coa_phase)'])
        ('spin2z', 'mass1', 'mass2', 'coa_phase', 'spin1z')

    3. By specifying a dtype. In this case, only the provided fields will
    be used, even if they are not in the defaults. Example:

    >>> warr = WaveformArray(10, dtype=[('foo', float)])
        ('foo',)

    Additional fields can also be specified using the additional_fields
    keyword argument. Example:

    >>> warr = WaveformArray(10, names=['mass1', 'mass2'], additional_fields=[('bar', float)])
        ('mass1', 'mass2', 'bar')

    .. note::
        If an array is initialized with all of the default fields (case 1,
        above), then the names come from waveform.parameters; i.e., they
        are actually Parameter instances, not just strings. This means that the
        field names carry all of the metadata that a Parameter has. For
        example:

        >>> warr = WaveformArray(10)
        >>> warr.fields[0]
            'distance'
        >>> warr.fields[0].description
            'Luminosity distance to the binary (in Mpc).'
        >>> warr.fields[0].label
            '$d_L$ (Mpc)'

    """
    _staticfields = (parameters.cbc_intrinsic_params +
                     parameters.extrinsic_params).dtype_dict

    _virtualfields = [
        parameters.mchirp, parameters.eta, parameters.mtotal,
        parameters.q, parameters.primary_mass, parameters.secondary_mass,
        parameters.chi_eff,
        parameters.spin_px, parameters.spin_py, parameters.spin_pz,
        parameters.spin_sx, parameters.spin_sy, parameters.spin_sz,
        parameters.spin1_a, parameters.spin1_azimuthal, parameters.spin1_polar,
        parameters.spin2_a, parameters.spin2_azimuthal, parameters.spin2_polar]

    @property
    def primary_mass(self):
        """Returns the larger of self.mass1 and self.mass2."""
        return conversions.primary_mass(self.mass1, self.mass2)

    @property
    def secondary_mass(self):
        """Returns the smaller of self.mass1 and self.mass2."""
        return conversions.secondary_mass(self.mass1, self.mass)

    @property
        """Returns the total mass."""
        return conversions.mtotal_from_mass1_mass2(self.mass1, self.mass2)

    @property
    def q(self):
        """Returns the mass ratio m1/m2, where m1 >= m2."""
        return conversions.q_from_mass1_mass2(self.mass1, self.mass2)

    @property
        """Returns the symmetric mass ratio."""

    @property
        """Returns the chirp mass."""

    @property
    def chi_eff(self):
        """Returns the effective spin."""
        return conversions.chi_eff(self.mass1, self.mass2, self.spin1z,
                                   self.spin2z)

    @property
    def spin_px(self):
        """Returns the x-component of the spin of the primary mass."""
        return conversions.primary_spin(self.mass1, self.mass2, self.spin1x,
                                        self.spin2x)

    @property
    def spin_py(self):
        """Returns the y-component of the spin of the primary mass."""
        return conversions.primary_spin(self.mass1, self.mass2, self.spin1y,
                                        self.spin2y)

    @property
    def spin_pz(self):
        """Returns the z-component of the spin of the primary mass."""
        return conversions.primary_spin(self.mass1, self.mass2, self.spin1z,
                                        self.spin2z)

    @property
    def spin_sx(self):
        """Returns the x-component of the spin of the secondary mass."""
        return conversions.secondary_spin(self.mass1, self.mass2, self.spin1x,

    @property
    def spin_sy(self):
        """Returns the y-component of the spin of the secondary mass."""
        return conversions.secondary_spin(self.mass1, self.mass2, self.spin1y,

    @property
    def spin_sz(self):
        """Returns the z-component of the spin of the secondary mass."""
        return conversions.secondary_spin(self.mass1, self.mass2, self.spin1z,

    @property
    def spin1_a(self):
        """Returns the dimensionless spin magnitude of mass 1."""
        return coordinates.cartesian_to_spherical_rho(
                                    self.spin1x, self.spin1y, self.spin1z)

    @property
    def spin1_azimuthal(self):
        """Returns the azimuthal spin angle of mass 1."""
        return coordinates.cartesian_to_spherical_azimuthal(
                                     self.spin1x, self.spin1y)

    @property
    def spin1_polar(self):
        """Returns the polar spin angle of mass 1."""
        return coordinates.cartesian_to_spherical_polar(
                                     self.spin1x, self.spin1y, self.spin1z)

    @property
    def spin2_a(self):
        """Returns the dimensionless spin magnitude of mass 2."""

    @property
    def spin2_azimuthal(self):
        """Returns the azimuthal spin angle of mass 2."""
                                     self.spin2x, self.spin2y)

    @property
    def spin2_polar(self):
        """Returns the polar spin angle of mass 2."""
                                     self.spin2x, self.spin2y, self.spin2z)

__all__ = ['FieldArray', 'WaveformArray']

from astropy.utils.data import download_file
from .hdf import *
from .record import *

def get_file(url, retry=5, **args):
    """ Retrieve file with retry upon failure

    Uses the astropy download_file but adds a retry feature for flaky
    connections. See astropy for full options
    """
    i = 0
    while True:
        i += 1
        try:
            return download_file(url, **args)
            logging.warning("Failed on attempt %d to download %s", i, url)
            if i >= retry:
                logging.error("Giving up on %s", url)
                raise e

#
#
#
#

#
#
#                                   Preamble
#
#
"""This module contains functions to generate gaussian noise colored with a
noise spectrum.
"""
from pycbc.types import complex_same_precision_as, FrequencySeries
from lalsimulation import SimNoise
import lal

def frequency_noise_from_psd(psd, seed=None):
    """ Create noise with a given psd.

    Return noise coloured with the given psd. The returned noise
    FrequencySeries has the same length and frequency step as the given psd.
    Note that if unique noise is desired a unique seed should be provided.

        The noise weighting to color the noise.
    seed : {0, int} or None
        The seed to generate the noise. If None specified,
        the seed will not be reset.

    Returns
    --------
    noise : FrequencySeriesSeries
        A FrequencySeries containing gaussian noise colored by the given psd.
    """
    sigma = 0.5 * (psd / psd.delta_f) ** (0.5)
    if seed is not None:
    sigma = sigma.numpy()
    dtype = complex_same_precision_as(psd)

    not_zero = (sigma != 0)

    sigma_red = sigma[not_zero]
    noise_re = numpy.random.normal(0, sigma_red)
    noise_co = numpy.random.normal(0, sigma_red)
    noise_red = noise_re + 1j * noise_co

    noise = numpy.zeros(len(sigma), dtype=dtype)
    noise[not_zero] = noise_red

    return FrequencySeries(noise,
                           delta_f=psd.delta_f,
                           dtype=dtype)

def noise_from_psd(length, delta_t, psd, seed=None):

    Return noise with a given psd. Note that if unique noise is desired
    a unique seed should be provided.

    length : int
        The length of noise to generate in samples.
        The time step of the noise.
    seed : {0, int}
        The seed to generate the noise.

    Returns
    --------
    noise : TimeSeries
        A TimeSeries containing gaussian noise colored by the given psd.
    """
    noise_ts = TimeSeries(zeros(length), delta_t=delta_t)

    if seed is None:
        seed = numpy.random.randint(2**32)

    randomness = lal.gsl_rng("ranlux", seed)

    N = int (1.0 / delta_t / psd.delta_f)
    n = N//2+1
    stride = N//2

    if n > len(psd):
        raise ValueError("PSD not compatible with requested delta_t")

    psd = (psd[0:n]).lal()
    psd.data.data[n-1] = 0
    psd.data.data[0] = 0

    segment = TimeSeries(zeros(N), delta_t=delta_t).lal()
    length_generated = 0

    SimNoise(segment, 0, psd, randomness)
    while (length_generated < length):
        if (length_generated + stride) < length:
            noise_ts.data[length_generated:length_generated+stride] = segment.data.data[0:stride]
        else:
            noise_ts.data[length_generated:length] = segment.data.data[0:length-length_generated]

        length_generated += stride
        SimNoise(segment, stride, psd, randomness)

    return noise_ts

def noise_from_string(psd_name, length, delta_t, seed=None, low_frequency_cutoff=10.0):
    """ Create noise from an analytic PSD

    Return noise from the chosen PSD. Note that if unique noise is desired

    psd_name : str
        Name of the analytic PSD to use.
    low_fr
    seed : {None, int}
    low_frequency_cutof : {10.0, float}
        The low frequency cutoff to pass to the PSD generation.

    Returns
    --------
    """
    import pycbc.psd

    # We just need enough resolution to resolve lines
    delta_f = 1.0 / 8
    flen = int(.5 / delta_t / delta_f) + 1
    psd = pycbc.psd.from_string(psd_name, flen, delta_f, low_frequency_cutoff)
    return noise_from_psd(int(length), delta_t, psd, seed=seed)

#
#
#
#
#
#
#                                   Preamble
#
#
import numpy, pycbc.psd
from pycbc.types import TimeSeries, FrequencySeries, complex_same_precision_as
from numpy.random import RandomState

# This constant need to be constant to be able to recover identical results.
BLOCK_SAMPLES = 1638400

def block(seed, sample_rate):
    """ Return block of normal random numbers

        The seed to generate the noise.sd
    sample_rate: float
        Sets the variance of the white noise

    Returns
    --------
    noise : numpy.ndarray
        Array of random numbers
    """
    num = BLOCK_SAMPLES
    rng = RandomState(seed % 2**32)
    variance = sample_rate / 2
    return rng.normal(size=num, scale=variance**0.5)

def normal(start, end, sample_rate=16384, seed=0):
    """ Generate data with a white Gaussian (normal) distribution

        Start time in GPS seconds to generate noise
        End time in GPS seconds to generate noise
    sample-rate: float
        Sample rate to generate the data at. Keep constant if you want to
        ensure continuity between disjoint time spans.

    Returns
    --------
        A TimeSeries containing gaussian noise
    """
    # This is reproduceable because we used fixed seeds from known values
    block_dur = BLOCK_SAMPLES / sample_rate
    s = int(numpy.floor(start / block_dur))
    e = int(numpy.floor(end / block_dur))

    # The data evenly divides so the last block would be superfluous
    if end % block_dur == 0:
        e -= 1

    sv = RandomState(seed).randint(-2**50, 2**50)
    data = numpy.concatenate([block(i + sv, sample_rate)
                              for i in numpy.arange(s, e + 1, 1)])
    ts = TimeSeries(data, delta_t=1.0 / sample_rate, epoch=(s * block_dur))
    return ts.time_slice(start, end)

def colored_noise(psd, start_time, end_time,
                  seed=0, sample_rate=16384,
                  low_frequency_cutoff=1.0,
                  filter_duration=128):
    """ Create noise from a PSD

    psd : pycbc.types.FrequencySeries
        PSD to color the noise
        End time in GPS seconds to generate nosie
    sample_rate: {16384, float}
        The sample rate of the output data. Keep constant if you want to
    low_frequency_cutof : {1.0, float}
    filter_duration : {128, float}
        The duration in seconds of the coloring filter

    Returns
    --------
    """
    psd = psd.copy()

    flen = int(sample_rate / psd.delta_f) // 2 + 1
    oldlen = len(psd)
    psd.resize(flen)

    # Want to avoid zeroes in PSD.
    max_val = psd.max()
    for i in range(len(psd)):
        if i >= (oldlen-1):
            psd.data[i] = psd[oldlen - 2]
        if psd[i] == 0:
            psd.data[i] = max_val

    fil_len = int(filter_duration * sample_rate)
    wn_dur = int(end_time - start_time) + 2 * filter_duration
    if psd.delta_f >= 1. / (2.*filter_duration):
        # If the PSD is short enough, this method is less memory intensive than
        # resizing and then calling inverse_spectrum_truncation
        psd = pycbc.psd.interpolate(psd, 1.0 / (2. * filter_duration))
        # inverse_spectrum_truncation truncates the inverted PSD. To truncate
        # the non-inverted PSD we give it the inverted PSD to truncate and then
        # invert the output.
        psd = 1. / pycbc.psd.inverse_spectrum_truncation(
                                1./psd,
                                fil_len,
                                low_frequency_cutoff=low_frequency_cutoff,
                                trunc_method='hann')
        psd = psd.astype(complex_same_precision_as(psd))
        # Zero-pad the time-domain PSD to desired length. Zeroes must be added
        # in the middle, so some rolling between a resize is used.
        psd = psd.to_timeseries()
        psd.roll(fil_len)
        psd.resize(int(wn_dur * sample_rate))
        psd.roll(-fil_len)
        # As time series is still mirrored the complex frequency components are
        # 0. But convert to real by using abs as in inverse_spectrum_truncate
        psd = psd.to_frequencyseries()
    else:
        psd = pycbc.psd.interpolate(psd, 1.0 / wn_dur)
                                1./psd,
                                fil_len,

    kmin = int(low_frequency_cutoff / psd.delta_f)
    psd[:kmin].clear()
    asd = (psd.squared_norm())**0.25
    del psd

    white_noise = normal(start_time - filter_duration,
                         end_time + filter_duration,
                         seed=seed,
                         sample_rate=sample_rate)
    white_noise = white_noise.to_frequencyseries()
    # Here we color. Do not want to duplicate memory here though so use '*='
    white_noise *= asd
    del asd
    colored = white_noise.to_timeseries()
    del white_noise
    return colored.time_slice(start_time, end_time)

def noise_from_string(psd_name, start_time, end_time,
                      seed=0,
                      sample_rate=16384,
                      low_frequency_cutoff=1.0,
                      filter_duration=128):

    Returns
    --------
    """
    delta_f = 1.0 / filter_duration
    flen = int(sample_rate / delta_f) // 2 + 1
    return colored_noise(psd, start_time, end_time,
                         sample_rate=sample_rate,
                         low_frequency_cutoff=low_frequency_cutoff,
                         filter_duration=filter_duration)

"""
A set of helper functions for evaluating rates.
"""

from scipy import integrate, optimize
import numpy as np, h5py
from numpy import log
import scipy.stats as ss

from pycbc.conversions import mchirp_from_mass1_mass2

def process_full_data(fname, rhomin, mass1, mass2, lo_mchirp, hi_mchirp):
    """Read the zero-lag and time-lag triggers identified by templates in
       a specified range of chirp mass.

       Parameters
       ----------
       hdfile:
              File that stores all the triggers
       rhomin: float
              Minimum value of SNR threhold (will need including ifar)
       mass1: array
              First mass of the waveform in the template bank
       mass2: array
              Second mass of the waveform in the template bank
       lo_mchirp: float
              Minimum chirp mass for the template
       hi_mchirp: float
              Maximum chirp mass for the template

       Returns
       -------
       dictionary
              containing foreground triggers and background information
    """
    with h5py.File(fname, 'r') as bulk:

        id_bkg = bulk['background_exc/template_id'][:]
        id_fg = bulk['foreground/template_id'][:]

        mchirp_bkg = mchirp_from_mass1_mass2(mass1[id_bkg], mass2[id_bkg])
        bound = np.sign((mchirp_bkg - lo_mchirp) * (hi_mchirp - mchirp_bkg))
        idx_bkg = np.where(bound == 1)
        mchirp_fg = mchirp_from_mass1_mass2(mass1[id_fg], mass2[id_fg])
        bound = np.sign((mchirp_fg - lo_mchirp) * (hi_mchirp - mchirp_fg))
        idx_fg = np.where(bound == 1)

        zerolagstat = bulk['foreground/stat'][:][idx_fg]
        cstat_back_exc = bulk['background_exc/stat'][:][idx_bkg]
        dec_factors = bulk['background_exc/decimation_factor'][:][idx_bkg]

    return {'zerolagstat': zerolagstat[zerolagstat > rhomin],
           'dec_factors': dec_factors[cstat_back_exc > rhomin],
           'cstat_back_exc': cstat_back_exc[cstat_back_exc > rhomin]}

def save_bkg_falloff(fname_statmap, fname_bank, path, rhomin, lo_mchirp, hi_mchirp):
    ''' Read the STATMAP files to derive snr falloff for the background events.
        Save the output to a txt file
        Bank file is also provided to restrict triggers to BBH templates.

        fname_statmap: string
               STATMAP file containing trigger information
        fname_bank: string
               File name of the template bank
        path: string
               Destination where txt file is saved
        rhomin: float
               Minimum value of SNR threhold (will need including ifar)
        lo_mchirp: float
               Minimum chirp mass for the template
        hi_mchirp: float
               Maximum chirp mass for template
    '''

    with h5py.File(fname_bank, 'r') as bulk:
        mass1_bank = bulk['mass1'][:]
        mass2_bank = bulk['mass2'][:]
        full_data = process_full_data(fname_statmap, rhomin,
                           mass1_bank, mass2_bank, lo_mchirp, hi_mchirp)

    max_bg_stat = np.max(full_data['cstat_back_exc'])
    bg_bins = np.linspace(rhomin, max_bg_stat, 76)
    bg_counts = np.histogram(full_data['cstat_back_exc'],
                         weights=full_data['dec_factors'], bins=bg_bins)[0]

    zerolagstat = full_data['zerolagstat']
    coincs = zerolagstat[zerolagstat >= rhomin]

    bkg = (bg_bins[:-1], bg_bins[1:], bg_counts)

    return bkg, coincs

def log_rho_bg(trigs, bins, counts):
    ''' Calculate the log of background fall-off

        trigs: array
               SNR values of all the triggers
        bins: string
               bins for histogrammed triggers
               counts for histogrammed triggers

        Returns
        -------
        array
    '''

    trigs = np.atleast_1d(trigs)

    N = sum(counts)

    assert np.all(trigs >= np.min(bins)), \
        'Trigger SNR values cannot all be below the lowest bin limit!'

    # If there are any triggers that are louder than the max bin, put one
    # fictitious count in a bin that extends from the limits of the slide
    # triggers out to the loudest trigger.

    # If there is no counts for a foreground trigger put a fictitious count
    # in the background bin
    if np.any(trigs >= np.max(bins)):
        N = N + 1
        #log_plimit = -np.log(N) - np.log(np.max(trigs) - bins[-1]) CHECK IT

    log_rhos = []
    for t in trigs:
        if t >= np.max(bins):
            log_rhos.append(-log(N)-log(np.max(trigs) - bins[-1]))
        else:
            i = bisect.bisect(bins, t) - 1

            if counts[i] == 0:
                counts[i] = 1
            log_rhos.append(log(counts[i]) - log(bins[i+1] - bins[i]) - log(N))
    return np.array(log_rhos)

def log_rho_fgmc(t, injstats, bins):
    counts, bins = np.histogram(injstats, bins)

    dens = counts / np.diff(bins) / float(N)

    assert np.min(t) >= np.min(bins)
    assert np.max(t) < np.max(bins)

    tinds = np.searchsorted(bins, t) - 1

    return log(dens[tinds])

def fgmc(log_fg_ratios, mu_log_vt, sigma_log_vt, Rf, maxfg):
    '''
    Function to fit the likelihood Fixme
    '''

    Lb = np.random.uniform(0., maxfg, len(Rf))
    pquit = 0

    while pquit < 0.1:
        # quit when the posterior on Lf is very close to its prior

        nsamp = len(Lb)
        Rf_sel = np.random.choice(Rf, nsamp)
        vt = np.random.lognormal(mu_log_vt, sigma_log_vt, len(Rf_sel))

        Lf = Rf_sel * vt

        log_Lf, log_Lb = log(Lf), log(Lb)

        plR = 0
        for lfr in log_fg_ratios:
            plR += np.logaddexp(lfr + log_Lf, log_Lb)

        plR -= (Lf + Lb)
        plRn = plR - max(plR)

        idx = np.exp(plRn) > np.random.random(len(plRn))

        pquit = ss.stats.ks_2samp(Lb, Lb[idx])[1]

        Lb = Lb[idx]

    return Rf_sel[idx], Lf[idx], Lb

def _optm(x, alpha, mu, sigma):
    '''Return probability density of skew-lognormal
       See scipy.optimize.curve_fit
    '''
    return ss.skewnorm.pdf(x, alpha, mu, sigma)

def fit(R):
    ''' Fit skew - lognormal to the rate samples achived from a prior analysis
        R: array
           Rate samples
        Returns
        -------
        ff[0]: float
            The skewness
        ff[1]: float
            The mean
        ff[2]: float
            The standard deviation
    '''

    lR = np.log(R)
    mu_norm, sigma_norm = np.mean(lR), np.std(lR)

    xs = np.linspace(min(lR), max(lR), 200)
    kde = ss.gaussian_kde(lR)
    pxs = kde(xs)

    # Initial guess has been taken as the mean and std-dev of the data
    # And a guess assuming small skewness
    ff = optimize.curve_fit(_optm, xs, pxs, p0 = [0.1, mu_norm, sigma_norm])[0]
    return ff[0], ff[1], ff[2]

def skew_lognormal_samples(alpha, mu, sigma, minrp, maxrp):
    ''' Returns a large number of Skew lognormal samples
        alpha: float
           Skewness of the distribution
        mu: float
           Mean of the distribution
        sigma: float
           Scale of the distribution
        minrp: float
           Minimum value for the samples
        maxrp: float
           Maximum value for the samples
        Returns
        -------
        Rfs: array
            Large number of samples (may need fixing)
    '''

    nsamp = 100000000
    lRu = np.random.uniform(minrp, maxrp, nsamp)
    plRu = ss.skewnorm.pdf(lRu, alpha, mu, sigma)
    rndn = np.random.random(nsamp)
    maxp = max(plRu)
    idx = np.where(plRu/maxp > rndn)
    log_Rf = lRu[idx]
    Rfs = np.exp(log_Rf)

    return Rfs

# The flat in log and power-law mass distribution models  #

# PDF for the two canonical models plus flat in mass model
def prob_lnm(m1, m2, s1z, s2z, **kwargs):
    ''' Return probability density for uniform in log
        m1: array
            Component masses 1
        m2: array
            Component masses 2
        s1z: array
            Aligned spin 1(Not in use currently)
        s2z:
            Aligned spin 2(Not in use currently)
        **kwargs: string
            Keyword arguments as model parameters
        Returns
        -------
        p_m1_m2: array
            The probability density for m1, m2 pair
    '''

    min_mass = kwargs.get('min_mass', 5.)
    max_mass = kwargs.get('max_mass', 95.)
    max_mtotal = min_mass + max_mass
    m1, m2 = np.array(m1), np.array(m2)

    C_lnm = integrate.quad(lambda x: (log(max_mtotal - x) - log(min_mass))/x, min_mass, max_mass)[0]

    xx = np.minimum(m1, m2)
    m1 = np.maximum(m1, m2)
    m2 = xx

    bound = np.sign(max_mtotal - m1 - m2)
    bound += np.sign(max_mass - m1) * np.sign(m2 - min_mass)
    idx = np.where(bound != 2)

    p_m1_m2 = (1/C_lnm)*(1./m1)*(1./m2)
    p_m1_m2[idx] = 0

    return p_m1_m2

def prob_imf(m1, m2, s1z, s2z, **kwargs):
    ''' Return probability density for power-law
        m1: array
        m2: array
        s1z: array
        s2z:

        Returns
        -------
           the probability density for m1, m2 pair
    '''

    alpha = kwargs.get('alpha', -2.35)

    C_imf = max_mass**(alpha + 1)/(alpha + 1)
    C_imf -= min_mass**(alpha + 1)/(alpha + 1)

    m2 = xx

    p_m1_m2 = np.zeros_like(m1)
    idx = np.where(m1 <= max_mtotal/2.)
    p_m1_m2[idx] = (1./C_imf) * m1[idx]**alpha /(m1[idx] - min_mass)
    idx = np.where(m1 > max_mtotal/2.)
    p_m1_m2[idx] = (1./C_imf) * m1[idx]**alpha /(max_mass - m1[idx])

    return p_m1_m2/2.

def prob_flat(m1, m2, s1z, s2z, **kwargs):
    ''' Return probability density for uniform in component mass
        m1: array
        m2: array
        s1z: array
            Aligned spin 1 (not in use currently)
        s2z:
            Aligned spin 2 (not in use currently)

        Returns
        -------
    '''

    min_mass = kwargs.get('min_mass', 1.)
    max_mass = kwargs.get('max_mass', 2.)

    bound = np.sign(m1 - m2)

    p_m1_m2 = 2. / (max_mass - min_mass)**2

# Generate samples for the two canonical models plus flat in mass model
def draw_imf_samples(**kwargs):
    ''' Draw samples for power-law model

           Keyword arguments as model parameters and number of samples

        Returns
        -------
        array
           The first mass
        array
           The second mass
    '''

    alpha_salpeter = kwargs.get('alpha', -2.35)
    nsamples = kwargs.get('nsamples', 1)

    a = (max_mass/min_mass)**(alpha_salpeter + 1.0) - 1.0
    beta = 1.0 / (alpha_salpeter + 1.0)

    k = nsamples * int(1.5 + log(1 + 100./nsamples))
    aa = min_mass * (1.0 + a * np.random.random(k))**beta
    bb = np.random.uniform(min_mass, aa, k)

    idx = np.where(aa + bb < max_mtotal)
    m1, m2 = (np.maximum(aa, bb))[idx], (np.minimum(aa, bb))[idx]

    return np.resize(m1, nsamples), np.resize(m2, nsamples)

def draw_lnm_samples(**kwargs):
    ''' Draw samples for uniform-in-log model

        Returns
        -------
        array
        array
    '''

    #PDF doesnt match with sampler
    lnmmin = log(min_mass)
    lnmmax = log(max_mass)

    aa = np.exp(np.random.uniform(lnmmin, lnmmax, k))
    bb = np.exp(np.random.uniform(lnmmin, lnmmax, k))

def draw_flat_samples(**kwargs):
    ''' Draw samples for uniform in mass

        Returns
        -------
        array
        array
    '''

    m1 = np.random.uniform(min_mass, max_mass, nsamples)
    m2 = np.random.uniform(min_mass, max_mass, nsamples)

    return np.maximum(m1, m2), np.minimum(m1, m2)

# Functions to generate chirp mass samples for the two canonical models
def mchirp_sampler_lnm(**kwargs):
    ''' Draw chirp mass samples for uniform-in-log model

        Returns
        -------
        mchirp-astro: array
           The chirp mass samples for the population
    '''
    m1, m2 = draw_lnm_samples(**kwargs)
    mchirp_astro = mchirp_from_mass1_mass2(m1, m2)

    return mchirp_astro

def mchirp_sampler_imf(**kwargs):
    ''' Draw chirp mass samples for power-law model

        Returns
        -------
    '''
    m1, m2 = draw_imf_samples(**kwargs)

def mchirp_sampler_flat(**kwargs):
    ''' Draw chirp mass samples for flat in mass model

        Returns
        -------
    '''
    m1, m2 = draw_flat_samples(**kwargs)

import copy, h5py
from astropy.cosmology import WMAP9 as cosmo

from pycbc.conversions import mchirp_from_mass1_mass2 as m1m2tomch

_mch_BNS = 1.4/2**.2
_redshifts, _d_lum, _I = np.arange(0., 5., 0.01), [], []
_save_params = ['mass1', 'mass2', 'spin1z', 'spin2z', 'spin1y', 'spin2y',
                                'spin1x', 'spin2x', 'distance', 'end_time']

for zz in _redshifts:
    _d_lum.append(cosmo.luminosity_distance(zz).value)
_dlum_interp = interp1d(_d_lum, _redshifts)

def read_injections(sim_files, m_dist, s_dist, d_dist):
    ''' Read all the injections from the files in the provided folder.
        The files must belong to individual set i.e. no files that combine
        all the injections in a run.
        Identify injection strategies and finds parameter boundaries.
        Collect injection according to GPS.

        sim_files: list
           List containign names of the simulation files
        m_dist: list
           The mass distribution used in the simulation runs
        s_dist: list
           The spin distribution used in the simulation runs
        d_dist: list
           The distance distribution used in the simulation runs

        Returns
        -------
        injections: dictionary
           Contains the organized information about the injections
    '''

    injections = {}
    min_d, max_d = 1e12, 0
    nf = len(sim_files)
    for i in range(nf):

        key = str(i)
        injections[key] = process_injections(sim_files[i])
        injections[key]['file_name'] = sim_files[i]
        injections[key]['m_dist'] = m_dist[i]
        injections[key]['s_dist'] = s_dist[i]
        injections[key]['d_dist'] = d_dist[i]

        mass1, mass2 = injections[key]['mass1'], injections[key]['mass2']
        distance = injections[key]['distance']

        mchirp = m1m2tomch(mass1, mass2)
        injections[key]['chirp_mass'] = mchirp
        injections[key]['total_mass'] = mass1 + mass2

        injections[key]['mtot_range'] = [min(mass1 + mass2), max(mass1 + mass2)]
        injections[key]['m1_range'] = [min(mass1), max(mass1)]
        injections[key]['m2_range'] = [min(mass2), max(mass2)]
        injections[key]['d_range'] = [min(distance), max(distance)]

        min_d, max_d = min(min_d, min(distance)), max(max_d, max(distance))

    injections['z_range'] = [dlum_to_z(min_d), dlum_to_z(max_d)]

def estimate_vt(injections, mchirp_sampler, model_pdf, **kwargs):
    #Try including ifar threshold
    '''Based on injection strategy and the desired astro model estimate the injected volume.
       Scale injections and estimate sensitive volume.

       injections: dictionary
            Dictionary obtained after reading injections from read_injections
       mchirp_sampler: function
            Sampler for producing chirp mass samples for the astro model.
       model_pdf: function
            The PDF for astro model in mass1-mass2-spin1z-spin2z space.
            This is easily extendible to include precession
       kwargs: key words
            Inputs for thresholds and astrophysical models

       Returns
       -------
       injection_chunks: dictionary
        The input dictionary with VT and VT error included with the injections
    '''

    thr_var = kwargs.get('thr_var')
    thr_val = kwargs.get('thr_val')

    nsamples = 1000000 #Used to calculate injected astro volume
    injections = copy.deepcopy(injections)
    min_z, max_z = injections['z_range']
    V = quad(contracted_dVdc, 0., max_z)[0]

    z_astro = astro_redshifts(min_z, max_z, nsamples)
    astro_lum_dist = cosmo.luminosity_distance(z_astro).value

    mch_astro = np.array(mchirp_sampler(nsamples = nsamples, **kwargs))
    mch_astro_det = mch_astro * (1. + z_astro)
    idx_within = np.zeros(nsamples)

    for key in injections.keys():

        if key == 'z_range':
            # This is repeated down again and is so
            continue

        mchirp = injections[key]['chirp_mass']
        min_mchirp, max_mchirp = min(mchirp),  max(mchirp)

        if injections[key]['d_dist'] == 'uniform':
            d_min, d_max = min(distance), max(distance)
        elif injections[key]['d_dist'] == 'dchirp':
            d_fid_min = min(distance / (mchirp/_mch_BNS)**(5/6.))
            d_fid_max = max(distance / (mchirp/_mch_BNS)**(5/6.))

            d_min = d_fid_min * (mch_astro_det/_mch_BNS)**(5/6.)
            d_max = d_fid_max * (mch_astro_det/_mch_BNS)**(5/6.)

        bound = np.sign((max_mchirp-mch_astro_det)*(mch_astro_det-min_mchirp))
        bound += np.sign((d_max - astro_lum_dist)*(astro_lum_dist - d_min))

        idx = np.where(bound == 2)
        idx_within[idx] = 1

    inj_V0 = 4*np.pi*V*len(idx_within[idx_within == 1])/float(nsamples)
    injections['inj_astro_vol'] = inj_V0

    # Estimate the sensitive volume
    z_range = injections['z_range']
    V_min = quad(contracted_dVdc, 0., z_range[0])[0]
    V_max = quad(contracted_dVdc, 0., z_range[1])[0]

    thr_falloff, i_inj, i_det, i_det_sq = [], 0, 0, 0
    gps_min, gps_max = 1e15, 0
    keys = injections.keys()
    for key in keys:

        if key == 'z_range' or key == 'inj_astro_vol':
            continue

        data = injections[key]
        distance = data['distance']
        mass1, mass2 = data['mass1'], data['mass2']
        spin1z, spin2z = data['spin1z'], data['spin2z']
        mchirp = data['chirp_mass']
        gps_min = min(gps_min, min(data['end_time']))
        gps_max = max(gps_max, max(data['end_time']))

        z_inj = dlum_to_z(distance)
        m1_sc, m2_sc = mass1/(1 + z_inj), mass2/(1 + z_inj)
        p_out = model_pdf(m1_sc, m2_sc, spin1z, spin2z)
        p_out *= pdf_z_astro(z_inj, V_min, V_max)

        p_in = 0
        J = cosmo.luminosity_distance(z_inj + 0.0005).value
        J -= cosmo.luminosity_distance(z_inj - 0.0005).value
        J = abs(J)/0.001 # A quick way to get dD_l/dz

        # Sum probability of injections from j-th set for all the strategies
        for key2 in keys:

            if key2 == 'z_range' or key2 == 'inj_astro_vol':
                continue

            dt_j = injections[key2]
            dist_j = dt_j['distance']
            m1_j, m2_j = dt_j['mass1'], dt_j['mass2']
            s1x_2, s2x_2 = dt_j['spin1x'], dt_j['spin2x']
            s1y_2, s2y_2 = dt_j['spin1y'], dt_j['spin2y']
            s1z_2, s2z_2 = dt_j['spin1z'], dt_j['spin2z']
            s1 = np.sqrt(s1x_2**2 + s1y_2**2 + s1z_2**2)
            s2 = np.sqrt(s2x_2**2 + s2y_2**2 + s2z_2**2)
            mch_j = dt_j['chirp_mass']

        #Get probability density for injections in mass-distance space
            if dt_j['m_dist'] == 'totalMass':
                lomass, himass = min(min(m1_j), min(m2_j), max(max(m1_j), max(m2_j)))
                lomass_2, himass_2 = lomass, himass
            elif dt_j['m_dist'] == 'componentMass' or dt_j['m_dist'] == 'log':
                lomass, himass = min(m1_j), max(m1_j)
                lomass_2, himass_2 = min(m2_j), max(m2_j)

            if dt_j['d_dist'] == 'dchirp':
                l_dist = min(dist_j / (mch_j/_mch_BNS)**(5/6.))
                h_dist = max(dist_j / (mch_j/_mch_BNS)**(5/6.))
            elif dt_j['d_dist'] == 'uniform':
                l_dist, h_dist = min(dist_j), max(dist_j)

            mdist = dt_j['m_dist']
            prob_mass = inj_mass_pdf(mdist, mass1, mass2,
                                          lomass, himass, lomass_2, himass_2)

            ddist = dt_j['d_dist']
            prob_dist = inj_distance_pdf(ddist, distance, l_dist,
                                                              h_dist, mchirp)

            hspin1, hspin2 = max(s1), max(s2)
            prob_spin = inj_spin_pdf(dt_j['s_dist'], hspin1, spin1z)
            prob_spin *= inj_spin_pdf(dt_j['s_dist'], hspin2, spin2z)

            p_in += prob_mass * prob_dist * prob_spin * J * (1 + z_inj)**2

        p_in[p_in == 0] = 1e12
        p_out_in = p_out/p_in

        i_inj += np.sum(p_out_in)
        i_det += np.sum((p_out_in)[data[thr_var] > thr_val])
        i_det_sq += np.sum((p_out_in)[data[thr_var] > thr_val]**2)

        idx_thr = np.where(data[thr_var] > thr_val)
        thrs = data[thr_var][idx_thr]
        ratios = p_out_in[idx_thr]/max(p_out_in[idx_thr])
        rndn = np.random.uniform(0, 1, len(ratios))
        idx_ratio = np.where(ratios > rndn)
        thr_falloff.append(thrs[idx_ratio])

    inj_V0 = injections['inj_astro_vol']
    injections['ninj'] = i_inj
    injections['ndet'] = i_det
    injections['ndetsq'] = i_det_sq
    injections['VT'] = ((inj_V0*i_det/i_inj) * (gps_max - gps_min)/31557600)
    injections['VT_err'] = injections['VT'] * np.sqrt(i_det_sq)/i_det
    injections['thr_falloff'] = np.hstack(np.array(thr_falloff).flat)

def process_injections(hdffile):
    """Function to read in the injection file and
       extract the found injections and all injections

       hdffile: hdf file
           File for which injections are to be processed

       Returns
       -------
       data: dictionary
           Dictionary containing injection read from the input file
    """
    data = {}

    with h5py.File(hdffile, 'r') as inp:
        found_index = inp['found_after_vetoes/injection_index'][:]

        for param in _save_params:
            data[param] = inp['injections/'+param][:]

        ifar = np.zeros_like(data[_save_params[0]])
        ifar[found_index] = inp['found_after_vetoes/ifar'][:]

        data['ifar'] = ifar

        stat = np.zeros_like(data[_save_params[0]])
        stat[found_index] = inp['found_after_vetoes/stat'][:]

        data['stat'] = stat

def dlum_to_z(dl):
    ''' Get the redshift for a luminosity distance

        dl: array
           The array of luminosity distances

        Returns
        -------
        array
           The redshift values corresponding to the luminosity distances
    '''

    return _dlum_interp(dl)

def astro_redshifts(min_z, max_z, nsamples):
    '''Sample the redshifts for sources, with redshift
            independent rate, using standard cosmology

       min_z: float
            Minimum redshift
       max_z: float
            Maximum redshift
       nsamples: int
            Number of samples

       Returns
       -------
       z_astro: array
            nsamples of redshift, between min_z, max_z, by standard cosmology
    '''

    dz, fac = 0.001, 3.0
    # use interpolation instead of directly estimating all the pdfz for rndz
    zbins = np.arange(min_z, max_z + dz/2., dz)
    zcenter = (zbins[:-1] + zbins[1:]) / 2
    pdfz = cosmo.differential_comoving_volume(zcenter).value/(1+zcenter)/V

    int_pdf = interp1d(zcenter, pdfz, bounds_error=False, fill_value=0)

    rndz = np.random.uniform(min_z, max_z, int(fac*nsamples))
    pdf_zs = int_pdf(rndz)
    maxpdf = max(pdf_zs)
    rndn = np.random.uniform(0, 1, int(fac*nsamples)) * maxpdf
    diff = pdf_zs - rndn
    idx = np.where(diff > 0)
    z_astro = rndz[idx]

    np.random.shuffle(z_astro)
    z_astro.resize(nsamples)

    return z_astro

def pdf_z_astro(z, V_min, V_max):
    ''' Get the probability density for the rate of events
        at a redshift assuming standard cosmology
    '''
    return contracted_dVdc(z)/(V_max - V_min)

def contracted_dVdc(z):
    #Return the time-dilated differential comoving volume
    return cosmo.differential_comoving_volume(z).value/(1+z)

##### Defining current standard strategies used for making injections #####

def inj_mass_pdf(key, mass1, mass2, lomass, himass, lomass_2 = 0, himass_2 = 0):

    '''Estimate the probability density based on the injection strategy

       key: string
          Injection strategy
          First mass of the injections
          Second mass of the injections
       lomass: float
          Lower value of the mass distributions
       himass: float
          higher value of the mass distribution

       Returns
       -------
       pdf: array
          Probability density of the injections
    '''

    mass1, mass2 = np.array(mass1), np.array(mass2)

    if key == 'totalMass':
        # Returns the PDF of mass when total mass is uniformly distributed.
        # Both the component masses have the same distribution for this case.

        # Parameters
        # ----------
        # lomass: lower component mass
        # himass: higher component mass

        bound = np.sign((lomass + himass) - (mass1 + mass2))
        bound += np.sign((himass - mass1)*(mass1 - lomass))
        bound += np.sign((himass - mass2)*(mass2 - lomass))
        idx = np.where(bound != 3)
        pdf = 1./(himass - lomass)/(mass1 + mass2 - 2 * lomass)
        pdf[idx] = 0

        return pdf

    if key == 'componentMass':
        # Returns the PDF of mass when component mass is uniformly
        # distributed. Component masses are independent for this case.

        bound = np.sign((himass - mass1)*(mass1 - lomass))
        bound += np.sign((himass_2 - mass2)*(mass2 - lomass_2))
        idx = np.where(bound != 2)
        pdf = np.ones_like(mass1) / (himass - lomass) / (himass_2 - lomass_2)

        return pdf

    if key == 'log':
        # Returns the PDF of mass when component mass is uniform in log.
        # Component masses are independent for this case.

        pdf = 1 / (log(himass) - log(lomass)) / (log(himass_2) - log(lomass_2))
        pdf /= (mass1 * mass2)

        return pdf

def inj_spin_pdf(key, high_spin, spinz):
    ''' Estimate the probability density of the
           injections for the spin distribution.

        key: string
          Injections strategy
        high_spin: float
          Maximum spin used in the strategy
        spinz: array
          Spin of the injections (for one component)
    '''

    # If the data comes from disable_spin simulation
    if spinz[0] == 0:
        return np.ones_like(spinz)

    spinz = np.array(spinz)

    bound = np.sign(np.absolute(high_spin) - np.absolute(spinz))
    bound += np.sign(1 - np.absolute(spinz))

    if key == 'precessing':
        # Returns the PDF of spins when total spin is
        # isotropically distributed. Both the component
        # masses have the same distribution for this case.

        pdf = (np.log(high_spin - np.log(abs(spinz)))/high_spin/2)

        return pdf

    if key == 'aligned':
        # Returns the PDF of mass when spins are aligned and uniformly
        # distributed. Component spins are independent for this case.

        pdf = (np.ones_like(spinz) / 2 / high_spin)

        return pdf

    if key == 'disable_spin':
        # Returns unit array

        pdf = np.ones_like(spinz)

        return pdf

def inj_distance_pdf(key, distance, low_dist, high_dist, mchirp = 1):
        injections for the distance distribution.

        distance: array
          Array of distances
        low_dist: float
          Lower value of distance used in the injection strategy
        high_dist: float
          Higher value of distance used in the injection strategy
    '''

    distance = np.array(distance)

    if key == 'uniform':
        # Returns the PDF at a distance when
        # distance is uniformly distributed.

        pdf = np.ones_like(distance)/(high_dist - low_dist)
        bound = np.sign((high_dist - distance)*(distance - low_dist))
        idx = np.where(bound != 1)
        return pdf

    if key == 'dchirp':
        # Returns the PDF at a distance when distance is uniformly
        # distributed but scaled by the chirp mass

        weight = (mchirp/_mch_BNS)**(5./6)
        pdf = np.ones_like(distance) / weight / (high_dist - low_dist)
        bound = np.sign((weight*high_dist - distance)*(distance - weight*low_dist))
        return pdf

# Copyright (C) 2012-2016 Alex Nitz, Tito Dal Canton, Leo Singer
#
#
#
"""Provides reference PSDs from LALSimulation.
"""

import numbers
import lal
import lalsimulation

# build a list of usable PSD functions from lalsimulation
_name_prefix = 'SimNoisePSD'
_name_suffix = 'Ptr'
_name_blacklist = ('FromFile', 'MirrorTherm', 'Quantum', 'Seismic', 'Shot', 'SuspTherm')
_psd_list = []
for _name in lalsimulation.__dict__:
    if _name != _name_prefix and _name.startswith(_name_prefix) and not _name.endswith(_name_suffix):
        _name = _name[len(_name_prefix):]
        if _name not in _name_blacklist:
            _psd_list.append(_name)
_psd_list = sorted(_psd_list)

# add functions wrapping lalsimulation PSDs
for _name in _psd_list:
    exec("""
def %s(length, delta_f, low_freq_cutoff):
    \"\"\"Return a FrequencySeries containing the %s PSD from LALSimulation.
    \"\"\"
    return from_string("%s", length, delta_f, low_freq_cutoff)
""" % (_name, _name, _name))

def get_psd_model_list():
    """ Returns a list of available reference PSD functions.

    Returns
    -------
    list
        Returns a list of names of reference PSD functions.
    """
    return get_lalsim_psd_list() + get_pycbc_psd_list()

def get_lalsim_psd_list():
    """Return a list of available reference PSD functions from LALSimulation.
    """
    return _psd_list

def get_pycbc_psd_list():
    """ Return a list of available reference PSD functions coded in PyCBC.

    Returns
    -------
    list
        Returns a list of names of all reference PSD functions coded in PyCBC.
    """
    pycbc_analytical_psd_list = pycbc_analytical_psds.keys()
    pycbc_analytical_psd_list = sorted(pycbc_analytical_psd_list)
    return pycbc_analytical_psd_list

def from_string(psd_name, length, delta_f, low_freq_cutoff):
    """Generate a frequency series containing a LALSimulation PSD specified
    by name.

    psd_name : string
        PSD name as found in LALSimulation, minus the SimNoisePSD prefix.
        Length of the frequency series in samples.
        Frequency resolution of the frequency series.
    low_freq_cutoff : float
        Frequencies below this value are set to zero.

    Returns
    -------
        The generated frequency series.
    """

    # check if valid PSD model
    if psd_name not in get_psd_model_list():
        raise ValueError(psd_name + ' not found among analytical '
                         'PSD functions.')

    # make sure length has the right type for CreateREAL8FrequencySeries
    if not isinstance(length, numbers.Integral) or length <= 0:
        raise TypeError('length must be a positive integer')
    length = int(length)

    # if PSD model is in LALSimulation
    if psd_name in get_lalsim_psd_list():
        lalseries = lal.CreateREAL8FrequencySeries(
            '', lal.LIGOTimeGPS(0), 0, delta_f, lal.DimensionlessUnit, length)
        try:
            func = lalsimulation.__dict__[
                                        _name_prefix + psd_name + _name_suffix]
            func = lalsimulation.__dict__[_name_prefix + psd_name]
            func(lalseries, low_freq_cutoff)
        else:
            lalsimulation.SimNoisePSD(lalseries, 0, func)
        psd = FrequencySeries(lalseries.data.data, delta_f=delta_f)

    # if PSD model is coded in PyCBC
    else:
        func = pycbc_analytical_psds[psd_name]
        psd = func(length, delta_f, low_freq_cutoff)

    # zero-out content below low-frequency cutoff
    kmin = int(low_freq_cutoff / delta_f)
    psd.data[:kmin] = 0

    return psd

def flat_unity(length, delta_f, low_freq_cutoff):
    """ Returns a FrequencySeries of ones above the low_frequency_cutoff.

        Length of output Frequencyseries.
        Frequency step for output FrequencySeries.
    low_freq_cutoff : int
        Low-frequency cutoff for output FrequencySeries.

    Returns
    -------
    FrequencySeries
        Returns a FrequencySeries containing the unity PSD model.
    """
    fseries = FrequencySeries(numpy.ones(length), delta_f=delta_f)
    kmin = int(low_freq_cutoff / fseries.delta_f)
    fseries.data[:kmin] = 0
    return fseries

# dict of analytical PSDs coded in PyCBC
pycbc_analytical_psds = {
    'flat_unity' : flat_unity,
}

# Copyright (C) 2012 Tito Dal Canton
#
#
#
"""Utilites to estimate PSDs from data.
"""

from pycbc.types import Array, FrequencySeries, TimeSeries, zeros
from pycbc.types import real_same_precision_as, complex_same_precision_as
from pycbc.fft import fft, ifft

def median_bias(n):
    """Calculate the bias of the median average PSD computed from `n` segments.

    n : int
        Number of segments used in PSD estimation.

    Returns
    -------
    ans : float
        Calculated bias.

    Raises
    ------
        For non-integer or non-positive `n`.

    Notes
    -----
    See arXiv:gr-qc/0509116 appendix B for details.
    """
    if type(n) is not int or n <= 0:
        raise ValueError('n must be a positive integer')
    if n >= 1000:
        return numpy.log(2)
    ans = 1
    for i in range(1, (n - 1) // 2 + 1):
        ans += 1.0 / (2*i + 1) - 1.0 / (2*i)
    return ans

def welch(timeseries, seg_len=4096, seg_stride=2048, window='hann',
          avg_method='median', num_segments=None, require_exact_data_fit=False):
    """PSD estimator based on Welch's method.

    timeseries : TimeSeries
        Time series for which the PSD is to be estimated.
    seg_len : int
        Segment length in samples.
        Separation between consecutive segments, in samples.
    window : {'hann', numpy.ndarray}
        Function used to window segments before Fourier transforming, or
        a `numpy.ndarray` that specifies the window.
    avg_method : {'median', 'mean', 'median-mean'}
        Method used for averaging individual segment PSDs.

    Returns
    -------
        Frequency series containing the estimated PSD.

    Raises
    ------
        For invalid choices of `seg_len`, `seg_stride` `window` and
        `avg_method` and for inconsistent combinations of len(`timeseries`),
        `seg_len` and `seg_stride`.

    Notes
    -----
    See arXiv:gr-qc/0509116 for details.
    """
    window_map = {
        'hann': numpy.hanning
    }

    # sanity checks
    if isinstance(window, numpy.ndarray) and window.size != seg_len:
        raise ValueError('Invalid window: incorrect window length')
    if not isinstance(window, numpy.ndarray) and window not in window_map:
        raise ValueError('Invalid window: unknown window {!r}'.format(window))
    if avg_method not in ('mean', 'median', 'median-mean'):
        raise ValueError('Invalid averaging method')
    if type(seg_len) is not int or type(seg_stride) is not int \
        or seg_len <= 0 or seg_stride <= 0:
        raise ValueError('Segment length and stride must be positive integers')

    if timeseries.precision == 'single':
        fs_dtype = numpy.complex64
    elif timeseries.precision == 'double':
        fs_dtype = numpy.complex128

    num_samples = len(timeseries)
    if num_segments is None:
        num_segments = int(num_samples // seg_stride)
        # NOTE: Is this not always true?
        if (num_segments - 1) * seg_stride + seg_len > num_samples:
            num_segments -= 1

    if not require_exact_data_fit:
        data_len = (num_segments - 1) * seg_stride + seg_len

        # Get the correct amount of data
        if data_len < num_samples:
            diff = num_samples - data_len
            start = diff // 2
            end = num_samples - diff // 2
            # Want this to be integers so if diff is odd, catch it here.
            if diff % 2:
                start = start + 1

            timeseries = timeseries[start:end]
            num_samples = len(timeseries)
        if data_len > num_samples:
            err_msg = "I was asked to estimate a PSD on %d " %(data_len)
            err_msg += "data samples. However the data provided only contains "
            err_msg += "%d data samples." %(num_samples)

    if num_samples != (num_segments - 1) * seg_stride + seg_len:
        raise ValueError('Incorrect choice of segmentation parameters')

    if not isinstance(window, numpy.ndarray):
        window = window_map[window](seg_len)
    w = Array(window.astype(timeseries.dtype))

    # calculate psd of each segment
    delta_f = 1. / timeseries.delta_t / seg_len
    segment_tilde = FrequencySeries(
        numpy.zeros(int(seg_len / 2 + 1)),
        delta_f=delta_f,
        dtype=fs_dtype,
    )

    segment_psds = []
    for i in range(num_segments):
        segment_start = i * seg_stride
        segment_end = segment_start + seg_len
        segment = timeseries[segment_start:segment_end]
        assert len(segment) == seg_len
        fft(segment * w, segment_tilde)
        seg_psd = abs(segment_tilde * segment_tilde.conj()).numpy()

        #halve the DC and Nyquist components to be consistent with TO10095
        seg_psd[0] /= 2
        seg_psd[-1] /= 2

        segment_psds.append(seg_psd)

    segment_psds = numpy.array(segment_psds)

    if avg_method == 'mean':
        psd = numpy.mean(segment_psds, axis=0)
    elif avg_method == 'median':
        psd = numpy.median(segment_psds, axis=0) / median_bias(num_segments)
    elif avg_method == 'median-mean':
        odd_psds = segment_psds[::2]
        even_psds = segment_psds[1::2]
        odd_median = numpy.median(odd_psds, axis=0) / \
            median_bias(len(odd_psds))
        even_median = numpy.median(even_psds, axis=0) / \
            median_bias(len(even_psds))
        psd = (odd_median + even_median) / 2

    psd *= 2 * delta_f * seg_len / (w*w).sum()

    return FrequencySeries(psd, delta_f=delta_f, dtype=timeseries.dtype,
                           epoch=timeseries.start_time)

def inverse_spectrum_truncation(psd, max_filter_len, low_frequency_cutoff=None, trunc_method=None):
    """Modify a PSD such that the impulse response associated with its inverse
    square root is no longer than `max_filter_len` time samples. In practice
    this corresponds to a coarse graining or smoothing of the PSD.

        PSD whose inverse spectrum is to be truncated.
    max_filter_len : int
        Maximum length of the time-domain filter in samples.
    low_frequency_cutoff : {None, int}
        Frequencies below `low_frequency_cutoff` are zeroed in the output.
    trunc_method : {None, 'hann'}
        Function used for truncating the time-domain filter.
        None produces a hard truncation at `max_filter_len`.

    Returns
    -------
        PSD whose inverse spectrum has been truncated.

    Raises
    ------
        For invalid types or values of `max_filter_len` and `low_frequency_cutoff`.

    Notes
    -----
    """
    if type(max_filter_len) is not int or max_filter_len <= 0:
        raise ValueError('max_filter_len must be a positive integer')
    if low_frequency_cutoff is not None and \
            (low_frequency_cutoff < 0 or
             low_frequency_cutoff > psd.sample_frequencies[-1]):
        raise ValueError('low_frequency_cutoff must be within the bandwidth of the PSD')

    N = (len(psd)-1)*2

    inv_asd = FrequencySeries(zeros(len(psd)), delta_f=psd.delta_f, \
        dtype=complex_same_precision_as(psd))

    kmin = 1
    if low_frequency_cutoff:
        kmin = int(low_frequency_cutoff / psd.delta_f)

    inv_asd[kmin:N//2] = (1.0 / psd[kmin:N//2]) ** 0.5
    q = TimeSeries(numpy.zeros(N), delta_t=(N / psd.delta_f), \
        dtype=real_same_precision_as(psd))
    ifft(inv_asd, q)

    trunc_start = max_filter_len // 2
    trunc_end = N - max_filter_len // 2
    if trunc_end < trunc_start:
        raise ValueError('Invalid value in inverse_spectrum_truncation')

    if trunc_method == 'hann':
        trunc_window = Array(numpy.hanning(max_filter_len), dtype=q.dtype)
        q[0:trunc_start] *= trunc_window[-trunc_start:]
        q[trunc_end:N] *= trunc_window[0:max_filter_len//2]

    if trunc_start < trunc_end:
        q[trunc_start:trunc_end] = 0
    psd_trunc = FrequencySeries(numpy.zeros(len(psd)), delta_f=psd.delta_f, \
                                dtype=complex_same_precision_as(psd))
    fft(q, psd_trunc)
    psd_trunc *= psd_trunc.conj()
    psd_out = 1. / abs(psd_trunc)

    return psd_out

def interpolate(series, delta_f):
    """Return a new PSD that has been interpolated to the desired delta_f.

    Returns
    -------
    """
    new_n = (len(series)-1) * series.delta_f / delta_f + 1
    samples = numpy.arange(0, numpy.rint(new_n)) * delta_f
    interpolated_series = numpy.interp(samples, series.sample_frequencies.numpy(), series.numpy())
    return FrequencySeries(interpolated_series, epoch=series.epoch,

def bandlimited_interpolate(series, delta_f):

    Returns
    -------
    """
    series = FrequencySeries(series, dtype=complex_same_precision_as(series), delta_f=series.delta_f)

    N = (len(series) - 1) * 2
    delta_t = 1.0 / series.delta_f / N

    new_N = int(1.0 / (delta_t * delta_f))
    new_n = new_N // 2 + 1

    series_in_time = TimeSeries(zeros(N), dtype=real_same_precision_as(series), delta_t=delta_t)
    ifft(series, series_in_time)

    padded_series_in_time = TimeSeries(zeros(new_N), dtype=series_in_time.dtype, delta_t=delta_t)
    padded_series_in_time[0:N//2] = series_in_time[0:N//2]
    padded_series_in_time[new_N-N//2:new_N] = series_in_time[N//2:N]

    interpolated_series = FrequencySeries(zeros(new_n), dtype=series.dtype, delta_f=delta_f)
    fft(padded_series_in_time, interpolated_series)

    return interpolated_series

# Copyright (C) 2012 Alex Nitz, Tito Dal Canton
#
#
#
"""Utilities to read PSDs from files.
"""

import scipy.interpolate

def from_numpy_arrays(freq_data, noise_data, length, delta_f, low_freq_cutoff):
    """Interpolate n PSD (as two 1-dimensional arrays of frequency and data)
    to the desired length, delta_f and low frequency cutoff.

    freq_data : array
        Array of frequencies.
    noise_data : array
        PSD values corresponding to frequencies in freq_arr.
        Frequency resolution of the frequency series in Herz.

    Returns
    -------
    """
    # Only include points above the low frequency cutoff
    if freq_data[0] > low_freq_cutoff:
        raise ValueError('Lowest frequency in input data '
          ' is higher than requested low-frequency cutoff ' + str(low_freq_cutoff))

    flow = kmin * delta_f

    data_start = (0 if freq_data[0]==low_freq_cutoff else numpy.searchsorted(freq_data, flow) - 1)

    # If the cutoff is exactly in the file, start there
    if freq_data[data_start+1] == low_freq_cutoff:
        data_start += 1

    freq_data = freq_data[data_start:]
    noise_data = noise_data[data_start:]

    flog = numpy.log(freq_data)
    slog = numpy.log(noise_data)

    psd_interp = scipy.interpolate.interp1d(flog, slog)

    psd = numpy.zeros(length, dtype=numpy.float64)

    vals = numpy.log(numpy.arange(kmin, length) * delta_f)
    psd[kmin:] =  numpy.exp(psd_interp(vals))

    return FrequencySeries(psd, delta_f=delta_f)

def from_txt(filename, length, delta_f, low_freq_cutoff, is_asd_file=True):
    """Read an ASCII file containing one-sided ASD or PSD  data and generate
    a frequency series with the corresponding PSD. The ASD or PSD data is
    interpolated in order to match the desired resolution of the
    generated frequency series.

    filename : string
        Path to a two-column ASCII file. The first column must contain
        the frequency (positive frequencies only) and the second column
        must contain the amplitude density OR power spectral density.
    is_asd_file : Boolean
        If false assume that the second column holds power spectral density.
        If true assume that the second column holds amplitude spectral density.
        Default: True

    Returns
    -------

    Raises
    ------
        If the ASCII file contains negative, infinite or NaN frequencies
        or amplitude densities.
    """
    file_data = numpy.loadtxt(filename)
    if (file_data < 0).any() or \
                            numpy.logical_not(numpy.isfinite(file_data)).any():
        raise ValueError('Invalid data in ' + filename)

    freq_data = file_data[:, 0]
    noise_data = file_data[:, 1]
    if is_asd_file:
        noise_data = noise_data ** 2

    return from_numpy_arrays(freq_data, noise_data, length, delta_f,
                             low_freq_cutoff)

def from_xml(filename, length, delta_f, low_freq_cutoff, ifo_string=None,
             root_name='psd'):

    ifo_string : string
        Use the PSD in the file's PSD dictionary with this ifo string.
        If not given and only one PSD present in the file return that, if not
        given and multiple (or zero) PSDs present an exception will be raised.
    root_name : string (default='psd')
        If given use this as the root name for the PSD XML file. If this means
        nothing to you, then it is probably safe to ignore this option.

    Returns
    -------

    """
    import lal.series
    from glue.ligolw import utils as ligolw_utils
    fp = open(filename, 'r')
    ct_handler = lal.series.PSDContentHandler
    fileobj, _ = ligolw_utils.load_fileobj(fp, contenthandler=ct_handler)
    psd_dict = lal.series.read_psd_xmldoc(fileobj, root_name=root_name)

    if ifo_string is not None:
        psd_freq_series = psd_dict[ifo_string]
    else:
        if len(psd_dict.keys()) == 1:
            psd_freq_series = psd_dict[tuple(psd_dict.keys())[0]]
        else:
            err_msg = "No ifo string given and input XML file contains not "
            err_msg += "exactly one PSD. Specify which PSD you want to use."

    noise_data = psd_freq_series.data.data[:]
    freq_data = numpy.arange(len(noise_data)) * psd_freq_series.deltaF

""" PSD Variation """

from numpy.fft import rfft, irfft
import scipy.signal as sig

import pycbc.psd

def mean_square(data, delta_t, srate, short_stride, stride):
    """ Calculate mean square of given time series once per stride

    First of all this function calculate the mean square of given time
    series once per short_stride. This is used to find and remove
    outliers due to short glitches. Here an outlier is defined as any
    element which is greater than two times the average of its closest
    neighbours. Every outlier is substituted with the average of the
    corresponding adjacent elements.
    Then, every second the function compute the mean square of the
    smoothed time series, within the stride.

    data : numpy.ndarray
        Duration of the time series
    srate : int
        Sample rate of the data were it given as a TimeSeries
    short_stride : float
        Stride duration for outlier removal
    stride ; float
        Stride duration

    Returns
    -------
    m_s: List
        Mean square of given time series
    """

    # Calculate mean square of data once per short stride and replace
    # outliers
    short_ms = numpy.mean(data.reshape(-1, int(srate * short_stride)) ** 2,
                          axis=1)
    # Define an array of averages that is used to substitute outliers
    ave = 0.5 * (short_ms[2:] + short_ms[:-2])
    outliers = short_ms[1:-1] > (2. * ave)
    short_ms[1:-1][outliers] = ave[outliers]

    # Calculate mean square of data every step within a window equal to
    # stride seconds
    m_s = []
    inv_time = int(1. / short_stride)
    for index in range(int(delta_t - stride + 1)):
        m_s.append(numpy.mean(short_ms[inv_time * index:inv_time *
                                       int(index+stride)]))
    return m_s

def calc_filt_psd_variation(strain, segment, short_segment, psd_long_segment,
                            psd_duration, psd_stride, psd_avg_method, low_freq,
                            high_freq):
    """ Calculates time series of PSD variability

    This function first splits the segment up into 512 second chunks. It
    then calculates the PSD over this 512 second. The PSD is used to
    to create a filter that is the composition of three filters:
    1. Bandpass filter between f_low and f_high.
    2. Weighting filter which gives the rough response of a CBC template.
    3. Whitening filter.
    Next it makes the convolution of this filter with the stretch of data.
    This new time series is given to the "mean_square" function, which
    computes the mean square of the timeseries within an 8 seconds window,
    once per second.
    The result, which is the variance of the S/N in that stride for the
    Parseval theorem, is then stored in a timeseries.

    strain : TimeSeries
        Input strain time series to estimate PSDs
    segment : {float, 8}
        Duration of the segments for the mean square estimation in seconds.
    short_segment : {float, 0.25}
        Duration of the short segments for the outliers removal.
    psd_long_segment : {float, 512}
        Duration of the long segments for PSD estimation in seconds.
    psd_duration : {float, 8}
        Duration of FFT segments for long term PSD estimation, in seconds.
    psd_stride : {float, 4}
        Separation between FFT segments for long term PSD estimation, in
        seconds.
    psd_avg_method : {string, 'median'}
        Method for averaging PSD estimation segments.
    low_freq : {float, 20}
        Minimum frequency to consider the comparison between PSDs.
    high_freq : {float, 480}
        Maximum frequency to consider the comparison between PSDs.

    Returns
    -------
    psd_var : TimeSeries
        Time series of the variability in the PSD estimation
    """
    # Calculate strain precision
    if strain.precision == 'single':
        fs_dtype = numpy.float32
    elif strain.precision == 'double':
        fs_dtype = numpy.float64

    # Convert start and end times immediately to floats
    start_time = numpy.float(strain.start_time)
    end_time = numpy.float(strain.end_time)

    # Resample the data
    strain = resample_to_delta_t(strain, 1.0 / 2048)
    srate = int(strain.sample_rate)

    # Fix the step for the PSD estimation and the time to remove at the
    # edge of the time series.
    step = 1.0
    strain_crop = 8.0

    # Find the times of the long segments
    times_long = numpy.arange(start_time, end_time,
                              psd_long_segment - 2 * strain_crop
                              - segment + step)

    # Create a bandpass filter between low_freq and high_freq
    filt = sig.firwin(4 * srate, [low_freq, high_freq], pass_zero=False,
                      window='hann', nyq=srate / 2)
    filt.resize(int(psd_duration * srate))
    # Fourier transform the filter and take the absolute value to get
    # rid of the phase.
    filt = abs(rfft(filt))

    psd_var_list = []
    for tlong in times_long:
        # Calculate PSD for long segment
        if tlong + psd_long_segment <= float(end_time):
            astrain = strain.time_slice(tlong, tlong + psd_long_segment)
            plong = pycbc.psd.welch(
                astrain,
                seg_len=int(psd_duration * strain.sample_rate),
                seg_stride=int(psd_stride * strain.sample_rate),
                avg_method=psd_avg_method)
        else:
            astrain = strain.time_slice(tlong, end_time)
                           strain.time_slice(end_time - psd_long_segment,
                                             end_time),
                           seg_len=int(psd_duration * strain.sample_rate),
                           seg_stride=int(psd_stride * strain.sample_rate),
                           avg_method=psd_avg_method)
        astrain = astrain.numpy()
        freqs = numpy.array(plong.sample_frequencies, dtype=fs_dtype)
        plong = plong.numpy()

        # Make the weighting filter - bandpass, which weight by f^-7/6,
        # and whiten. The normalization is chosen so that the variance
        # will be one if this filter is applied to white noise which
        # already has a variance of one.
        fweight = freqs ** (-7./6.) * filt / numpy.sqrt(plong)
        fweight[0] = 0.
        norm = (sum(abs(fweight) ** 2) / (len(fweight) - 1.)) ** -0.5
        fweight = norm * fweight
        fwhiten = numpy.sqrt(2. / srate) / numpy.sqrt(plong)
        fwhiten[0] = 0.
        full_filt = sig.hann(int(psd_duration * srate)) * numpy.roll(
            irfft(fwhiten * fweight), int(psd_duration / 2) * srate)
        # Convolve the filter with long segment of data
        wstrain = sig.fftconvolve(astrain, full_filt, mode='same')
        wstrain = wstrain[int(strain_crop * srate):-int(strain_crop * srate)]
        # compute the mean square of the chunk of data
        delta_t = len(wstrain) * strain.delta_t
        variation = mean_square(wstrain, delta_t, srate, short_segment, segment)
        psd_var_list.append(numpy.array(variation, dtype=wstrain.dtype))

    # Package up the time series to return
    psd_var = TimeSeries(numpy.concatenate(psd_var_list), delta_t=step,
                         epoch=start_time + strain_crop + segment)

    return psd_var

def find_trigger_value(psd_var, idx, start, sample_rate):
    """ Find the PSD variation value at a particular time with the filter
    method. If the time is outside the timeseries bound, 1. is given.

        Time series of the varaibility in the PSD estimation
    idx : numpy.ndarray
        Time indices of the triggers
        GPS start time
    sample_rate : float
        Sample rate defined in ini file

    Returns
    -------
    vals : Array
        PSD variation value at a particular time
    """
    # Find gps time of the trigger
    time = start + idx / sample_rate
    # Extract the PSD variation at trigger time through linear
    # interpolation
    if not hasattr(psd_var, 'cached_psd_var_interpolant'):
        from scipy import interpolate
        psd_var.cached_psd_var_interpolant = \
            interpolate.interp1d(psd_var.sample_times.numpy(), psd_var.numpy(),
                                 fill_value=1.0, bounds_error=False)
    vals = psd_var.cached_psd_var_interpolant(time)

    return vals

# Copyright (C) 2014 Alex Nitz, Andrew Miller, Tito Dal Canton
#
#
#
from pycbc.psd.read import *
from pycbc.psd.analytical import *
from pycbc.psd.estimate import *
from pycbc.psd.variation import *
from pycbc.types import float32,float64
from pycbc.types import MultiDetOptionAppendAction, MultiDetOptionAction
from pycbc.types import copy_opts_for_single_ifo
from pycbc.types import required_opts, required_opts_multi_ifo
from pycbc.types import ensure_one_opt, ensure_one_opt_multi_ifo

def from_cli(opt, length, delta_f, low_frequency_cutoff,
             strain=None, dyn_range_factor=1, precision=None):
    """Parses the CLI options related to the noise PSD and returns a
    FrequencySeries with the corresponding PSD. If necessary, the PSD is
    linearly interpolated to achieve the resolution specified in the CLI.

        required attributes (psd_model, psd_file, asd_file, psd_estimation,
        psd_segment_length, psd_segment_stride, psd_inverse_length,
        psd_output).
        The length in samples of the output PSD.
        The frequency step of the output PSD.
    low_frequency_cutoff: float
        The low frequncy cutoff to use when calculating the PSD.
    strain : {None, TimeSeries}
        Time series containing the data from which the PSD should be measured,
        when psd_estimation is in use.
    dyn_range_factor : {1, float}
        For PSDs taken from models or text files, if `dyn_range_factor` is
        not None, then the PSD is multiplied by `dyn_range_factor` ** 2.
    precision : str, choices (None,'single','double')
        If not specified, or specified as None, the precision of the returned
        PSD will match the precision of the data, if measuring a PSD, or will
        match the default precision of the model if using an analytical PSD.
        If 'single' the PSD will be converted to float32, if not already in
        that precision. If 'double' the PSD will be converted to float64, if
        not already in that precision.

    Returns
    -------
        The frequency series containing the PSD.
    """
    f_low = low_frequency_cutoff
    sample_rate = int((length -1) * 2 * delta_f)

    try:
        psd_estimation = opt.psd_estimation is not None
        psd_estimation = False

    exclusive_opts = [opt.psd_model, opt.psd_file, opt.asd_file,
                      psd_estimation]
    if sum(map(bool, exclusive_opts)) != 1:
        err_msg = "You must specify exactly one of '--psd-file', "
        err_msg += "'--psd-model', '--asd-file', '--psd-estimation'"

    if (opt.psd_model or opt.psd_file or opt.asd_file):
        # PSD from lalsimulation or file
        if opt.psd_model:
            psd = from_string(opt.psd_model, length, delta_f, f_low)
        elif opt.psd_file or opt.asd_file:
            if opt.asd_file:
                psd_file_name = opt.asd_file
            else:
                psd_file_name = opt.psd_file
            if psd_file_name.endswith(('.dat', '.txt')):
                is_asd_file = bool(opt.asd_file)
                psd = from_txt(psd_file_name, length,
                               delta_f, f_low, is_asd_file=is_asd_file)
            elif opt.asd_file:
                err_msg = "ASD files are only valid as ASCII files (.dat or "
                err_msg += ".txt). Supplied {}.".format(psd_file_name)
            elif psd_file_name.endswith(('.xml', '.xml.gz')):
                psd = from_xml(psd_file_name, length, delta_f, f_low,
                               ifo_string=opt.psd_file_xml_ifo_string,
                               root_name=opt.psd_file_xml_root_name)
        # Set values < flow to the value at flow
        psd[0:kmin] = psd[kmin]

        psd *= dyn_range_factor ** 2

    elif psd_estimation:
        # estimate PSD from data
        psd = welch(strain, avg_method=opt.psd_estimation,
                    seg_len=int(opt.psd_segment_length * sample_rate),
                    seg_stride=int(opt.psd_segment_stride * sample_rate),
                    num_segments=opt.psd_num_segments,
                    require_exact_data_fit=False)

        if delta_f != psd.delta_f:
            psd = interpolate(psd, delta_f)
    else:
        # Shouldn't be possible to get here
        raise ValueError("Shouldn't be possible to raise this!")

    if opt.psd_inverse_length:
        psd = inverse_spectrum_truncation(psd,
            int(opt.psd_inverse_length * sample_rate),
            low_frequency_cutoff=f_low)

    if hasattr(opt, 'psd_output') and opt.psd_output:
        (psd.astype(float64) / (dyn_range_factor ** 2)).save(opt.psd_output)

    if precision is None:
        return psd
    elif precision == 'single':
        return psd.astype(float32)
    elif precision == 'double':
        return psd.astype(float64)
    else:
        err_msg = "If provided the precision kwarg must be either 'single' "
        err_msg += "or 'double'. You provided %s." %(precision)

def from_cli_single_ifo(opt, length, delta_f, low_frequency_cutoff, ifo,
             **kwargs):
    """
    Get the PSD for a single ifo when using the multi-detector CLI
    """
    single_det_opt = copy_opts_for_single_ifo(opt, ifo)
    return from_cli(single_det_opt, length, delta_f, low_frequency_cutoff,
                    **kwargs)

def from_cli_multi_ifos(opt, length_dict, delta_f_dict,
                        low_frequency_cutoff_dict, ifos, strain_dict=None,
                        **kwargs):
    """
    Get the PSD for all ifos when using the multi-detector CLI
    """
    psd = {}
        if strain_dict is not None:
            strain = strain_dict[ifo]
        else:
            strain = None
        psd[ifo] = from_cli_single_ifo(opt, length_dict[ifo], delta_f_dict[ifo],
                                       low_frequency_cutoff_dict[ifo], ifo,
                                       strain=strain, **kwargs)
    return psd

def insert_psd_option_group(parser, output=True, include_data_options=True):
    """
    Adds the options used to call the pycbc.psd.from_cli function to an
    optparser as an OptionGroup. This should be used if you
    want to use these options in your code.

    """
    psd_options = parser.add_argument_group(
                          "Options to select the method of PSD generation",
                          "The options --psd-model, --psd-file, --asd-file, "
                          "and --psd-estimation are mutually exclusive.")
    psd_options.add_argument("--psd-model",
                             help="Get PSD from given analytical model. ",
                             choices=get_psd_model_list())
    psd_options.add_argument("--psd-file",
                             help="Get PSD using given PSD ASCII file")
    psd_options.add_argument("--asd-file",
                             help="Get PSD using given ASD ASCII file")
    psd_options.add_argument("--psd-inverse-length", type=float,
                             help="(Optional) The maximum length of the "
                             "impulse response of the overwhitening "
                             "filter (s)")
    # Options specific to XML PSD files
    psd_options.add_argument("--psd-file-xml-ifo-string",
                             help="If using an XML PSD file, use the PSD in "
                                  "the file's PSD dictionary with this "
                                  "ifo string. If not given and only one "
                                  "PSD present in the file return that, if "
                                  "not given and multiple (or zero) PSDs "
                                  "present an exception will be raised.")
    psd_options.add_argument("--psd-file-xml-root-name", default='psd',
                             help="If given use this as the root name for "
                                  "the PSD XML file. If this means nothing "
                                  "to you, then it is probably safe to "
                                  "ignore this option.")
    # Options for PSD variation
    psd_options.add_argument("--psdvar-segment", type=float,
                             metavar="SECONDS", help="Length of segment "
                             "for mean square calculation of PSD variation.")
    psd_options.add_argument("--psdvar-short-segment", type=float,
                             metavar="SECONDS", help="Length of short segment "
                             "for outliers removal in PSD variability "
                             "calculation.")
    psd_options.add_argument("--psdvar-long-segment", type=float,
                             metavar="SECONDS", help="Length of long segment "
                             "when calculating the PSD variability.")
    psd_options.add_argument("--psdvar-psd-duration", type=float,
                             metavar="SECONDS", help="Duration of short "
                             "segments for PSD estimation.")
    psd_options.add_argument("--psdvar-psd-stride", type=float,
                             metavar="SECONDS", help="Separation between PSD "
                             "estimation segments.")
    psd_options.add_argument("--psdvar-low-freq", type=float, metavar="HERTZ",
                             help="Minimum frequency to consider in strain "
                             "bandpass.")
    psd_options.add_argument("--psdvar-high-freq", type=float, metavar="HERTZ",
                             help="Maximum frequency to consider in strain "

    if include_data_options :
        psd_options.add_argument("--psd-estimation",
                                 help="Measure PSD from the data, using "
                                 "given average method.",
                                 choices=["mean", "median", "median-mean"])
        psd_options.add_argument("--psd-segment-length", type=float,
                                 help="(Required for --psd-estimation) The "
                                 "segment length for PSD estimation (s)")
        psd_options.add_argument("--psd-segment-stride", type=float,
                                 help="(Required for --psd-estimation) "
                                 "The separation between consecutive "
                                 "segments (s)")
        psd_options.add_argument("--psd-num-segments", type=int, default=None,
                                 help="(Optional, used only with "
                                 "--psd-estimation). If given, PSDs will "
                                 "be estimated using only this number of "
                                 "segments. If more data is given than "
                                 "needed to make this number of segments "
                                 "then excess data will not be used in "
                                 "the PSD estimate. If not enough data "
                                 "is given, the code will fail.")
    if output:
        psd_options.add_argument("--psd-output",
                          help="(Optional) Write PSD to specified file")

    return psd_options

def insert_psd_option_group_multi_ifo(parser):
    """

    """
    psd_options.add_argument("--psd-model", nargs="+",
                          action=MultiDetOptionAction, metavar='IFO:MODEL',
                          help="Get PSD from given analytical model. "
                          "Choose from %s" %(', '.join(get_psd_model_list()),))
    psd_options.add_argument("--psd-file", nargs="+",
                          action=MultiDetOptionAction, metavar='IFO:FILE',
                          help="Get PSD using given PSD ASCII file")
    psd_options.add_argument("--asd-file", nargs="+",
                          help="Get PSD using given ASD ASCII file")
    psd_options.add_argument("--psd-estimation", nargs="+",
                          help="Measure PSD from the data, using given "
                          "average method. Choose from "
                          "mean, median or median-mean.")
    psd_options.add_argument("--psd-segment-length", type=float, nargs="+",
                          action=MultiDetOptionAction, metavar='IFO:LENGTH',
                          help="(Required for --psd-estimation) The segment "
                               "length for PSD estimation (s)")
    psd_options.add_argument("--psd-segment-stride", type=float, nargs="+",
                          action=MultiDetOptionAction, metavar='IFO:STRIDE',
                          help="(Required for --psd-estimation) The separation"
                               " between consecutive segments (s)")
    psd_options.add_argument("--psd-num-segments", type=int, nargs="+",
                          default=None,
                          action=MultiDetOptionAction, metavar='IFO:NUM',
                          help="(Optional, used only with --psd-estimation). "
                               "If given PSDs will be estimated using only "
                               "this number of segments. If more data is "
                               "given than needed to make this number of "
                               "segments than excess data will not be used in "
                               "the PSD estimate. If not enough data is given "
                               "the code will fail.")
    psd_options.add_argument("--psd-inverse-length", type=float, nargs="+",
                          help="(Optional) The maximum length of the impulse"
                          " response of the overwhitening filter (s)")
    psd_options.add_argument("--psd-output", nargs="+",

ensure_one_opt_groups = []
ensure_one_opt_groups.append(['--psd-file', '--psd-model',
                              '--psd-estimation', '--asd-file'])

def verify_psd_options(opt, parser):
    """Parses the CLI options and verifies that they are consistent and
    reasonable.

        psd_segment_length, psd_segment_stride, psd_inverse_length, psd_output).
    """
    try:

    for opt_group in ensure_one_opt_groups:
        ensure_one_opt(opt, parser, opt_group)

    if psd_estimation:
        required_opts(opt, parser,
                      ['--psd-segment-stride', '--psd-segment-length'],
                      required_by = "--psd-estimation")

def verify_psd_options_multi_ifo(opt, parser, ifos):

    """
        for opt_group in ensure_one_opt_groups:
            ensure_one_opt_multi_ifo(opt, parser, ifo, opt_group)

        if opt.psd_estimation[ifo]:
            required_opts_multi_ifo(opt, parser, ifo,
                          required_by = "--psd-estimation")

def generate_overlapping_psds(opt, gwstrain, flen, delta_f, flow,
                              dyn_range_factor=1., precision=None):
    """Generate a set of overlapping PSDs to cover a stretch of data. This
    allows one to analyse a long stretch of data with PSD measurements that
    change with time.

    gwstrain : Strain object
        The timeseries of raw data on which to estimate PSDs.
    flen : int
        The length in samples of the output PSDs.
        The frequency step of the output PSDs.

    Returns
    --------
    psd_and_times : list of (start, end, PSD) tuples
        This is a list of tuples containing one entry for each PSD. The first
        and second entries (start, end) in each tuple represent the index
        range of the gwstrain data that was used to estimate that PSD. The
        third entry (psd) contains the PSD estimate between that interval.
    """
    if not opt.psd_estimation:
        psd = from_cli(opt, flen, delta_f, flow, strain=gwstrain,
                       dyn_range_factor=dyn_range_factor, precision=precision)
        psds_and_times = [ (0, len(gwstrain), psd) ]
        return psds_and_times

    # Figure out the data length used for PSD generation
    seg_stride = int(opt.psd_segment_stride * gwstrain.sample_rate)
    seg_len = int(opt.psd_segment_length * gwstrain.sample_rate)
    input_data_len = len(gwstrain)

    if opt.psd_num_segments is None:
        # FIXME: Should we make --psd-num-segments mandatory?
        #        err_msg = "You must supply --num-segments."
        #        raise ValueError(err_msg)
        num_segments = int(input_data_len // seg_stride) - 1
    else:
        num_segments = int(opt.psd_num_segments)

    psd_data_len = (num_segments - 1) * seg_stride + seg_len

    # How many unique PSD measurements is this?
    psds_and_times = []
    if input_data_len < psd_data_len:
        err_msg = "Input data length must be longer than data length needed "
        err_msg += "to estimate a PSD. You specified that a PSD should be "
        err_msg += "estimated with %d seconds. " %(psd_data_len)
        err_msg += "Input data length is %d seconds. " %(input_data_len)
    elif input_data_len == psd_data_len:
        num_psd_measurements = 1
        psd_stride = 0
    else:
        num_psd_measurements = int(2 * (input_data_len-1) / psd_data_len)
        psd_stride = int((input_data_len - psd_data_len) / num_psd_measurements)

    for idx in range(num_psd_measurements):
        if idx == (num_psd_measurements - 1):
            start_idx = input_data_len - psd_data_len
            end_idx = input_data_len
        else:
            start_idx = psd_stride * idx
            end_idx = psd_data_len + psd_stride * idx
        strain_part = gwstrain[start_idx:end_idx]
        psd = from_cli(opt, flen, delta_f, flow, strain=strain_part,
        psds_and_times.append( (start_idx, end_idx, psd) )
    return psds_and_times

def associate_psds_to_segments(opt, fd_segments, gwstrain, flen, delta_f, flow,
                               dyn_range_factor=1., precision=None):
    """Generate a set of overlapping PSDs covering the data in GWstrain.
    Then associate these PSDs with the appropriate segment in strain_segments.

    fd_segments : StrainSegments.fourier_segments() object
        The fourier transforms of the various analysis segments. The psd
        attribute of each segment is updated to point to the appropriate PSD.
    flen : int
    """
    psds_and_times = generate_overlapping_psds(opt, gwstrain, flen, delta_f,
                                       flow, dyn_range_factor=dyn_range_factor,
                                       precision=precision)

    for fd_segment in fd_segments:
        best_psd = None
        psd_overlap = 0
        inp_seg = segments.segment(fd_segment.seg_slice.start,
                                   fd_segment.seg_slice.stop)
        for start_idx, end_idx, psd in psds_and_times:
            psd_seg = segments.segment(start_idx, end_idx)
            if psd_seg.intersects(inp_seg):
                curr_overlap = abs(inp_seg & psd_seg)
                if curr_overlap > psd_overlap:
                    psd_overlap = curr_overlap
                    best_psd = psd
        if best_psd is None:
            err_msg = "No PSDs found intersecting segment!"
        fd_segment.psd = best_psd

def associate_psds_to_single_ifo_segments(opt, fd_segments, gwstrain, flen,
                                          delta_f, flow, ifo,
                                          dyn_range_factor=1., precision=None):
    """
    Associate PSDs to segments for a single ifo when using the multi-detector
    CLI
    """
    associate_psds_to_segments(single_det_opt, fd_segments, gwstrain, flen,
                               delta_f, flow, dyn_range_factor=dyn_range_factor,
                               precision=precision)

def associate_psds_to_multi_ifo_segments(opt, fd_segments, gwstrain, flen,
                                         delta_f, flow, ifos,
                                         dyn_range_factor=1., precision=None):
    """
    Associate PSDs to segments for all ifos when using the multi-detector CLI
    """
        if gwstrain is not None:
            strain = gwstrain[ifo]
        else:

        if fd_segments is not None:
            segments = fd_segments[ifo]
        else:
            segments = None

        associate_psds_to_single_ifo_segments(opt, segments, strain, flen,
                delta_f, flow, ifo, dyn_range_factor=dyn_range_factor,
                precision=precision)

""" Utilities for managing matplotlib colors and mapping ifos to color
"""

_ifo_color_map = {
    'G1': '#222222',  # dark gray
    'K1': '#ffb200',  # yellow/orange
    'H1': '#ee0000',  # red
    'I1': '#b0dd8b',  # light green
    'L1': '#4ba6ff',  # blue
    'V1': '#9b59b6',  # magenta/purple
}

_source_color_map = {
    'BNS': '#A2C8F5',   # light blue
    'NSBH': '#FFB482',  # light orange
    'BBH': '#FE9F9B',   # light red
    'Mass Gap': '#8EE5A1',  # light green
    'GNS': '#98D6CB',   # turquoise
    'GG': '#79BB87',    # green
    'BHG': '#C6C29E'    # dark khaki
}

def ifo_color(ifo):
    return _ifo_color_map[ifo]

def source_color(source):
    return _source_color_map[source]

'''This module contains utilities for following up search triggers'''

# JavaScript for searching the aLOG
redirect_javascript = """<script type="text/javascript">
function redirect(form,way)
{
        // Set location to form and submit.
        if(form != '')
        {
                document.forms[form].action=way;
                document.forms[form].submit();
        }
        else
        {
                window.top.location = way;
        }
}
</script>"""

search_form_string="""<form name="%s_alog_search" id="%s_alog_search" method="post">
<input type="hidden" name="srcDateFrom" id="srcDateFrom" value="%s" size="20"/>
<input type="hidden" name="srcDateTo" id="srcDateTo" value="%s" size="20"/>
</form>"""

data_h1_string = """H1
&nbsp;
<a href=https://ldas-jobs.ligo-wa.caltech.edu/~detchar/summary/day/%s>
Summary</a>
&nbsp;
<a onclick="redirect('h1_alog_search',
'https://alog.ligo-wa.caltech.edu/aLOG/includes/search.php?adminType=search');
return true;">aLOG</a>"""

data_l1_string="""L1
&nbsp;
<a href=https://ldas-jobs.ligo-la.caltech.edu/~detchar/summary/day/%s>
&nbsp;
<a onclick="redirect('l1_alog_search',
'https://alog.ligo-la.caltech.edu/aLOG/includes/search.php?adminType=search');

def get_summary_page_link(ifo, utc_time):
    """Return a string that links to the summary page and aLOG for this ifo

    ifo : string
        The detector name
    utc_time : sequence
        First three elements must be strings giving year, month, day resp.

    Returns
    -------
    return_string : string
        String containing HTML for links to summary page and aLOG search
    """
    search_form = search_form_string
    data = {'H1': data_h1_string, 'L1': data_l1_string}
    if ifo not in data:
        return ifo
    else:
        # alog format is day-month-year
        alog_utc = '%02d-%02d-%4d' % (utc_time[2], utc_time[1], utc_time[0])
        # summary page is exactly the reverse
        ext = '%4d%02d%02d' % (utc_time[0], utc_time[1], utc_time[2])
        return_string = search_form % (ifo.lower(), ifo.lower(), alog_utc, alog_utc)
        return return_string + data[ifo] % ext

# Copyright (C) 2014 Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
""" This module provides functions to generate followup plots and trigger
time series.
"""
import h5py, numpy, matplotlib
# Only if a backend is not already set ... This should really *not* be done
# here, but in the executables you should set matplotlib.use()
# This matches the check that matplotlib does internally, but this *may* be
# version dependenant. If this is a problem then remove this and control from
# the executables directly.
import sys
if 'matplotlib.backends' not in sys.modules:
    matplotlib.use('agg')
import pylab, mpld3, mpld3.plugins
from ligo.segments import segment

def columns_from_file_list(file_list, columns, ifo, start, end):
    """ Return columns of information stored in single detector trigger
    files.

    file_list_file : string
        pickle file containing the list of single detector
    triggers.
        The ifo to return triggers for.
    columns : list of strings
        The list of columns to read from the trigger files.
    start : int
        The start time to get triggers from
    end : int
        The end time to get triggers from

    Returns
    -------
    trigger_dict : dict
        A dictionary of column vectors with column names as keys.
    """
    file_list = file_list.find_output_with_ifo(ifo)
    file_list = file_list.find_all_output_in_range(ifo, segment(start, end))

    trig_dict = {}
    for trig_file in file_list:
        f = h5py.File(trig_file.storage_path, 'r')

        time = f['end_time'][:]
        pick = numpy.logical_and(time < end, time > start)
        pick_loc = numpy.where(pick)[0]

        for col in columns:
            if col not in trig_dict:
                trig_dict[col] = []
            trig_dict[col] = numpy.concatenate([trig_dict[col], f[col][:][pick_loc]])

    return trig_dict

ifo_color = {'H1': 'blue', 'L1':'red', 'V1':'green'}

def coinc_timeseries_plot(coinc_file, start, end):
    fig = pylab.figure()
    f = h5py.File(coinc_file, 'r')

    stat1 = f['foreground/stat1']
    stat2 = f['foreground/stat2']
    time1 = f['foreground/time1']
    time2 = f['foreground/time2']
    ifo1 = f.attrs['detector_1']
    ifo2 = f.attrs['detector_2']

    pylab.scatter(time1, stat1, label=ifo1, color=ifo_color[ifo1])
    pylab.scatter(time2, stat2, label=ifo2, color=ifo_color[ifo2])

    fmt = '.12g'
    mpld3.plugins.connect(fig, mpld3.plugins.MousePosition(fmt=fmt))
    pylab.legend()
    pylab.xlabel('Time (s)')
    pylab.ylabel('NewSNR')
    pylab.grid()
    return mpld3.fig_to_html(fig)

def trigger_timeseries_plot(file_list, ifos, start, end):

        trigs = columns_from_file_list(file_list,
                                       ['snr', 'end_time'],
                                       ifo, start, end)
        print(trigs)
        pylab.scatter(trigs['end_time'], trigs['snr'], label=ifo,
                      color=ifo_color[ifo])

        fmt = '.12g'
        mpld3.plugins.connect(fig, mpld3.plugins.MousePosition(fmt=fmt))
    pylab.ylabel('SNR')

def times_to_urls(times, window, tag):
    base = '/../followup/%s/%s/%s'
    return times_to_links(times, window, tag, base=base)

def times_to_links(times, window, tag, base=None):
    if base is None:
        base = "<a href='/../followup/%s/%s/%s' target='_blank'>followup</a>"

    urls = []
    for time in times:
        start = time - window
        end = time + window
        urls.append(base % (tag, start, end))
    return urls

def get_gracedb_search_link(time):
    # Set up a search string for a 3s window around the coincidence
    gdb_search_query = '%.0f+..+%.0f' % (numpy.floor(time) - 1,
                                         numpy.ceil(time) + 1)
    gdb_search_url = ('https://gracedb.ligo.org/search/?query='
                      '{}&query_type=S'.format(gdb_search_query))
    gdb_search_link = '<a href="' + gdb_search_url + '">Search</a>'
    return gdb_search_link

# Copyright (C) 2015 Alexander Harvey Nitz
#
#
#
""" This module contains result page layout and numbering helper functions
"""
import os.path
from six.moves import zip_longest

def two_column_layout(path, cols, unique='', **kwargs):
    """ Make a well layout in a two column format

    path: str
        Location to make the well html file
    unique: str
        String to add to end of well name. Used if you want more than one well.
    cols: list of tuples
        The format of the items on the well result section. Each tuple
        contains the two files that are shown in the left and right hand
        side of a row in the well.html page.
    """
    path = os.path.join(os.getcwd(), path, 'well{}.html'.format(unique))
    from pycbc.results.render import render_workflow_html_template
    render_workflow_html_template(path, 'two_column.html', cols, **kwargs)

def single_layout(path, files, **kwargs):
    """ Make a well layout in  single column format

    path: str
    files: list of pycbc.workflow.core.Files
        This list of images to show in order within the well layout html file.
    """
    two_column_layout(path, [(f,) for f in files], **kwargs)

def grouper(iterable, n, fillvalue=None):
    """ Group items into chunks of n length
    """
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

def group_layout(path, files, **kwargs):
    """ Make a well layout in chunks of two from a list of files

    path: str
        Every two are placed on the same row.
    """
    if len(files) > 0:
        two_column_layout(path, list(grouper(files, 2)), **kwargs)

class SectionNumber(object):
    """ Class to help with numbering sections in an output page.
    """
    def __init__(self, base, secs):
        """ Create section numbering instance

        base: path
            The path of the of output html results directory
        secs: list of strings
            List of the subsections of the output html page
        """
        self.base = base
        self.secs = secs
        self.name = {}
        self.count = {}
        self.num = {}

        for num, sec in enumerate(secs):
            self.name[sec] = '%s._%s' % (num + 1, sec)
            self.num[sec] = num
            self.count[sec] = 1

    def __getitem__ (self, path):
        """ Return the path to use for the given subsection with numbering
        included. The numbering increments for each new subsection request. If
        a section is re-requested, it gets the original numbering.
        """
        if path in self.name:
            name = self.name[path]
        else:
            sec, subsec = path.split('/')
            subnum = self.count[sec]
            num = self.num[sec]
            name = '%s/%s.%02d_%s' % (self.name[sec], num + 1, subnum, subsec)
            self.count[sec] += 1
            self.name[path] = name
        path = os.path.join(os.getcwd(), self.base, name)
        return path

#!/usr/bin/env python

# Copyright (C) 2015 Andrew R. Williamson
#
#
#

# Preamble

import re
import os
from glue import markup

def initialize_page(title, style, script, header=None):
    """
    A function that returns a markup.py page object with the required html
    header.
    """

    page = markup.page(mode="strict_html")
    page._escape = False
    page.init(title=title, css=style, script=script, header=header)

    return page

def write_banner(title, text='&nbsp;'):
    """
    Write html <title> tag into markup.page object
    """

    page.div(id="header")
    page.h1()
    page.add(title)
    page.h1.close()
    page.h3()
    page.add(text)
    page.h3.close()

    page.hr(class_="short")
    page.hr(class_="long")

    page.div.close()

    page.div(id="container")

def write_table(page, headers, data, cl=''):

    """
    Write table in html
    """

    page.table(class_=cl)

    # list
    if cl=='list':
        for i in range(len(headers)):

            page.tr()
            page.th()
            page.add('%s' % headers[i])
            page.th.close()
            page.td()
            page.add('%s' % data[i])
            page.td.close()
            page.tr.close()

    else:
        page.tr()
        for n in headers:
            page.th()
            page.add('%s' % n)
        page.tr.close()

        if data and not re.search('list',str(type(data[0]))):
            data = [data]

        for row in data:
            page.tr()
            for item in row:
                page.td()
                page.add('%s' % item)
                page.td.close()

    page.table.close()

def write_summary(page, args, ifos, skyError=None, ipn=False, ipnError=False):

    """
        Write summary of information to markup.page object page
    """
    from pylal import antenna
    from lal.gpstime import gps_to_utc, LIGOTimeGPS

    gps = args.start_time
    grbdate = gps_to_utc(LIGOTimeGPS(gps))\
                                .strftime("%B %d %Y, %H:%M:%S %ZUTC")
    page.h3()
    page.add('Basic information')

    if ipn:
        ra = []
        dec = []
        td1 = []
        td2 = []
        td3 = []
        timedelay = {}
        search_file = '../../../S5IPN_GRB%s_search_180deg.txt' % args.grb_name
        for line in open(search_file):
            ra.append(line.split()[0])
            dec.append(line.split()[1])
        th1 = [ 'GPS', 'Date', 'Error Box (sq.deg.)', 'IFOs' ]
        td1 = [ gps, grbdate, ipnError, ifos ]
        th2 = [ 'RA', 'DEC' ]
        th3 = ['Timedelays (ms)', '', '' ]
        for ra_i,dec_i in zip(ra,dec):
            td_i = [ ra_i, dec_i ]
            td2.append(td_i)
        ifo_list = [ ifos[i*2:(i*2)+2] for i in range(int(len(ifos)/2)) ]
        for j in td2:
            for p in range(0, len(ifo_list)):
                for q in range(0, len(ifo_list)):
                    pairs = [ifo_list[p], ifo_list[q]]
                    ifo_pairs = "".join(pairs)
                    timedelay[ifo_pairs] = antenna.timeDelay(int(gps),
                            float(j[0]), float(j[1]), 'degree', ifo_list[p],
                            ifo_list[q])
                    timedelay[ifo_pairs]="%.4f" % timedelay[ifo_pairs]
            if ifos == 'H1H2L1':
                td3.append(['H1L1: %f' % float(timedelay['H1L1'])])
            if ifos == 'H1H2L1V1':
                td3.append(['H1L1: %f' % float(timedelay['H1L1']),
                            'H1V1: %f' % float(timedelay['H1V1']),
                            'L1V1: %f' % float(timedelay['L1V1'])])
            if ifos == 'L1V1':
                td3.append(['L1V1: %f' % float(timedelay['L1V1'])])
        page = write_table(page, th1, td1)
        page = write_table(page, th2, td2)
        page = write_table(page, th3, td3)

    else:
        ra = args.ra
        dec = args.dec
        if skyError:
            th = [ 'GPS', 'Date', 'RA', 'DEC', 'Sky Error', 'IFOs' ]
            td = [ gps, grbdate, ra, dec, skyError, ifos ]
        else:
            th = [ 'GPS', 'Date', 'RA', 'DEC', 'IFOs' ]
            td = [ gps, grbdate, ra, dec, ifos ]

        page = write_table(page, th, td)

def write_antenna(page, args, seg_plot=None, grid=False, ipn=False):

    """
    Write antenna factors to merkup.page object page and generate John's
    detector response plot.
    """

    page.h3()
    page.add('Antenna factors and sky locations')

    th = []
    td = []
    th2 = []
    td2 = []

    ifos = [args.ifo_tag[i:i+2] for i in range(0, len(args.ifo_tag), 2)]

    if ipn:
        antenna_ifo = {}
        ra = []
        dec = []
        # FIXME: Remove hardcoding here and show this in all cases
        search_file = open('../../../S5IPN_GRB%s_search_180deg.txt'
                           % args.grb_name)
        for line in search_file:
            antenna_ifo[ifo] = []
            for k, l in zip(ra, dec):
                _, _, _, f_q = antenna.response(args.start_time, float(k),
                                                float(l), 0.0, 0.0, 'degree',
                                                ifo)
                antenna_ifo[ifo].append(round(f_q,3))
        dectKeys = antenna_ifo.keys()

        for elements in range(len(tuple(antenna_ifo.values())[0])):
            newDict={}
            for detectors in range(len(antenna_ifo.keys())):
                newDict[dectKeys[detectors]] = antenna_ifo[\
                                               dectKeys[detectors]][elements]
            for key in newDict.keys():
                th.append(key)
            td.append(newDict.values())
        page = write_table(page, list(set(th)), td)
        _, _, _, f_q = antenna.response(args.start_time, args.ra, args.dec,
                                        0.0, 0.0, 'degree',ifo)
        th.append(ifo)
        td.append(round(f_q, 3))

    #FIXME: Work out a way to make these external calls safely
    #cmmnd = 'projectedDetectorTensor --gps-sec %d --ra-deg %f --dec-deg %f' \
    #         % (args.start_time,args.ra, args.dec)
    #for ifo in ifos:
    #    if ifo == 'H1':
    #        cmmnd += ' --display-lho'
    #    elif ifo == 'L1':
    #        cmmnd += ' --display-llo'
    #    elif ifo == 'V1':
    #        cmmnd += ' --display-virgo'
    #status = make_external_call(cmmnd)

    page = write_table(page, th, td)

#    plot = markup.page()
#    p = "projtens.png"
#    plot.a(href=p, title="Detector response and polarization")
#    plot.img(src=p)
#    plot.a.close()
#    th2 = ['Response Diagram']
#    td2 = [plot() ]

# FIXME: Add these in!!
#    p = "ALL_TIMES/plots_clustered/GRB%s_search.png"\
#        % args.grb_name
#    plot.a(href=p, title="Error Box Search")
#    th2.append('Error Box Search')
#    td2.append(plot())

#    p = "ALL_TIMES/plots_clustered/GRB%s_simulations.png"\
#    plot.a(href=p, title="Error Box Simulations")
#    th2.append('Error Box Simulations')

    if seg_plot is not None:
        plot = markup.page()
        p = os.path.basename(seg_plot)
        plot.a(href=p, title="Science Segments")
        plot.img(src=p)
        plot.a.close()
        th2.append('Science Segments')
        td2.append(plot())

    plot = markup.page()
    p = "ALL_TIMES/plots_clustered/GRB%s_sky_grid.png"\
            % args.grb_name
    plot.a(href=p, title="Sky Grid")
    plot.img(src=p)
    plot.a.close()
    th2.append('Sky Grid')
    td2.append(plot())

#    p = "GRB%s_inspiral_horizon_distance.png"\
#            % args.grb_name
#    plot.a(href=p, title="Inspiral Horizon Distance")
#    th2.append('Inspiral Horizon Distance')

    page = write_table(page, th2, td2)

def write_offsource(page, args, grbtag, onsource=False):

    """
        Write offsource SNR versus time plots to markup.page object page
    """

    th = ['Re-weighted SNR', 'Coherent SNR']

    if args.time_slides:
        if onsource:
            out_dir = 'ZEROLAG_ALL'
        else:
            out_dir = 'ZEROLAG_OFF'
    else:
            out_dir = 'ALL_TIMES'
        else:
            out_dir = 'OFFSOURCE'

    p = "%s/plots_clustered/GRB%s_bestnr_vs_time_noinj.png" % (out_dir, grbtag)
    plot.a(href=p, title="Detection statistic versus time")
    td = [ plot() ]

    p = "%s/plots_clustered/GRB%s_triggers_vs_time_noinj.png" % (out_dir, grbtag)
    plot.a(href=p, title="Coherent SNR versus time")
    td.append(plot())

        th.append('%s SNR' % ifo)
        p = "%s/plots_clustered/GRB%s_%s_triggers_vs_time_noinj.png"\
            % (out_dir, grbtag, ifo)
        plot.a(href=p, title="%s SNR versus time" % ifo)
        td.append(plot())

def write_chisq(page, injList, grbtag):

    """
        Write injection chisq plots to markup.page object page
    """

    if injList:
        th = ['']+injList + ['OFFSOURCE']
    else:
        th= ['','OFFSOURCE']
        injList = ['OFFSOURCE']
    td = []

    plots = ['bank_veto','auto_veto','chi_square', 'mchirp']

    for test in plots:
        pTag = test.replace('_',' ').title()
        d = [pTag]
        for inj in injList + ['OFFSOURCE']:
            plot = markup.page()
            p = "%s/plots_clustered/GRB%s_%s_vs_snr_zoom.png" % (inj, grbtag,
                                                                 test)
            plot.a(href=p, title="%s %s versus SNR" % (inj, pTag))
            plot.img(src=p)
            plot.a.close()
            d.append(plot())

        td.append(d)

def write_inj_snrs(page, ifos, injList, grbtag):

    """
    """

    else:
    td = []

    ifos = [ifos[i:i+2] for i in range(0, len(ifos), 2)]
    plots = ['null_stat2']+['%s_snr' % ifo for ifo in ifos]

    for row in plots:
        pTag = row.replace('_',' ').title()
        d = [pTag]
                                                                 row)

def write_found_missed(page, args, injList):

    """
        Write injection found/missed plots to markup.page object page
    """

    th = ['']+injList
    td = []

    #FIXME: Work out a way to make externals calls safely
    #d = ['Number of injections']
    #for inj in injList:
    #    cmmnd = 'lwtprint ../*' + inj + '*MISSED*xml -t sim_inspiral | wc -l'
    #    output,status = make_external_call(cmmnd, shell=True)
    #    numInjs = int(output)
    #    cmmnd = 'lwtprint ../*' + inj + '*FOUND*xml -t sim_inspiral | wc -l'
    #    numInjs += int(output)
    #    d.append(str(numInjs))
    #td.append(d)

    plots = []
    text  = {}
    plots.extend(['dist', 'dist_time'])
    text['dist'] = 'Dist vs Mchirp'
    text['dist_time'] = 'Dist vs Time'
        plots.extend(['effdist_%s' % ifo[0].lower(),\
                      'effdist_time_%s' % ifo[0].lower()])
        text['effdist_%s' % ifo[0].lower()] = 'Eff. dist. %s vs Mchirp' % ifo
        text['effdist_time_%s' % ifo[0].lower()] = 'Eff. dist %s vs Time' % ifo

        pTag = text[row]
        d = [pTag]
        for inj in injList:
            p = "%s/efficiency_OFFTRIAL_1/found_missed_injections_%s.png"\
                % (inj, row)
            plot.a(href=p, title=pTag)

    td.append(['Close injections without FAP = 0']+\
              ['<a href="%s/efficiency_OFFTRIAL_1/quiet_found_triggers.html"> '
               'here</a>' % inj for inj in injList])

def write_recovery(page, injList):

    """
        Write injection recovery plots to markup.page object page
    """

    td = []

    plots = ['sky_error_time','sky_error_mchirp','sky_error_distance']
    text = { 'sky_error_time':'Sky error vs time',\
                      'sky_error_mchirp':'Sky error vs mchirp',\
                      'sky_error_distance':'Sky error vs distance' }

        d = [pTag]
            p = "%s/efficiency_OFFTRIAL_1/found_%s.png" % (inj, row)

def write_loudest_events(page, bins, onsource=False):

    """
    """

    th = ['']+['Mchirp %s - %s' % tuple(bin) for bin in bins]
    td = []

    plots = ['BestNR','SNR']

    if onsource:
        trial = 'ONSOURCE'
    else:
        trial = 'OFFTRIAL_1'

    for pTag in plots:
        row = pTag.lower()
        d = [pTag]
        for bin in bins:
            b = '%s_%s' % tuple(bin)
            p = "%s/efficiency/%s_vs_fap_%s.png" % (trial, row, b)
            plot.a(href=p, title="FAP versus %s" % pTag)

    row = 'snruncut'
    d = ['SNR after cuts <br> have been applied']
    for bin in bins:
        b = '%s_%s' % tuple(bin)
        p = "%s/efficiency/%s_vs_fap_%s.png" % (trial, row, b)
        plot.a(href=p, title="FAP versus %s" % pTag)
        d.append(plot())
    td.append(d)

    page.add('For more details on the loudest offsource events see')
    page.a(href='%s/efficiency/loudest_offsource_trigs.html' % (trial))
    page.add('here.')
    page.a.close()

def write_exclusion_distances(page , trial, injList, massbins, reduced=False,
                              onsource=False):
    file = open('%s/efficiency/loud_numbers.txt' % (trial), 'r')
    FAPS = []
    for line in file:
        line = line.replace('\n','')
        if line == "-2":
            FAPS.append('No event')
        else:
            FAPS.append(line)

    file.close()

    th = ['']+['Mchirp %s - %s' % tuple(bin) for bin in massbins]
    td = ['FAP']+FAPS
    page.add('For more details on the loudest onsource events see')
    page.a(href='%s/efficiency/loudest_events.html' % (trial))

    if reduced or not injList:
        return page

    page.h3()
    page.add('Detection efficiency plots - injections louder than loudest '
             'background trigger')

    th = injList
    td = []
    d = []
    for inj in injList:
        p = "%s/efficiency_%s/BestNR_max_efficiency.png" % (inj, trial)
        plot.a(href=p, title="Detection efficiency")

    page.h3()
    page.add('Exclusion distance plots - injections louder than loudest '
             'foreground trigger')

    td = []
    d = []
        p = "%s/efficiency_%s/BestNR_on_efficiency.png" % (inj, trial)
        plot.a(href=p, title="Exclusion efficiency")

    for percentile in [90, 50]:
        page.h3()
        page.add('%d%% confidence exclusion distances (Mpc)' % percentile)
        th = injList
        td = []
        d = []
            file = open('%s/efficiency_%s/exclusion_distance_%d.txt'
                        % (inj, trial, percentile), 'r')
            for line in file:
                line = line.replace('\n','')
                excl_dist = float(line)
            d.append(excl_dist)
            file.close()

def make_grb_segments_plot(wkflow, science_segs, trigger_time, trigger_name,
                           out_dir, coherent_seg=None, fail_criterion=None):

    import matplotlib.pyplot as plt
    from matplotlib.patches import Rectangle
    from matplotlib.lines import Line2D
    from pycbc.results.color import ifo_color

    ifos = wkflow.ifos
    if len(science_segs.keys()) == 0:
        extent = segments.segment(int(wkflow.cp.get("workflow", "start-time")),
                                  int(wkflow.cp.get("workflow", "end-time")))
    else:
        pltpad = [science_segs.extent_all()[1] - trigger_time,
                  trigger_time - science_segs.extent_all()[0]]
        extent = segments.segmentlist([science_segs.extent_all(),
            segments.segment(trigger_time - pltpad[0],
                             trigger_time + pltpad[1])]).extent()

    ifo_colors = {}
        ifo_colors[ifo] = ifo_color(ifo)
        if ifo not in science_segs.keys():
            science_segs[ifo] = segments.segmentlist([])

    # Make plot
    fig, subs = plt.subplots(len(ifos), sharey=True)
    if len(ifos) == 1:
        subs = [subs]
    plt.xticks(rotation=20, ha='right')
    for sub, ifo in zip(subs, ifos):
        for seg in science_segs[ifo]:
            sub.add_patch(Rectangle((seg[0], 0.1), abs(seg), 0.8,
                                    facecolor=ifo_colors[ifo], edgecolor='none'))
        if coherent_seg:
            if len(science_segs[ifo]) > 0 and \
                    coherent_seg in science_segs[ifo]:
                sub.plot([trigger_time, trigger_time], [0, 1], '-',
                         c='orange')
                sub.add_patch(Rectangle((coherent_seg[0], 0),
                                        abs(coherent_seg), 1, alpha=0.5,
                                        facecolor='orange', edgecolor='none'))
            else:
                sub.plot([trigger_time, trigger_time], [0, 1], ':',
                sub.plot([coherent_seg[0], coherent_seg[0]], [0, 1], '--',
                         c='orange', alpha=0.5)
                sub.plot([coherent_seg[1], coherent_seg[1]], [0, 1], '--',
        else:
            sub.plot([trigger_time, trigger_time], [0, 1], ':k')
        if fail_criterion:
            if len(science_segs[ifo]) > 0:
                style_str = '--'
            else:
                style_str = '-'
            sub.plot([fail_criterion[0], fail_criterion[0]], [0, 1], style_str,
                     c='black', alpha=0.5)
            sub.plot([fail_criterion[1], fail_criterion[1]], [0, 1], style_str,

        sub.set_frame_on(False)
        sub.set_yticks([])
        sub.set_ylabel(ifo, rotation=45)
        sub.set_ylim([0, 1])
        sub.set_xlim([float(extent[0]), float(extent[1])])
        sub.get_xaxis().get_major_formatter().set_useOffset(False)
        sub.get_xaxis().get_major_formatter().set_scientific(False)
        sub.get_xaxis().tick_bottom()
        if sub is subs[-1]:
            sub.tick_params(labelsize=10, pad=1)
        else:
            sub.get_xaxis().set_ticks([])
            sub.get_xaxis().set_ticklabels([])

    xmin, xmax = fig.axes[-1].get_xaxis().get_view_interval()
    ymin, _ = fig.axes[-1].get_yaxis().get_view_interval()
    fig.axes[-1].add_artist(Line2D((xmin, xmax), (ymin, ymin), color='black',
                                   linewidth=2))
    fig.axes[-1].set_xlabel('GPS Time')

    fig.axes[0].set_title('Science Segments for GRB%s' % trigger_name)
    plt.tight_layout()
    fig.subplots_adjust(hspace=0)

    plot_name = 'GRB%s_segments.png' % trigger_name
    plot_url = 'file://localhost%s/%s' % (out_dir, plot_name)
    fig.savefig('%s/%s' % (out_dir, plot_name))

    return [ifos, plot_name, extent, plot_url]

"""
This Module contains generic utility functions for creating plots within
PyCBC.
"""
import os.path, pycbc.version
from six.moves.html_parser import HTMLParser
from xml.sax.saxutils import escape, unescape

escape_table = {
                '"': "&quot;",
                "'": "&apos;",
                "@": "&#64;",
                }
unescape_table = {
                  "&#64;" : "@",
                 }
for k, v in escape_table.items():
    unescape_table[v] = k

def html_escape(text):
    """ Sanitize text for html parsing """
    return escape(text, escape_table)

class MetaParser(HTMLParser):
        self.metadata = {}
        HTMLParser.__init__(self)

    def handle_data(self, data):
        pass

    def handle_starttag(self, tag, attrs):
        attr= {}
        for key, value in attrs:
            attr[key] = value
        if tag == 'div' and 'class' in attr and attr['class'] == 'pycbc-meta':
            self.metadata[attr['key']] = unescape(attr['value'], unescape_table)

def save_html_with_metadata(fig, filename, fig_kwds, kwds):
    """ Save a html output to file with metadata """
    if isinstance(fig, str):
        text = fig
    else:
        from mpld3 import fig_to_html
        text = fig_to_html(fig, **fig_kwds)

    f = open(filename, 'w')
    for key, value in kwds.items():
        value = escape(value, escape_table)
        line = "<div class=pycbc-meta key=\"%s\" value=\"%s\"></div>" % (str(key), value)
        f.write(line)

    f.write(text)

def load_html_metadata(filename):
    """ Get metadata from html file """
    parser = MetaParser()
    data = open(filename, 'r').read()

    if 'pycbc-meta' in data:
        print("LOADING HTML FILE %s" % filename)
    parser.feed(data)
    cp = ConfigParser.ConfigParser(parser.metadata)
    cp.add_section(os.path.basename(filename))
    return cp

def save_png_with_metadata(fig, filename, fig_kwds, kwds):
    """ Save a matplotlib figure to a png with metadata
    """
    from PIL import Image, PngImagePlugin
    fig.savefig(filename, **fig_kwds)

    im = Image.open(filename)
    meta = PngImagePlugin.PngInfo()

    for key in kwds:
        meta.add_text(str(key), str(kwds[key]))

    im.save(filename, "png", pnginfo=meta)

def save_pdf_with_metadata(fig, filename, fig_kwds, kwds):
    """Save a matplotlib figure to a PDF file with metadata.
    """
    # https://stackoverflow.com/a/17462125
    from matplotlib.backends.backend_pdf import PdfPages

    with PdfPages(filename) as pdfp:
        fig.savefig(pdfp, format='pdf', **fig_kwds)
        metadata = pdfp.infodict()
        for key in kwds:
            if str(key).lower() == 'title':
                # map the title to the official PDF keyword (capitalized)
                metadata['Title'] = str(kwds[key])
            else:
                metadata[str(key)] = str(kwds[key])

def load_png_metadata(filename):
    data = Image.open(filename).info
    cp = ConfigParser.ConfigParser(data)
    return cp

_metadata_saver = {'.png': save_png_with_metadata,
                   '.html': save_html_with_metadata,
                   '.pdf': save_pdf_with_metadata,
                  }
_metadata_loader = {'.png': load_png_metadata,
                    '.html': load_html_metadata,
                   }

def save_fig_with_metadata(fig, filename, fig_kwds=None, **kwds):
    """ Save plot to file with metadata included. Kewords translate to metadata
    that is stored directly in the plot file. Limited format types available.

    fig: matplotlib figure
        The matplotlib figure to save to the file
        Name of file to store the plot.
    """
    if fig_kwds is None:
        fig_kwds = {}
    try:
        extension = os.path.splitext(filename)[1]
        kwds['version'] = pycbc.version.git_verbose_msg
        _metadata_saver[extension](fig, filename, fig_kwds, kwds)
        raise TypeError('Cannot save file %s with metadata, extension %s not '
                        'supported at this time' % (filename, extension))

def load_metadata_from_file(filename):
    """ Load the plot related metadata saved in a file

        Name of file load metadata from.

    Returns
    -------
    cp: ConfigParser
        A configparser object containing the metadata
    """
    try:
        return _metadata_loader[extension](filename)
        raise TypeError('Cannot read metadata from file %s, extension %s not '

""" This module provides functionality to extend mpld3
"""
import mpld3, mpld3.plugins, mpld3.utils

class ClickLink(mpld3.plugins.PluginBase):
    """Plugin for following a link on click"""

    JAVASCRIPT = """
    mpld3.register_plugin("clicklink", ClickLink);
    ClickLink.prototype = Object.create(mpld3.Plugin.prototype);
    ClickLink.prototype.constructor = ClickLink;
    ClickLink.prototype.requiredProps = ["id"];
    ClickLink.prototype.defaultProps = {
        links: null
    }
    function ClickLink(fig, props){
        mpld3.Plugin.call(this, fig, props);
    };

    ClickLink.prototype.draw = function(){
        var obj = mpld3.get_element(this.props.id);
        var links = this.props.links;

        obj.elements().on("mousedown",
                          function(d, i){
                                           window.open(links[i]);
                                        }
                          );
    }
    """
    def __init__(self, points, links):
        self.dict_ = {"type": "clicklink",
                      "id": mpld3.utils.get_id(points),
                      "links": links,
                      }

class MPLSlide(mpld3.plugins.PluginBase):
         mpld3.Axes.prototype.zoomed = function(propagate) {
            propagate = typeof propagate == "undefined" ? true : propagate;
            if (propagate) {
              var dt0 = this.zoom.translate()[0] - this.zoom.last_t[0];
              var dt1 = this.zoom.translate()[1] - this.zoom.last_t[1];
              var ds = this.zoom.scale() / this.zoom.last_s;
              this.zoom_x.translate([ this.zoom_x.translate()[0] + dt0, 0 ]);
              this.zoom_x.scale(this.zoom_x.scale() * ds);

              this.zoom.last_t = this.zoom.translate();
              this.zoom.last_s = this.zoom.scale();
              this.sharex.forEach(function(ax) {
                ax.zoom_x.translate(this.zoom_x.translate()).scale(this.zoom_x.scale());
              }.bind(this));

                ax.zoomed(false);
              });
            }
            for (var i = 0; i < this.elements.length; i++) {
              this.elements[i].zoomed();
            }
          };

            mpld3.ZoomPlugin = mpld3_ZoomPlugin;
            mpld3.register_plugin("zoom", mpld3_ZoomPlugin);
            mpld3_ZoomPlugin.prototype = Object.create(mpld3.Plugin.prototype);
            mpld3_ZoomPlugin.prototype.constructor = mpld3_ZoomPlugin;
            mpld3_ZoomPlugin.prototype.requiredProps = [];
            mpld3_ZoomPlugin.prototype.defaultProps = {
                button: true,
                enabled: null
            };
            function mpld3_ZoomPlugin(fig, props) {
                mpld3.Plugin.call(this, fig, props);
                if (this.props.enabled === null) {
                    this.props.enabled = !this.props.button;
                }
                var enabled = this.props.enabled;
                if (this.props.button) {
                    var ZoomButton = mpld3.ButtonFactory({
                        buttonID: "zoom",
                        sticky: true,
                        actions: [ "scroll", "drag" ],
                        onActivate: this.activate.bind(this),
                        onDeactivate: this.deactivate.bind(this),
                        onDraw: function() {
                            this.setState(enabled);
                        },
                        icon: function() {
                            return mpld3.icons["move"];
                        }
                    });
                this.fig.buttons.push(ZoomButton);
                }
            }
            mpld3_ZoomPlugin.prototype.activate = function() {
                this.fig.enable_zoom();
            };
            mpld3_ZoomPlugin.prototype.deactivate = function() {
                this.fig.disable_zoom();
            };
            mpld3_ZoomPlugin.prototype.draw = function() {
                if (this.props.enabled) this.fig.enable_zoom(); else this.fig.disable_zoom();
            };
        """
    def __init__(self, button=True, enabled=None):
        if enabled is None:
            enabled = not button
        self.dict_ = {"type": "zoom",
                      "button": button,
                      "enabled": enabled}

class Tooltip(mpld3.plugins.PointHTMLTooltip):
    JAVASCRIPT = ""
    def __init__(self, points, labels=None,
                 hoffset=0, voffset=10, css=None):
        super(Tooltip, self).__init__(points, labels, hoffset, voffset, "")

class LineTooltip(mpld3.plugins.LineHTMLTooltip):
    def __init__(self, line, label=None, hoffset=0, voffset=10, css=None):
        super(LineTooltip, self).__init__(line, label, hoffset, voffset, "")

""" Plotting utilities and premade plot configurations
"""

def hist_overflow(val, val_max, **kwds):
    """ Make a histogram with an overflow bar above val_max """
    import pylab, numpy

    overflow = len(val[val>=val_max])
    pylab.hist(val[val<val_max], **kwds)

    if 'color' in kwds:
        color = kwds['color']
    else:
        color = None

    if overflow > 0:
        rect = pylab.bar(val_max+0.05, overflow, .5, color=color)[0]
        pylab.text(rect.get_x(),
                   1.10*rect.get_height(), '%s+' % val_max)

def add_style_opt_to_parser(parser, default=None):
    """Adds an option to set the matplotlib style to a parser.

    parser : argparse.ArgumentParser
        The parser to add the option to.
    default : str, optional
        The default style to use. Default, None, will result in the default
        matplotlib style to be used.
    """
    from matplotlib import pyplot
    parser.add_argument('--mpl-style', default=default,
                        choices=['default']+pyplot.style.available+['xkcd'],
                        help='Set the matplotlib style to use.')

def set_style_from_cli(opts):
    """Uses the mpl-style option to set the style for plots.

    Note: This will change the global rcParams.
    """
    if opts.mpl_style == 'xkcd':
        # this is treated differently for some reason
        pyplot.xkcd()
    elif opts.mpl_style is not None:
        pyplot.style.use(opts.mpl_style)

# Copyright (C) 2019 Francesco Pannarale, Gino Contestabile
#
#
#

# Preamble

"""
Module to generate PyGRB figures: scatter plots and timeseries.
"""

import sys
import os
from pycbc.results import save_fig_with_metadata
# TODO: imports to fix/remove
try:
    from glue import segments
    from glue.ligolw import utils, lsctables, ligolw, table
    pass
try:
    from pylal import MultiInspiralUtils
    from pylal.coh_PTF_pyutils import get_bestnr, get_det_response
    from pylal.coh_PTF_pyutils import readSegFiles
    from pylal.dq import dqSegmentUtils
    pass
import matplotlib
if 'matplotlib.backends' not in sys.modules:  # nopep8
from matplotlib import rc
from matplotlib import pyplot as plt

# Parse command line

def pygrb_plot_opts_parser(usage='', description=None, version=None):
    """Parses options for PyGRB plotting scripts"""
    parser = argparse.ArgumentParser(usage=usage, description=description)

    parser.add_argument("--version", action="version", version=version)

    parser.add_argument("-v", "--verbose", default=False, action="store_true",
                        help="verbose output")

    parser.add_argument("-t", "--trig-file", action="store",
                        default=None, required=True,
                        help="The location of the trigger file")

    parser.add_argument("-I", "--inj-file", action="store", default=None,
                        help="The location of the injection file")

    parser.add_argument("-a", "--segment-dir", action="store",
                        required=True, help="directory holding buffer, on " +
                        "and off source segment files.")

    parser.add_argument("-o", "--output-file", default=None, required=True,
                        help="Output file.")

    parser.add_argument("-O", "--zoomed-output-file", default=None,
                        required=False, help="Output file for a zoomed in " +
                        "version of the plot.")

    parser.add_argument("-Q", "--chisq-index", action="store", type=float,
                        default=4.0, help="chisq_index for newSNR calculation")

    parser.add_argument("-N", "--chisq-nhigh", action="store", type=float,
                        default=3.0, help="nhigh for newSNR calculation")

    parser.add_argument("-B", "--sngl-snr-threshold", action="store",
                        type=float, default=4.0, help="Single detector SNR " +
                        "threshold, the two most sensitive detectors " +
                        "should have SNR above this")

    parser.add_argument("-d", "--snr-threshold", action="store", type=float,
                        default=6.0, help="SNR threshold for recording " +
                        "triggers")

    parser.add_argument("-c", "--newsnr-threshold", action="store", type=float,
                        default=None, help="NewSNR threshold for " +
                        "calculating the chisq of triggers (based on value " +
                        "of auto and bank chisq  values. By default will " +
                        "take the same value as snr-threshold")

    parser.add_argument("-A", "--null-snr-threshold", action="store",
                        default="4.25,6",
                        help="comma separated lower,higher null SNR " +
                        "threshold for null SNR cut")

    parser.add_argument("-C", "--null-grad-thresh", action="store", type=float,
                        default=20., help="Threshold above which to " +
                        "increase the values of the null SNR cut")

    parser.add_argument("-D", "--null-grad-val", action="store", type=float,
                        default=0.2, help="Rate the null SNR cut will " +
                        "increase above the threshold")

    parser.add_argument("-l", "--veto-directory", action="store", default=None,
                        help="The location of the CATX veto files")

    parser.add_argument("-b", "--veto-category", action="store", type=int,
                        default=None, help="Apply vetoes up to this level " +
                        "inclusive")

    parser.add_argument("-i", "--ifo", default=None, help="IFO used for IFO " +
                        "specific plots")

    parser.add_argument("--use-sngl-ifo-snr", default=False,
                        action="store_true", help="Plots are vs single IFO " +
                        "SNR, rather than coherent SNR")

    parser.add_argument("--variable", default=None, help="Quantity to plot " +
                        "the vertical axis. Supported choices are: " +
                        "coherent, single, reweighted, or null (for " +
                        "timeeries plots), standard, bank, or auto (for " +
                        "chi-square veto plots), coincident, nullstat, " +
                        "or overwhitened (for null statistics plots)")

    parser.add_argument('--plot-title',
                        help="If given, use this as the plot caption")

    parser.add_argument('--plot-caption',

    return parser.parse_args()

# Format single detector chi-square data as numpy array and floor at 0.005

def format_single_chisqs(trig_ifo_cs, ifos):
    """Format single IFO chi-square data as numpy array and floor at 0.005"""
        trig_ifo_cs[ifo] = numpy.asarray(trig_ifo_cs[ifo])
        numpy.putmask(trig_ifo_cs[ifo], trig_ifo_cs[ifo] == 0, 0.005)

    return trig_ifo_cs

# Reset times so that t=0 is corresponds to the GRB trigger time

def reset_times(seg_dir, trig_data, inj_data, inj_file):
    """Reset times so that t=0 is corresponds to the GRB trigger time"""
    segs = readSegFiles(seg_dir)
    grb_time = segs['on'][1] - 1
    start = int(min(trig_data.time)) - grb_time
    end = int(max(trig_data.time)) - grb_time
    duration = end-start
    start -= duration*0.05
    end += duration*0.05
    trig_data.time = [t-grb_time for t in trig_data.time]

    if inj_file:
        inj_data.time = [t-grb_time for t in inj_data.time]

    return grb_time, start, end, trig_data, inj_data

# Extract trigger/injection data produced by PyGRB

class PygrbFilterOutput(object):
    """Extract trigger/injection data produced by PyGRB search"""
    def __init__(self, trigs_or_injs, ifos, columns, output_type, opts):
        logging.info("Extracting data from the %s just loaded...", output_type)
        # Initialize all content of self
        self.snr = numpy.array(None)
        self.reweighted_snr = None
        self.null_snr = None
        self.null_stat = None
        self.trace_snr = None
        self.chi_square = numpy.array(None)
        self.bank_veto = None
        self.auto_veto = None
        self.coinc_snr = None
        self.ifo_snr = dict((ifo, None) for ifo in ifos)
        self.ifo_bank_cs = dict((ifo, None) for ifo in ifos)
        self.ifo_auto_cs = dict((ifo, None) for ifo in ifos)
        self.ifo_stan_cs = dict((ifo, None) for ifo in ifos)
        self.rel_amp_1 = None
        self.norm_3 = None
        self.rel_amp_2 = None
        self.inclination = None
        # Exctract data and fill in content of self
        null_thresh = map(float, opts.null_snr_threshold.split(','))
        if trigs_or_injs is not None:
            # Work out if using sngl chisqs
            ifo_att = {'G1': 'g', 'H1': 'h1', 'H2': 'h2', 'L1': 'l', 'V1': 'v',
                       'T1': 't'}
            i = ifo_att[ifos[0]]

            self.sngl_chisq = 'chisq_%s' % i in columns
            self.sngl_bank_chisq = 'bank_chisq_%s' % i in columns
            self.sngl_cont_chisq = 'cont_chisq_%s' % i in columns

            # Set basic data
            self.time = numpy.asarray(trigs_or_injs.get_end())
            self.snr = numpy.asarray(trigs_or_injs.get_column('snr'))
            self.reweighted_snr = [get_bestnr(t, q=opts.chisq_index,
                                              n=opts.chisq_nhigh,
                                              null_thresh=null_thresh,
                                              snr_threshold=opts.snr_threshold,
                                              sngl_snr_threshold=opts.sngl_snr_threshold,
                                              chisq_threshold=opts.newsnr_threshold,
                                              null_grad_thresh=opts.null_grad_thresh,
                                              null_grad_val=opts.null_grad_val)
                                   for t in trigs_or_injs]
            self.reweighted_snr = numpy.array(self.reweighted_snr)
            self.null_snr = numpy.asarray(trigs_or_injs.get_null_snr())
            self.null_stat = numpy.asarray(trigs_or_injs.get_column(
                'null_statistic'))
            self.trace_snr = numpy.asarray(trigs_or_injs.get_column(
                'null_stat_degen'))

            # Get chisq data
            self.chi_square = numpy.asarray(trigs_or_injs.get_column('chisq'))
            self.bank_veto = numpy.asarray(trigs_or_injs.get_column(
                'bank_chisq'))
            self.auto_veto = numpy.asarray(trigs_or_injs.get_column(
                'cont_chisq'))
            numpy.putmask(self.chi_square, self.chi_square == 0, 0.005)
            numpy.putmask(self.bank_veto, self.bank_veto == 0, 0.005)
            numpy.putmask(self.auto_veto, self.auto_veto == 0, 0.005)

            # Get single detector data
            self.coinc_snr = (trigs_or_injs.get_column('coinc_snr'))
            self.ifo_snr = dict((ifo, trigs_or_injs.get_sngl_snr(ifo))
                                for ifo in ifos)
            if self.sngl_bank_chisq:
                self.ifo_bank_cs = trigs_or_injs.get_sngl_bank_chisqs(ifos)
                self.ifo_bank_cs = format_single_chisqs(self.ifo_bank_cs, ifos)
            if self.sngl_cont_chisq:
                self.ifo_auto_cs = trigs_or_injs.get_sngl_cont_chisqs(ifos)
                self.ifo_auto_cs = format_single_chisqs(self.ifo_auto_cs, ifos)
            if self.sngl_chisq:
                self.ifo_stan_cs = trigs_or_injs.get_sngl_chisqs(ifos)
                self.ifo_stan_cs = format_single_chisqs(self.ifo_stan_cs, ifos)

            # Initiate amplitude generator
            num_amp = 4
            amplitudes = range(1, num_amp+1)

            # Get amplitude terms
            amp = dict((amplitude,
                        numpy.asarray(trigs_or_injs.get_column(
                            'amp_term_%d' % amplitude)))
                       for amplitude in amplitudes)
            #
            # All 0, hence the 3 warnings
            # for i in amplitudes:
            #     print numpy.count_nonzero(amp[amplitudes])
            #
            self.rel_amp_1 = numpy.sqrt((amp[1]**2 + amp[2]**2) /
                                        (amp[3]**2 + amp[4]**2))
            gamma_r = amp[1] - amp[4]
            gamma_i = amp[2] + amp[3]
            delta_r = amp[1] + amp[4]
            delta_i = amp[3] - amp[2]
            norm_1 = delta_r*delta_r + delta_i*delta_i
            norm_2 = gamma_r*gamma_r + gamma_i*gamma_i
            self.norm_3 = ((norm_1**0.25) + (norm_2**0.25))**2
            amp_plus = (norm_1)**0.5 + (norm_2)**0.5
            amp_cross = abs((norm_1)**0.5 - (norm_2)**0.5)
            self.rel_amp_2 = amp_plus/amp_cross
            self.inclination = amp_cross/self.norm_3

            num_trigs_or_injs = len(trigs_or_injs)
            if num_trigs_or_injs < 1:
                logging.warning("No %s found.", output_type)
            elif num_trigs_or_injs >= 1:
                logging.info("%d %s found.", num_trigs_or_injs, output_type)
            # Deal with the sigma-squares (historically called sigmas here)
            if output_type == "triggers":
                sigma = trigs_or_injs.get_sigmasqs()
                self.sigma_tot = numpy.zeros(num_trigs_or_injs)
                # Get antenna response based parameters
                self.longitude = numpy.degrees(trigs_or_injs.get_column('ra'))
                self.latitude = numpy.degrees(trigs_or_injs.get_column('dec'))
                self.f_resp = dict((ifo, numpy.empty(num_trigs_or_injs))
                                   for ifo in ifos)
                for i in range(num_trigs_or_injs):
                    # Calculate f_resp for each IFO if we haven't done so yet
                    f_plus, f_cross = get_det_response(self.longitude[i],
                                                       self.latitude[i],
                                                       self.time[i])
                    for ifo in ifos:
                        self.f_resp[ifo][i] = sum(numpy.array([f_plus[ifo],
                                                               f_cross[ifo]]
                                                              )**2)
                        self.sigma_tot[i] += (sigma[ifo][i] *
                                              self.f_resp[ifo][i])

                for ifo in ifos:
                    self.f_resp[ifo] = self.f_resp[ifo].mean()

                # Normalise trig_sigma
                self.sigma_tot = numpy.array(self.sigma_tot)
                    sigma[ifo] = numpy.asarray(sigma[ifo]) / self.sigma_tot

                self.sigma_mean = {}
                self.sigma_max = {}
                self.sigma_min = {}
                    try:
                        self.sigma_mean[ifo] = sigma[ifo].mean()
                        self.sigma_max[ifo] = sigma[ifo].max()
                        self.sigma_min[ifo] = sigma[ifo].min()
                        self.sigma_mean[ifo] = 0
                        self.sigma_max[ifo] = 0
                        self.sigma_min[ifo] = 0
        logging.info("%s parameters extracted", output_type)

# Function to open trigger and injection xml files

def load_xml_file(filename):
    """Wrapper to ligolw's utils.load_filename"""

    xml_doc = utils.load_filename(filename, gz=filename.endswith("gz"),
                                  contenthandler=lsctables.use_in(
                                      ligolw.LIGOLWContentHandler))

    return xml_doc

# Function to extract ifos

def extract_ifos(trig_file):
    """Extracts IFOs from search summary table"""

    # Load search summary
    xml_doc = load_xml_file(trig_file)
    search_summ = table.get_table(xml_doc,
                                  lsctables.SearchSummaryTable.tableName)

    # Extract IFOs
    ifos = sorted(map(str, search_summ[0].get_ifos()))

    return ifos

# Function to extract vetoes

def extract_vetoes(veto_files, ifos):
    """Extracts vetoes from veto filelist"""

    # Initialize vetoe containers
    vetoes = segments.segmentlistdict()
        vetoes[ifo] = segments.segmentlist()

    # Construct veto list from veto filelist
    if veto_files:
        for file in veto_files:
            ifo = os.path.basename(file)[:2]
            if ifo in ifos:
                # This returns a coalesced list of the vetoes
                tmp_veto_segs = dqSegmentUtils.fromsegmentxml(open(file, 'r'))
                for entry in tmp_veto_segs:
                    vetoes[ifo].append(entry)
        vetoes[ifo].coalesce()

    return vetoes

# Function to load triggers

def load_triggers(trig_file, vetoes, ifos):
    """"Loads triggers from PyGRB output file"""
    logging.info("Loading triggers...")

    # Extract time-slides
    multis, slide_dict, _ = \
        MultiInspiralUtils.ReadMultiInspiralTimeSlidesFromFiles([trig_file])
    num_slides = len(slide_dict)
    lsctables.MultiInspiralTable.loadcolumns =\
        [slot for slot in multis[0].__slots__ if hasattr(multis[0], slot)]

    # Extract triggers
    trigs = lsctables.New(lsctables.MultiInspiralTable,
                          columns=lsctables.MultiInspiralTable.loadcolumns)
    logging.info("%d triggers found.", len(trigs))

    # Time-slid vetoes
    for slide_id in range(num_slides):
        slid_vetoes = copy.deepcopy(vetoes)
            slid_vetoes[ifo].shift(-slide_dict[slide_id][ifo])

        # Add time-slid triggers
        vets = slid_vetoes.union(slid_vetoes.keys())
        trigs.extend(t for t in multis.veto(vets)
                     if int(t.time_slide_id) == slide_id)

    logging.info("%d triggers found when including timeslides.", len(trigs))

    return trigs

# Function to load injections

def load_injections(inj_file, vetoes):
    """"Loads injections from PyGRB output file"""
    logging.info("Loading injections...")

    # Load injection file
    xml_doc = load_xml_file(inj_file)
    multis = table.get_table(xml_doc, lsctables.MultiInspiralTable.tableName)

    # Extract injections
    injs = lsctables.New(lsctables.MultiInspiralTable,
                         columns=lsctables.MultiInspiralTable.loadcolumns)

    # Injections in time-slid non-vetoed data
    injs.extend(t for t in multis if t.get_end() not in vetoes)

    logging.info("%d injections found.", len(injs))

    return injs

def new_snr_chisq(snr, new_snr, chisq_dof, chisq_index=4.0, chisq_nhigh=3.0):
    """Returns the chi-square value needed to weight SNR into new SNR"""

    chisqnorm = (snr/new_snr)**chisq_index
    if chisqnorm <= 1:
        return 1E-20

    return chisq_dof * (2*chisqnorm - 1)**(chisq_nhigh/chisq_index)

# Given the trigger and injection values of a quantity, determine the maximum

def axis_max_value(trig_values, inj_values, inj_file):
    """Deterime the maximum of a quantity in the trigger and injection data"""

    axis_max = trig_values.max()
    if inj_file and inj_values.size and inj_values.max() > axis_max:
        axis_max = inj_values.max()

    return axis_max

# Calculate all chi-square contours for diagnostic plots

def calculate_contours(trigs, opts, new_snrs=None):
    """Generate the plot contours for chisq variable plots"""

    if new_snrs is None:
        new_snrs = [5.5, 6, 6.5, 7, 8, 9, 10, 11]
    chisq_index = opts.chisq_index
    chisq_nhigh = opts.chisq_nhigh
    new_snr_thresh = opts.newsnr_threshold
    null_thresh = []
    for val in map(float, opts.null_snr_threshold.split(',')):
        null_thresh.append(val)
    null_thresh = null_thresh[-1]
    null_grad_snr = opts.null_grad_thresh
    null_grad_val = opts.null_grad_val
    chisq_dof = trigs[0].chisq_dof
    bank_chisq_dof = trigs[0].bank_chisq_dof
    cont_chisq_dof = trigs[0].cont_chisq_dof

    # Add the new SNR threshold contour to the list if necessary
    # and keep track of where it is
    cont_value = None
    try:
        cont_value = new_snrs.index(new_snr_thresh)
        new_snrs.append(new_snr_thresh)
        cont_value = -1

    # Initialise chisq contour values and colours
    colors = ["k-" if snr == new_snr_thresh else
              "y-" if snr == int(snr) else
              "y--" for snr in new_snrs]

    # Get SNR values for contours
    snr_low_vals = numpy.arange(4, 30, 0.1)
    snr_high_vals = numpy.arange(30, 500, 1)
    snr_vals = numpy.asarray(list(snr_low_vals) + list(snr_high_vals))

    # Initialise contours
    bank_conts = numpy.zeros([len(new_snrs), len(snr_vals)],
                             dtype=numpy.float64)
    auto_conts = numpy.zeros([len(new_snrs), len(snr_vals)],
    chi_conts = numpy.zeros([len(new_snrs), len(snr_vals)],
                            dtype=numpy.float64)
    null_cont = []

    # Loop over each and calculate chisq variable needed for SNR contour
    for j, snr in enumerate(snr_vals):
        for i, new_snr in enumerate(new_snrs):
            bank_conts[i][j] = new_snr_chisq(snr, new_snr, bank_chisq_dof,
                                             chisq_index, chisq_nhigh)
            auto_conts[i][j] = new_snr_chisq(snr, new_snr, cont_chisq_dof,
            chi_conts[i][j] = new_snr_chisq(snr, new_snr, chisq_dof,
                                            chisq_index, chisq_nhigh)

        if snr > null_grad_snr:
            null_cont.append(null_thresh + (snr-null_grad_snr)*null_grad_val)
        else:
            null_cont.append(null_thresh)
    null_cont = numpy.asarray(null_cont)

    return bank_conts, auto_conts, chi_conts, null_cont, snr_vals, \
        cont_value, colors

# Plot contours in a scatter plot where SNR is on the horizontal axis

def contour_plotter(axis, snr_vals, contours, colors, vert_spike=False):
    """Plot contours in a scatter plot where SNR is on the horizontal axis"""
    for i, _ in enumerate(contours):
        plot_vals_x = []
        plot_vals_y = []
        if vert_spike:
            for j, _ in enumerate(snr_vals):
                # Workaround to ensure vertical spike is shown on veto plots
                if contours[i][j] > 1E-15 and not plot_vals_x:
                    plot_vals_x.append(snr_vals[j])
                    plot_vals_y.append(0.1)
                if contours[i][j] > 1E-15 and plot_vals_x:
                    plot_vals_y.append(contours[i][j])
        else:
            plot_vals_x = snr_vals
            plot_vals_y = contours[i]
        axis.plot(plot_vals_x, plot_vals_y, colors[i])

# Contains plotting setups shared by PyGRB plots

def pygrb_shared_plot_setups():
    """Master function to plot PyGRB results"""

    # Get rcParams
    rc('font', size=14)
    # Set color for out-of-range values
    plt.cm.spring.set_over('g')

# Master plotting function: fits all plotting needs in for PyGRB results

def pygrb_plotter(trig_x, trig_y, inj_x, inj_y, inj_file, xlabel, ylabel,
                  fig_path, snr_vals=None, conts=None,
                  shade_cont_value=None, colors=None, vert_spike=False,
                  xlims=None, ylims=None, use_logs=True,
                  cmd=None, plot_title=None, plot_caption=None):

    fig_name = os.path.split(os.path.abspath(fig_path))[1]
    logging.info(" * %s (%s vs %s)...", fig_name, xlabel, ylabel)
    # Set up plot
    fig = plt.figure()
    cax = fig.gca()
    # Plot trigger-related quantities
    if use_logs:
        cax.loglog(trig_x, trig_y, 'bx')
    else:
        cax.plot(trig_x, trig_y, 'bx')
    cax.grid()
    # Plot injection-related quantities
        if use_logs:
            cax.loglog(inj_x, inj_y, 'r+')
        else:
            cax.plot(inj_x, inj_y, 'r+')
    # Plot contours
    if conts is not None:
        contour_plotter(cax, snr_vals, conts, colors, vert_spike=vert_spike)
    # Add shading above a specific contour (typically used for vetoed area)
    if shade_cont_value is not None:
        limy = cax.get_ylim()[1]
        polyx = copy.deepcopy(snr_vals)
        polyy = copy.deepcopy(conts[shade_cont_value])
        polyx = numpy.append(polyx, [max(snr_vals), min(snr_vals)])
        polyy = numpy.append(polyy, [limy, limy])
        cax.fill(polyx, polyy, color='#dddddd')
    # Axes: labels and limits
    cax.set_xlabel(xlabel)
    cax.set_ylabel(ylabel)
    if xlims:
        cax.set_xlim(xlims)
    if ylims:
        cax.set_ylim(ylims)
    # Wrap up
    save_fig_with_metadata(fig, fig_path, cmd=cmd, title=plot_title,
                           caption=plot_caption)
    # fig_kwds=fig_kwds,
    plt.close()

# Copyright (C) 2015 Christopher M. Biwer
#
#
#

import os.path, types
import codecs

from six.moves.configparser import ConfigParser
from jinja2 import Environment, FileSystemLoader
from xml.sax.saxutils import unescape

import pycbc.results
from pycbc.results import unescape_table
from pycbc.results.metadata import save_html_with_metadata
from pycbc.workflow.core import SegFile, makedir

def render_workflow_html_template(filename, subtemplate, filelists, **kwargs):
    """ Writes a template given inputs from the workflow generator. Takes
    a list of tuples. Each tuple is a pycbc File object. Also the name of the
    subtemplate to render and the filename of the output.
    """

    dirnam = os.path.dirname(filename)
    makedir(dirnam)

    try:
        filenames = [f.name for filelist in filelists for f in filelist if f is not None]
    except TypeError:
        filenames = []

    # render subtemplate
    subtemplate_dir = pycbc.results.__path__[0] + '/templates/wells'
    env = Environment(loader=FileSystemLoader(subtemplate_dir))
    env.globals.update(get_embedded_config=get_embedded_config)
    env.globals.update(path_exists=os.path.exists)
    env.globals.update(len=len)
    subtemplate = env.get_template(subtemplate)
    context = {'filelists' : filelists,
               'dir' : dirnam}
    context.update(kwargs)
    output = subtemplate.render(context)

    # save as html page
    kwds = {'render-function' : 'render_tmplt',
            'filenames' : ','.join(filenames)}
    kwds.update(kwargs)

        kwds[key] = str(kwds[key])

    save_html_with_metadata(str(output), filename, None, kwds)

def get_embedded_config(filename):
    """ Attempt to load config data attached to file
    """
    def check_option(self, section, name):
        return (self.has_section(section) and
               (self.has_option(section, name) or (name in self.defaults())))

    try:
        cp = pycbc.results.load_metadata_from_file(filename)
        cp = ConfigParser()

    cp.check_option = types.MethodType(check_option, cp)

    return cp

def setup_template_render(path, config_path):
    """ This function is the gateway for rendering a template for a file.
    """

    # initialization
    cp = get_embedded_config(path)
    output = ''
    filename = os.path.basename(path)

    # use meta-data if not empty for rendering
    if cp.has_option(filename, 'render-function'):
        render_function_name = cp.get(filename, 'render-function')
        render_function = eval(render_function_name)
        output = render_function(path, cp)

    # read configuration file for rendering
    elif os.path.exists(config_path):
        cp.read(config_path)

        # render template
        if cp.has_option(filename, 'render-function'):
            render_function_name = cp.get(filename, 'render-function')
            render_function = eval(render_function_name)
            output = render_function(path, cp)
        else:
            output = render_default(path, cp)

    # if no configuration file is present
    # then render the default template
    else:
        output = render_default(path, cp)

    return output

def render_default(path, cp):
    """ This is the default function that will render a template to a string of HTML. The
    string will be for a drop-down tab that contains a link to the file.

    If the file extension requires information to be read, then that is passed to the
    content variable (eg. a segmentlistdict).
    """

    # define filename and slug from path
    slug = filename.replace('.', '_')

    # initializations
    content = None

    if path.endswith('.xml') or path.endswith('.xml.gz'):
        # segment or veto file return a segmentslistdict instance
        try:
            wf_file = SegFile.from_segment_xml(path)
            # FIXME: This is a dictionary, but the code wants a segmentlist
            #        for now I just coalesce.
            wf_file.return_union_seglist()
            print('No segment table found in %s : %s' % (path, e))

    # render template
    template_dir = pycbc.results.__path__[0] + '/templates/files'
    env = Environment(loader=FileSystemLoader(template_dir))
    env.globals.update(abs=abs)
    env.globals.update(open=open)
    template = env.get_template('file_default.html')
    context = {'path'     : path,
               'filename' : filename,
               'slug'     : slug,
               'cp'       : cp,
               'content'  : content}
    output = template.render(context)

def render_glitchgram(path, cp):
    """ Render a glitchgram file template.
    """

    template = env.get_template(cp.get(filename, 'template'))
    context = {'filename' : filename,
               'cp'       : cp}

def render_text(path, cp):
    """ Render a file as text.
    """

    # read file as a string
    with codecs.open(path, 'r', encoding='utf-8', errors='replace') as fp:
        content = fp.read()

    # replace all the escaped characters
    content = unescape(content, unescape_table)

    template = env.get_template('file_pre.html')
               'cp'       : cp,

def render_ignore(path, cp):
    """ Does not render anything.
    """

    return ''

def render_tmplt(path, cp):
    """

    with open(path, 'r') as fp:

    template_dir = '/'.join(path.split('/')[:-1])
    env.globals.update(setup_template_render=setup_template_render)
    template = env.get_template(filename)
               'cp'       : cp,

#
#
#

#
#
#                                   Preamble
#
#
"""
Module to generate figures with scatter plots and histograms.
"""

import sys

from matplotlib import (offsetbox, pyplot, gridspec)

from pycbc.results import str_utils

def create_axes_grid(parameters, labels=None, height_ratios=None,
                     width_ratios=None, no_diagonals=False):
    """Given a list of parameters, creates a figure with an axis for
    every possible combination of the parameters.

    parameters : list
        Names of the variables to be plotted.
    labels : {None, dict}, optional
        A dictionary of parameters -> parameter labels.
    height_ratios : {None, list}, optional
        Set the height ratios of the axes; see `matplotlib.gridspec.GridSpec`
    width_ratios : {None, list}, optional
        Set the width ratios of the axes; see `matplotlib.gridspec.GridSpec`
    no_diagonals : {False, bool}, optional
        Do not produce axes for the same parameter on both axes.

    Returns
    -------
    fig : pyplot.figure
        The figure that was created.
    axis_dict : dict
        A dictionary mapping the parameter combinations to the axis and their
        location in the subplots grid; i.e., the key, values are:
        `{('param1', 'param2'): (pyplot.axes, row index, column index)}`
    """
    if labels is None:
        labels = {p: p for p in parameters}
    elif any(p not in labels for p in parameters):
        raise ValueError("labels must be provided for all parameters")
    # Create figure with adequate size for number of parameters.
    ndim = len(parameters)
    if no_diagonals:
        ndim -= 1
    if ndim < 3:
        fsize = (8, 7)
    else:
        fsize = (ndim*3 - 1, ndim*3 - 2)
    fig = pyplot.figure(figsize=fsize)
    # create the axis grid
    gs = gridspec.GridSpec(ndim, ndim, width_ratios=width_ratios,
                           height_ratios=height_ratios,
                           wspace=0.05, hspace=0.05)
    # create grid of axis numbers to easily create axes in the right locations
    axes = numpy.arange(ndim**2).reshape((ndim, ndim))

    # Select possible combinations of plots and establish rows and columns.
    combos = list(itertools.combinations(parameters, 2))
    # add the diagonals
    if not no_diagonals:
        combos += [(p, p) for p in parameters]

    # create the mapping between parameter combos and axes
    axis_dict = {}
    # cycle over all the axes, setting thing as needed
    for nrow in range(ndim):
        for ncolumn in range(ndim):
            ax = pyplot.subplot(gs[axes[nrow, ncolumn]])
            # map to a parameter index
            px = parameters[ncolumn]
            if no_diagonals:
                py = parameters[nrow+1]
            else:
                py = parameters[nrow]
            if (px, py) in combos:
                axis_dict[px, py] = (ax, nrow, ncolumn)
                # x labels only on bottom
                if nrow + 1 == ndim:
                    ax.set_xlabel('{}'.format(labels[px]), fontsize=18)
                else:
                    pyplot.setp(ax.get_xticklabels(), visible=False)
                # y labels only on left
                if ncolumn == 0:
                    ax.set_ylabel('{}'.format(labels[py]), fontsize=18)
                else:
                    pyplot.setp(ax.get_yticklabels(), visible=False)
            else:
                # make non-used axes invisible
                ax.axis('off')
    return fig, axis_dict

def get_scale_fac(fig, fiducial_width=8, fiducial_height=7):
    """Gets a factor to scale fonts by for the given figure. The scale
    factor is relative to a figure with dimensions
    (`fiducial_width`, `fiducial_height`).
    """
    width, height = fig.get_size_inches()
    return (width*height/(fiducial_width*fiducial_height))**0.5

def construct_kde(samples_array, use_kombine=False):
    """Constructs a KDE from the given samples.
    """
    if use_kombine:
        try:
            import kombine
            raise ImportError("kombine is not installed.")
    # construct the kde
        kde = kombine.clustered_kde.KDE(samples_array)
    else:
        kde = scipy.stats.gaussian_kde(samples_array.T)
    return kde

def create_density_plot(xparam, yparam, samples, plot_density=True,
                        plot_contours=True, percentiles=None, cmap='viridis',
                        contour_color=None, xmin=None, xmax=None,
                        ymin=None, ymax=None, exclude_region=None,
                        fig=None, ax=None, use_kombine=False):
    """Computes and plots posterior density and confidence intervals using the
    given samples.

    xparam : string
        The parameter to plot on the x-axis.
    yparam : string
        The parameter to plot on the y-axis.
    samples : dict, numpy structured array, or FieldArray
        The samples to plot.
    plot_density : {True, bool}
        Plot a color map of the density.
    plot_contours : {True, bool}
        Plot contours showing the n-th percentiles of the density.
    percentiles : {None, float or array}
        What percentile contours to draw. If None, will plot the 50th
        and 90th percentiles.
    cmap : {'viridis', string}
        The name of the colormap to use for the density plot.
    contour_color : {None, string}
        What color to make the contours. Default is white for density
        plots and black for other plots.
    xmin : {None, float}
        Minimum value to plot on x-axis.
    xmax : {None, float}
        Maximum value to plot on x-axis.
    ymin : {None, float}
        Minimum value to plot on y-axis.
    ymax : {None, float}
        Maximum value to plot on y-axis.
    exclue_region : {None, str}
        Exclude the specified region when plotting the density or contours.
        Must be a string in terms of `xparam` and `yparam` that is
        understandable by numpy's logical evaluation. For example, if
        `xparam = m_1` and `yparam = m_2`, and you want to exclude the region
        for which `m_2` is greater than `m_1`, then exclude region should be
        `'m_2 > m_1'`.
    fig : {None, pyplot.figure}
        Add the plot to the given figure. If None and ax is None, will create
        a new figure.
    ax : {None, pyplot.axes}
        Draw plot on the given axis. If None, will create a new axis from
        `fig`.
    use_kombine : {False, bool}
        Use kombine's KDE to calculate density. Otherwise, will use
        `scipy.stats.gaussian_kde.` Default is False.

    Returns
    -------
        The figure the plot was made on.
    ax : pyplot.axes
        The axes the plot was drawn on.
    """
    if percentiles is None:
        percentiles = numpy.array([50., 90.])
    percentiles = 100. - numpy.array(percentiles)
    percentiles.sort()

    if ax is None and fig is None:
        fig = pyplot.figure()
    if ax is None:
        ax = fig.add_subplot(111)

    # convert samples to array and construct kde
    xsamples = samples[xparam]
    ysamples = samples[yparam]
    arr = numpy.vstack((xsamples, ysamples)).T
    kde = construct_kde(arr, use_kombine=use_kombine)

    # construct grid to evaluate on
    if xmin is None:
        xmin = xsamples.min()
    if xmax is None:
        xmax = xsamples.max()
    if ymin is None:
        ymin = ysamples.min()
    if ymax is None:
        ymax = ysamples.max()
    npts = 100
    X, Y = numpy.mgrid[
        xmin:xmax:complex(0, npts),  # pylint:disable=invalid-slice-index
        ymin:ymax:complex(0, npts)]  # pylint:disable=invalid-slice-index
    pos = numpy.vstack([X.ravel(), Y.ravel()])
        Z = numpy.exp(kde(pos.T).reshape(X.shape))
        draw = kde.draw
    else:
        Z = kde(pos).T.reshape(X.shape)
        draw = kde.resample

    if exclude_region is not None:
        # convert X,Y to a single FieldArray so we can use it's ability to
        # evaluate strings
        farr = FieldArray.from_kwargs(**{xparam: X, yparam: Y})
        Z[farr[exclude_region]] = 0.

    if plot_density:
        ax.imshow(numpy.rot90(Z), extent=[xmin, xmax, ymin, ymax],
                  aspect='auto', cmap=cmap, zorder=1)
        if contour_color is None:
            contour_color = 'w'

    if plot_contours:
        # compute the percentile values
        resamps = kde(draw(int(npts**2)))
        if use_kombine:
            resamps = numpy.exp(resamps)
        s = numpy.percentile(resamps, percentiles)
            contour_color = 'k'
        # make linewidths thicker if not plotting density for clarity
        if plot_density:
            lw = 1
        else:
            lw = 2
        ct = ax.contour(X, Y, Z, s, colors=contour_color, linewidths=lw,
                        zorder=3)
        # label contours
        lbls = ['{p}%'.format(p=int(p)) for p in (100. - percentiles)]
        fmt = dict(zip(ct.levels, lbls))
        fs = 12
        ax.clabel(ct, ct.levels, inline=True, fmt=fmt, fontsize=fs)

    return fig, ax

def create_marginalized_hist(ax, values, label, percentiles=None,
                             color='k', fillcolor='gray', linecolor='navy',
                             linestyle='-',
                             title=True, expected_value=None,
                             expected_color='red', rotated=False,
                             plot_min=None, plot_max=None):
    """Plots a 1D marginalized histogram of the given param from the given
    samples.

    ax : pyplot.Axes
        The axes on which to draw the plot.
    values : array
        The parameter values to plot.
    label : str
        A label to use for the title.
        What percentiles to draw lines at. If None, will draw lines at
        `[5, 50, 95]` (i.e., the bounds on the upper 90th percentile and the
        median).
    color : {'k', string}
        What color to make the histogram; default is black.
    fillcolor : {'gray', string, or None}
        What color to fill the histogram with. Set to None to not fill the
        histogram. Default is 'gray'.
    linestyle : str, optional
        What line style to use for the histogram. Default is '-'.
    linecolor : {'navy', string}
        What color to use for the percentile lines. Default is 'navy'.
    title : bool, optional
        Add a title with a estimated value +/- uncertainty. The estimated value
        is the pecentile halfway between the max/min of ``percentiles``, while
        the uncertainty is given by the max/min of the ``percentiles``. If no
        percentiles are specified, defaults to quoting the median +/- 95/5
        percentiles.
    rotated : {False, bool}
        Plot the histogram on the y-axis instead of the x. Default is False.
    plot_min : {None, float}
        The minimum value to plot. If None, will default to whatever `pyplot`
        creates.
    plot_max : {None, float}
        The maximum value to plot. If None, will default to whatever `pyplot`
        creates.
    scalefac : {1., float}
        Factor to scale the default font sizes by. Default is 1 (no scaling).
    """
    if fillcolor is None:
        htype = 'step'
    else:
        htype = 'stepfilled'
    if rotated:
        orientation = 'horizontal'
    else:
        orientation = 'vertical'
    ax.hist(values, bins=50, histtype=htype, orientation=orientation,
            facecolor=fillcolor, edgecolor=color, ls=linestyle, lw=2,
            density=True)
        percentiles = [5., 50., 95.]
    if len(percentiles) > 0:
        plotp = numpy.percentile(values, percentiles)
    else:
        plotp = []
    for val in plotp:
        if rotated:
            ax.axhline(y=val, ls='dashed', color=linecolor, lw=2, zorder=3)
        else:
            ax.axvline(x=val, ls='dashed', color=linecolor, lw=2, zorder=3)
    # plot expected
    if expected_value is not None:
            ax.axhline(expected_value, color=expected_color, lw=1.5, zorder=2)
        else:
            ax.axvline(expected_value, color=expected_color, lw=1.5, zorder=2)
    if title:
        if len(percentiles) > 0:
            minp = min(percentiles)
            maxp = max(percentiles)
            medp = (maxp + minp) / 2.
        else:
            minp = 5
            medp = 50
            maxp = 95
        values_min = numpy.percentile(values, minp)
        values_med = numpy.percentile(values, medp)
        values_max = numpy.percentile(values, maxp)
        negerror = values_med - values_min
        poserror = values_max - values_med
        fmt = '${0}$'.format(str_utils.format_value(
            values_med, negerror, plus_error=poserror))
            ax.yaxis.set_label_position("right")
            # sets colored title for marginal histogram
            set_marginal_histogram_title(ax, fmt, color,
                                         label=label, rotated=rotated)
        else:
            set_marginal_histogram_title(ax, fmt, color, label=label)
    # remove ticks and set limits
        # Remove x-ticks
        ax.set_xticks([])
        # turn off x-labels
        ax.set_xlabel('')
        # set limits
        ymin, ymax = ax.get_ylim()
        if plot_min is not None:
            ymin = plot_min
        if plot_max is not None:
            ymax = plot_max
        ax.set_ylim(ymin, ymax)
    else:
        # Remove y-ticks
        ax.set_yticks([])
        # turn off y-label
        ax.set_ylabel('')
        xmin, xmax = ax.get_xlim()
            xmin = plot_min
            xmax = plot_max
        ax.set_xlim(xmin, xmax)

def set_marginal_histogram_title(ax, fmt, color, label=None, rotated=False):
    """ Sets the title of the marginal histograms.

    ax : Axes
        The `Axes` instance for the plot.
    fmt : str
        The string to add to the title.
    color : str
        The color of the text to add to the title.
    label : str
        If title does not exist, then include label at beginning of the string.
    rotated : bool
        If `True` then rotate the text 270 degrees for sideways title.
    """

    # get rotation angle of the title
    rotation = 270 if rotated else 0

    # get how much to displace title on axes
    xscale = 1.05 if rotated else 0.0
        yscale = 1.0
    elif len(ax.get_figure().axes) > 1:
        yscale = 1.15
    else:
        yscale = 1.05

    # get class that packs text boxes vertical or horizonitally
    packer_class = offsetbox.VPacker if rotated else offsetbox.HPacker

    # if no title exists
    if not hasattr(ax, "title_boxes"):

        # create a text box
        title = "{} = {}".format(label, fmt)
        tbox1 = offsetbox.TextArea(
                   title,
                   textprops=dict(color=color, size=15, rotation=rotation,
                                  ha='left', va='bottom'))

        # save a list of text boxes as attribute for later
        ax.title_boxes = [tbox1]

        # pack text boxes
        ybox = packer_class(children=ax.title_boxes,
                            align="bottom", pad=0, sep=5)

    # else append existing title
    else:

        # delete old title
        ax.title_anchor.remove()

        # add new text box to list
                   " {}".format(fmt),
        ax.title_boxes = ax.title_boxes + [tbox1]

    # add new title and keep reference to instance as an attribute
    anchored_ybox = offsetbox.AnchoredOffsetbox(
                      loc=2, child=ybox, pad=0.,
                      frameon=False, bbox_to_anchor=(xscale, yscale),
                      bbox_transform=ax.transAxes, borderpad=0.)
    ax.title_anchor = ax.add_artist(anchored_ybox)

def create_multidim_plot(parameters, samples, labels=None,
                         mins=None, maxs=None, expected_parameters=None,
                         expected_parameters_color='r',
                         plot_marginal=True, plot_scatter=True,
                         marginal_percentiles=None, contour_percentiles=None,
                         marginal_title=True, marginal_linestyle='-',
                         zvals=None, show_colorbar=True, cbar_label=None,
                         vmin=None, vmax=None, scatter_cmap='plasma',
                         plot_density=False, plot_contours=True,
                         density_cmap='viridis',
                         contour_color=None, hist_color='black',
                         line_color=None, fill_color='gray',
                         use_kombine=False, fig=None, axis_dict=None):
    """Generate a figure with several plots and histograms.

    parameters: list
    samples : FieldArray
        A field array of the samples to plot.
    labels: dict, optional
        A dictionary mapping parameters to labels. If none provided, will just
        use the parameter strings as the labels.
    mins : {None, dict}, optional
        Minimum value for the axis of each variable in `parameters`.
        If None, it will use the minimum of the corresponding variable in
        `samples`.
    maxs : {None, dict}, optional
        Maximum value for the axis of each variable in `parameters`.
        If None, it will use the maximum of the corresponding variable in
    expected_parameters : {None, dict}, optional
        Expected values of `parameters`, as a dictionary mapping parameter
        names -> values. A cross will be plotted at the location of the
        expected parameters on axes that plot any of the expected parameters.
    expected_parameters_color : {'r', string}, optional
        What color to make the expected parameters cross.
    plot_marginal : {True, bool}
        Plot the marginalized distribution on the diagonals. If False, the
        diagonal axes will be turned off.
    plot_scatter : {True, bool}
        Plot each sample point as a scatter plot.
    marginal_percentiles : {None, array}
        What percentiles to draw lines at on the 1D histograms.
        If None, will draw lines at `[5, 50, 95]` (i.e., the bounds on the
        upper 90th percentile and the median).
    marginal_title : bool, optional
        Add a title over the 1D marginal plots that gives an estimated value
        +/- uncertainty. The estimated value is the pecentile halfway between
        the max/min of ``maginal_percentiles``, while the uncertainty is given
        by the max/min of the ``marginal_percentiles. If no
        ``marginal_percentiles`` are specified, the median +/- 95/5 percentiles
        will be quoted.
    marginal_linestyle : str, optional
        What line style to use for the marginal histograms.
    contour_percentiles : {None, array}
        What percentile contours to draw on the scatter plots. If None,
        will plot the 50th and 90th percentiles.
    zvals : {None, array}
        An array to use for coloring the scatter plots. If None, scatter points
        will be the same color.
    show_colorbar : {True, bool}
        Show the colorbar of zvalues used for the scatter points. A ValueError
        will be raised if zvals is None and this is True.
    cbar_label : {None, str}
        Specify a label to add to the colorbar.
    vmin: {None, float}, optional
        Minimum value for the colorbar. If None, will use the minimum of zvals.
    vmax: {None, float}, optional
        Maximum value for the colorbar. If None, will use the maxmimum of
        zvals.
    scatter_cmap : {'plasma', string}
        The color map to use for the scatter points. Default is 'plasma'.
    plot_density : {False, bool}
        Plot the density of points as a color map.
        Draw contours showing the 50th and 90th percentile confidence regions.
    density_cmap : {'viridis', string}
        The color map to use for the density plot.
        The color to use for the contour lines. Defaults to white for
        density plots, navy for scatter plots without zvals, and black
        otherwise.

    Returns
    -------
    """
    # set up the figure with a grid of axes
    # if only plotting 2 parameters, make the marginal plots smaller
    nparams = len(parameters)
    if nparams == 2:
        width_ratios = [3, 1]
        height_ratios = [1, 3]
    else:
        width_ratios = height_ratios = None

    # only plot scatter if more than one parameter
    plot_scatter = plot_scatter and nparams > 1

    # Sort zvals to get higher values on top in scatter plots
    if plot_scatter:
        if zvals is not None:
            sort_indices = zvals.argsort()
            zvals = zvals[sort_indices]
            samples = samples[sort_indices]
            if contour_color is None:
                contour_color = 'k'
        elif show_colorbar:
            raise ValueError("must provide z values to create a colorbar")
        else:
            # just make all scatter points same color
            zvals = 'gray'
            if plot_contours and contour_color is None:
                contour_color = 'navy'

    # convert samples to a dictionary to avoid re-computing derived parameters
    # every time they are needed
    samples = dict([[p, samples[p]] for p in parameters])

    # values for axis bounds
    if mins is None:
        mins = {p: samples[p].min() for p in parameters}
    else:
        # copy the dict
        mins = {p: val for p, val in mins.items()}
    if maxs is None:
        maxs = {p: samples[p].max() for p in parameters}
    else:
        maxs = {p: val for p, val in maxs.items()}

    if fig is None and axis_dict is None:
        fig, axis_dict = create_axes_grid(
            parameters, labels=labels,
            width_ratios=width_ratios, height_ratios=height_ratios,
            no_diagonals=not plot_marginal)

    # Diagonals...
    if plot_marginal:
        for pi, param in enumerate(parameters):
            ax, _, _ = axis_dict[param, param]
            # if only plotting 2 parameters and on the second parameter,
            # rotate the marginal plot
            rotated = nparams == 2 and pi == nparams-1
            # see if there are expected values
            if expected_parameters is not None:
                try:
                    expected_value = expected_parameters[param]
                except KeyError:
                    expected_value = None
            else:
                expected_value = None
            create_marginalized_hist(
                ax, samples[param], label=labels[param],
                color=hist_color, fillcolor=fill_color,
                linestyle=marginal_linestyle, linecolor=line_color,
                title=marginal_title, expected_value=expected_value,
                expected_color=expected_parameters_color,
                rotated=rotated, plot_min=mins[param], plot_max=maxs[param],
                percentiles=marginal_percentiles)

    # Off-diagonals...
    for px, py in axis_dict:
        if px == py:
            continue
        ax, _, _ = axis_dict[px, py]
        if plot_scatter:
            if plot_density:
                alpha = 0.3
            else:
                alpha = 1.
            plt = ax.scatter(x=samples[px], y=samples[py], c=zvals, s=5,
                             edgecolors='none', vmin=vmin, vmax=vmax,
                             cmap=scatter_cmap, alpha=alpha, zorder=2)

        if plot_contours or plot_density:
            # Exclude out-of-bound regions
            # this is a bit kludgy; should probably figure out a better
            # solution to eventually allow for more than just m_p m_s
            if (px == 'm_p' and py == 'm_s') or (py == 'm_p' and px == 'm_s'):
                exclude_region = 'm_s > m_p'
            else:
                exclude_region = None
            create_density_plot(
                px, py, samples, plot_density=plot_density,
                plot_contours=plot_contours, cmap=density_cmap,
                percentiles=contour_percentiles,
                contour_color=contour_color, xmin=mins[px], xmax=maxs[px],
                ymin=mins[py], ymax=maxs[py],
                exclude_region=exclude_region, ax=ax,
                use_kombine=use_kombine)

        if expected_parameters is not None:
            try:
                ax.axvline(expected_parameters[px], lw=1.5,
                           color=expected_parameters_color, zorder=5)
                pass
            try:
                ax.axhline(expected_parameters[py], lw=1.5,
                pass

        ax.set_xlim(mins[px], maxs[px])
        ax.set_ylim(mins[py], maxs[py])

    # adjust tick number for large number of plots
    if len(parameters) > 3:
        for px, py in axis_dict:
            ax, _, _ = axis_dict[px, py]
            ax.set_xticks(reduce_ticks(ax, 'x', maxticks=3))
            ax.set_yticks(reduce_ticks(ax, 'y', maxticks=3))

    if plot_scatter and show_colorbar:
        # compute font size based on fig size
        scale_fac = get_scale_fac(fig)
        fig.subplots_adjust(right=0.85, wspace=0.03)
        cbar_ax = fig.add_axes([0.9, 0.1, 0.03, 0.8])
        cb = fig.colorbar(plt, cax=cbar_ax)
        if cbar_label is not None:
            cb.set_label(cbar_label, fontsize=12*scale_fac)
        cb.ax.tick_params(labelsize=8*scale_fac)

def remove_common_offset(arr):
    """Given an array of data, removes a common offset > 1000, returning the
    removed value.
    """
    offset = 0
    isneg = (arr <= 0).all()
    # make sure all values have the same sign
    if isneg or (arr >= 0).all():
        # only remove offset if the minimum and maximum values are the same
        # order of magintude and > O(1000)
        minpwr = numpy.log10(abs(arr).min())
        maxpwr = numpy.log10(abs(arr).max())
        if numpy.floor(minpwr) == numpy.floor(maxpwr) and minpwr > 3:
            offset = numpy.floor(10**minpwr)
            if isneg:
                offset *= -1
            arr = arr - offset
    return arr, int(offset)

def reduce_ticks(ax, which, maxticks=3):
    """Given a pyplot axis, resamples its `which`-axis ticks such that are at most
    `maxticks` left.

    ax : axis
        The axis to adjust.
    which : {'x' | 'y'}
        Which axis to adjust.
    maxticks : {3, int}
        Maximum number of ticks to use.

    Returns
    -------
    array
        An array of the selected ticks.
    """
    ticks = getattr(ax, 'get_{}ticks'.format(which))()
    if len(ticks) > maxticks:
        # make sure the left/right value is not at the edge
        minax, maxax = getattr(ax, 'get_{}lim'.format(which))()
        dw = abs(maxax-minax)/10.
        start_idx, end_idx = 0, len(ticks)
        if ticks[0] < minax + dw:
            start_idx += 1
        if ticks[-1] > maxax - dw:
            end_idx -= 1
        # get reduction factor
        fac = int(len(ticks) / maxticks)
        ticks = ticks[start_idx:end_idx:fac]
    return ticks

#
#

#
#
#                                   Preamble
#
#
"""
This modules provides functions for formatting values into strings for display.
"""

mjax_header = """
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$']]}});
</script>
<script type="text/javascript"
    src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
"""

def mathjax_html_header():
    """Standard header to use for html pages to display latex math.

    Returns
    -------
    header: str
        The necessary html head needed to use latex on an html page.
    """
    return mjax_header

def drop_trailing_zeros(num):
    """
    Drops the trailing zeros in a float that is printed.
    """
    txt = '%f' %(num)
    txt = txt.rstrip('0')
    if txt.endswith('.'):
        txt = txt[:-1]
    return txt

def get_signum(val, err, max_sig=numpy.inf):
    """
    Given an error, returns a string for val formated to the appropriate
    number of significant figures.
    """
    coeff, pwr = ('%e' % err).split('e')
    if pwr.startswith('-'):
        pwr = int(pwr[1:])
        if round(float(coeff)) == 10.:
            pwr -= 1
        pwr = min(pwr, max_sig)
        tmplt = '%.' + str(pwr+1) + 'f'
        return tmplt % val
    else:
            pwr += 1
        # if the error is large, we can sometimes get 0;
        # adjust the round until we don't get 0 (assuming the actual
        # value isn't 0)
        return_val = round(val, -pwr+1)
        if val != 0.:
            loop_count = 0
            max_recursion = 100
            while return_val == 0.:
                pwr -= 1
                return_val = round(val, -pwr+1)
                loop_count += 1
                if loop_count > max_recursion:
                    raise ValueError("Maximum recursion depth hit! Input " +\
                        "values are: val = %f, err = %f" %(val, err))
        return drop_trailing_zeros(return_val)

def format_value(value, error, plus_error=None, use_scientific_notation=3,
        include_error=True, use_relative_error=False, ndecs=None):
    """Given a numerical value and some bound on it, formats the number into a
    string such that the value is rounded to the nearest significant figure,
    which is determined by the error = abs(value-bound).

    Note: if either use_scientific_notation or include_error are True, the
    returned string will include LaTeX characters.

        The value to format.
    error : float
        The uncertainty in the value. This is used to determine the
        number of significant figures to print. If the value has no
        uncertainty, you can just do value*1e-k, where k+1 is the number
        of significant figures you want.
    plus_error : {None, float}
        The upper uncertainty on the value; i.e., what you need to add to the
        value to get its upper bound. If provided, ``error`` is assumed to be
        the negative; i.e., value +plus_error -error. The number of
        significant figures printed is determined from min(error,
        plus_error).
    use_scientific_notation : int, optional
        If ``abs(log10(value))`` is greater than the given, the return string
        will be formated to "\%.1f \\times 10^{p}", where p is the powers of 10
        needed for the leading number in the value to be in the singles spot.
        Otherwise will return "\%.(p+1)f". Default is 3.  To turn off, set to
        ``numpy.inf``. Note: using scientific notation assumes that the
        returned value will be enclosed in LaTeX math mode.
    include_error : {True, bool}
        Include the error in the return string; the output will be formated
        val \\pm err, where err is the error rounded to the same
        power of 10 as val. Otherwise, just the formatted value will
        be returned. If plus_error is provided then the return text will be
        formatted as ``val^{+plus_error}_{-error}``.
    use_relative_error : {False, bool}
        If include_error, the error will be formatted as a percentage of the
        the value.
    ndecs: {None, int}
        Number of values after the decimal point. If not provided,
        it will default to the number of values in the error.

    Returns
    -------
    string
        The value (and error, if include_error is True) formatted as a string.

    Examples
    --------
    Given a value and its uncertainty:

    >>> val, err
    (3.9278372067613837e-22, 2.2351435286500487e-23)

    Format with error quoted:

    >>> format_value(val, err)
    '3.93 \\pm 0.22\\times 10^{-22}'

    Quote error as a relative error:

    >>> format_value(val, err, use_relative_error=True)
    '3.93 \\times 10^{-22} \\pm5.6\\%'

    Format without the error and without scientific notation:

    >>> format_value(val, err, use_scientific_notation=float('inf'),
                     include_error=False)
    '0.000000000000000000000393'

    Given an plus error:

    >>> err_plus
    8.2700310560051804e-24

    Format with both bounds quoted:

    >>> format_value(val, err, plus_error=err_plus)
    '3.928^{+0.083}_{-0.224}\\times 10^{-22}'

    Format with both bounds quoted as a relative error:

    >>> format_value(val, err, plus_error=err_plus, use_relative_error=True)
    '3.928\\times 10^{-22}\\,^{+2.1\\%}_{-5.7\\%}'

    """
    minus_sign = '-' if value < 0. else ''
    value = abs(value)
    minus_err = abs(error)
    if plus_error is None:
        plus_err = minus_err
    else:
        plus_err = abs(plus_error)
    error = min(minus_err, plus_err)
    if value == 0. or abs(numpy.log10(value)) < use_scientific_notation:
        conversion_factor = 0.
    else:
        conversion_factor = numpy.floor(numpy.log10(value))
    value = value * 10**(-conversion_factor)
    error = error * 10**(-conversion_factor)
    if conversion_factor == 0.:
        powfactor = ''
    elif conversion_factor == 1.:
        powfactor = r'\times 10'
    else:
        powfactor = r'\times 10^{%i}' %(int(conversion_factor))

    if ndecs is not None:
        decs = value * 10**(-ndecs)
    else:
        decs = error
    # now round the the appropriate number of sig figs
    valtxt = get_signum(value, decs)
    valtxt = '{}{}'.format(minus_sign, valtxt)

    if include_error:
        if plus_error is None:
            errtxt = get_signum(error, error)
            if use_relative_error and float(valtxt) != 0.:
                relative_err = 100.*float(errtxt)/float(valtxt)
                # we round the relative error to the nearest 1% using
                # get_signum; Note that if the relative error is < 1%,
                # get_signum will automatically increase the number of values
                # after the decimal until it gets to the first non-zero value
                relative_err = get_signum(relative_err, 1.)
                txt = r'%s %s \pm%s\%%' %(valtxt, powfactor, relative_err)
            else:
                txt = r'%s \pm %s%s' %(valtxt, errtxt, powfactor)
        else:
            plus_err = plus_err * 10**(-conversion_factor)
            minus_err = minus_err * 10**(-conversion_factor)
            minus_err_txt = get_signum(minus_err, decs)
            plus_err_txt = get_signum(plus_err, decs)
                # same as above, but with plus and minus
                rel_plus_err = get_signum(
                    100.*float(plus_err_txt)/float(valtxt), 1.)
                rel_minus_err = get_signum(
                    100.*float(minus_err_txt)/float(valtxt), 1.)
                txt = r'%s%s\,^{+%s\%%}_{-%s\%%}' %(valtxt, powfactor,
                    rel_plus_err, rel_minus_err)
            else:
                txt = r'%s^{+%s}_{-%s}%s' %(valtxt, plus_err_txt,
                    minus_err_txt, powfactor)
    else:
        txt = r'%s%s' %(valtxt, powfactor)
    return txt

#
#
#

#
#
#                                   Preamble
#
#
""" This module provides functions to generate sortable html tables
"""
import mako.template, uuid

google_table_template = mako.template.Template("""
    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
    <script type='text/javascript'>
      google.load('visualization', '1', {packages:['table']});
      google.setOnLoadCallback(drawTable);
      function drawTable() {
        var data = new google.visualization.DataTable();
        % for type, name in column_descriptions:
            data.addColumn('${str(type)}', '${str(name)}');
        % endfor
        data.addRows(${data});

        % if format_strings is not None:
            % for i, format_string in enumerate(format_strings):
                % if format_string is not None:
                    var formatter = new google.visualization.NumberFormat({pattern:'${format_string}'});
                    formatter.format(data, ${i});
                % endif
            % endfor
        % endif
        var table = new google.visualization.Table(document.getElementById('${div_id}'));
        table.draw(data, {showRowNumber: 'true',
                          page: '${page_enable}',
                          allowHtml: 'true',
                          pageSize: ${page_size}});
      }
    </script>
    <div id='${div_id}'></div>
""")

def html_table(columns, names, page_size=None, format_strings=None):
    """ Return an html table of this data

    columns : list of numpy arrays
    names : list of strings
        The list of columns names
    page_size : {int, None}, optional
        The number of items to show on each page of the table
    format_strings : {lists of strings, None}, optional
        The ICU format string for this column, None for no formatting. All
    columns must have a format string if provided.

    Returns
    -------
    html_table : str
        A str containing the html code to display a table of this data
    """
    if page_size is None:
        page = 'disable'
    else:
        page = 'enable'

    div_id = uuid.uuid4()

    column_descriptions = []
    for column, name in zip(columns, names):
        if column.dtype.kind == 'S':
            ctype = 'string'
        else:
            ctype = 'number'
        column_descriptions.append((ctype, name))

    data = []
    for item in zip(*columns):
        data.append(list(item))

    return google_table_template.render(div_id=div_id,
                                page_enable=page,
                                column_descriptions = column_descriptions,
                                page_size=page_size,
                                data=data,
                                format_strings=format_strings,
                               )

static_table_template = mako.template.Template("""
    <table class="table">
        % if titles is not None:
            <tr>
            % for i in range(len(titles)):
                <th>
                    ${titles[i]}
                </th>
            % endfor
            </tr>
        % endif

        % for i in range(len(data)):
            <tr>
            % for j in range(len(data[i])):
                <td>
                    ${data[i][j]}
                </td>
            % endfor
            </tr>
        % endfor
    </table>
""")

def static_table(data, titles=None):
    """ Return an html tableo of this data

    data : two-dimensional numpy string array
        Array containing the cell values
    titles : numpy array
        Vector str of titles

    Returns
    -------
        A string containing the html table.
    """
    return static_table_template.render(data=data, titles=titles)

# Copyright (C) 2015 Ian Harry
#
#
#

import os
import subprocess
from pycbc.results import save_fig_with_metadata, html_escape

import lal, lalframe, lalsimulation
import pycbc.version, glue.git_version

def get_library_version_info():
    """This will return a list of dictionaries containing versioning
    information about the various LIGO libraries that PyCBC will use in an
    analysis run."""
    library_list = []

    def add_info_new_version(info_dct, curr_module, extra_str):
        vcs_object = getattr(curr_module, extra_str +'VCSInfo')
        info_dct['ID'] =  vcs_object.vcsId
        info_dct['Status'] = vcs_object.vcsStatus
        info_dct['Version'] = vcs_object.version
        info_dct['Tag'] = vcs_object.vcsTag
        info_dct['Author'] = vcs_object.vcsAuthor
        info_dct['Branch'] = vcs_object.vcsBranch
        info_dct['Committer'] = vcs_object.vcsCommitter
        info_dct['Date'] = vcs_object.vcsDate

    lalinfo = {}
    lalinfo['Name'] = 'LAL'
    try:
        lalinfo['ID'] = lal.VCSId
        lalinfo['Status'] = lal.VCSStatus
        lalinfo['Version'] = lal.VCSVersion
        lalinfo['Tag'] = lal.VCSTag
        lalinfo['Author'] = lal.VCSAuthor
        lalinfo['Branch'] = lal.VCSBranch
        lalinfo['Committer'] = lal.VCSCommitter
        lalinfo['Date'] = lal.VCSDate
        add_info_new_version(lalinfo, lal, '')
    library_list.append(lalinfo)

    lalframeinfo = {}
    try:
        lalframeinfo['Name'] = 'LALFrame'
        lalframeinfo['ID'] = lalframe.FrameVCSId
        lalframeinfo['Status'] = lalframe.FrameVCSStatus
        lalframeinfo['Version'] = lalframe.FrameVCSVersion
        lalframeinfo['Tag'] = lalframe.FrameVCSTag
        lalframeinfo['Author'] = lalframe.FrameVCSAuthor
        lalframeinfo['Branch'] = lalframe.FrameVCSBranch
        lalframeinfo['Committer'] = lalframe.FrameVCSCommitter
        lalframeinfo['Date'] = lalframe.FrameVCSDate
        add_info_new_version(lalframeinfo, lalframe, 'Frame')
    library_list.append(lalframeinfo)

    lalsimulationinfo = {}
    lalsimulationinfo['Name'] = 'LALSimulation'
    try:
        lalsimulationinfo['ID'] = lalsimulation.SimulationVCSId
        lalsimulationinfo['Status'] = lalsimulation.SimulationVCSStatus
        lalsimulationinfo['Version'] = lalsimulation.SimulationVCSVersion
        lalsimulationinfo['Tag'] = lalsimulation.SimulationVCSTag
        lalsimulationinfo['Author'] = lalsimulation.SimulationVCSAuthor
        lalsimulationinfo['Branch'] = lalsimulation.SimulationVCSBranch
        lalsimulationinfo['Committer'] = lalsimulation.SimulationVCSCommitter
        lalsimulationinfo['Date'] = lalsimulation.SimulationVCSDate
        add_info_new_version(lalsimulationinfo, lalsimulation, 'Simulation')
    library_list.append(lalsimulationinfo)

    glueinfo = {}
    glueinfo['Name'] = 'LSCSoft-Glue'
    glueinfo['ID'] = glue.git_version.id
    glueinfo['Status'] = glue.git_version.status
    glueinfo['Version'] = glue.git_version.version
    glueinfo['Tag'] = glue.git_version.tag
    glueinfo['Author'] = glue.git_version.author
    glueinfo['Builder'] = glue.git_version.builder
    glueinfo['Branch'] = glue.git_version.branch
    glueinfo['Committer'] = glue.git_version.committer
    glueinfo['Date'] = glue.git_version.date
    library_list.append(glueinfo)

    pycbcinfo = {}
    pycbcinfo['Name'] = 'PyCBC'
    pycbcinfo['ID'] = pycbc.version.version
    pycbcinfo['Status'] = pycbc.version.git_status
    pycbcinfo['Version'] = pycbc.version.release or ''
    pycbcinfo['Tag'] = pycbc.version.git_tag
    pycbcinfo['Author'] = pycbc.version.git_author
    pycbcinfo['Builder'] = pycbc.version.git_builder
    pycbcinfo['Branch'] = pycbc.version.git_branch
    pycbcinfo['Committer'] = pycbc.version.git_committer
    pycbcinfo['Date'] = pycbc.version.git_build_date
    library_list.append(pycbcinfo)

    return library_list

def write_library_information(path):
    library_list = get_library_version_info()
    for curr_lib in library_list:
        lib_name = curr_lib['Name']
        text = ''
        for key, value in curr_lib.items():
            text+='<li> %s : %s </li>\n' %(key,value)
        kwds = {'render-function' : 'render_text',
                'title' : '%s Version Information'%lib_name,
        }
        
        save_fig_with_metadata(html_escape(text), 
          os.path.join(path,'%s_version_information.html' %(lib_name)), **kwds)

def get_code_version_numbers(cp):
    """Will extract the version information from the executables listed in
    the executable section of the supplied ConfigParser object.

    Returns
    --------
    dict
        A dictionary keyed by the executable name with values giving the
        version string for each executable.
    """
    code_version_dict = {}
    for _, value in cp.items('executables'):
        _, exe_name = os.path.split(value)
        version_string = None
        if value.startswith('gsiftp://') or value.startswith('http://'):
            code_version_dict[exe_name] = "Using bundle downloaded from %s" % value
        else:
            try:
                if value.startswith('file://'):
                    value = value[7:]
                version_string = subprocess.check_output([value, '--version'],
                                                        stderr=subprocess.STDOUT) 
            except subprocess.CalledProcessError:
                version_string = "Executable fails on %s --version" % (value)
            except OSError:
                version_string = "Executable doesn't seem to exist(!)"
            code_version_dict[exe_name] = version_string
    return code_version_dict

def write_code_versions(path, cp):
    code_version_dict = get_code_version_numbers(cp)
    html_text = ''
    for key,value in code_version_dict.items():
        # value might be a str or a bytes object in python3. python2 is happy
        # to combine these objects (or uniocde and str, their equivalents)
        # but python3 is not.
        try:
            value = value.decode()
            pass
        html_text+= '<li><b>%s</b>:<br><pre>%s</pre></li><hr><br><br>\n' \
            % (key, str(value).replace('@', '&#64;'))
    kwds = {'render-function' : 'render_text',
            'title' : 'Version Information from Executables',
    }
    save_fig_with_metadata(html_escape(html_text), 
        os.path.join(path,'version_information_from_executables.html'), **kwds)

def create_versioning_page(path, cp):
    logging.info("Entering versioning module")
    if not os.path.exists(path):
        os.mkdir(path)
    write_library_information(path)
    write_code_versions(path, cp)
    logging.info("Leaving versioning module")

# Copyright (C) 2018 Colm Talbot
#
#
#
""" Functions for adding calibration factors to waveform templates.
"""

from scipy.interpolate import UnivariateSpline

class Recalibrate(object):
    name = None

    def __init__(self, ifo_name):
        self.ifo_name = ifo_name
        self.params = dict()

    def apply_calibration(self, strain):
        """Apply calibration model

        This method should be overwritten by subclasses

        strain : FrequencySeries
            The strain to be recalibrated.

        Return
        ------
        strain_adjusted : FrequencySeries
            The recalibrated strain.
        """
        return

    def map_to_adjust(self, strain, prefix='recalib_', **params):
        """Map an input dictionary of sampling parameters to the
        adjust_strain function by filtering the dictionary for the
        calibration parameters, then calling adjust_strain.

        prefix: str
            Prefix for calibration parameter names
        params : dict
            Dictionary of sampling parameters which includes
            calibration parameters.
        Return
        ------
        """

        self.params.update({
            key[len(prefix):]: params[key]
            for key in params if prefix in key and self.ifo_name in key})

        strain_adjusted = self.apply_calibration(strain)

        return strain_adjusted

    def from_config(cls, cp, ifo, section):
        """Read a config file to get calibration options and transfer
        functions which will be used to intialize the model.

            An open config file.
        ifo : string
            The detector (H1, L1) for which the calibration model will
            be loaded.
        section : string
            The section name in the config file from which to retrieve
            the calibration options.
        Return
        ------
        instance
        """
        all_params = dict(cp.items(section))
        params = {key[len(ifo)+1:]: all_params[key]
                  for key in all_params if ifo.lower() in key}
        model = params.pop('model')
        params['ifo_name'] = ifo.lower()

        return all_models[model](**params)

class CubicSpline(Recalibrate):
    name = 'cubic_spline'

    def __init__(self, minimum_frequency, maximum_frequency, n_points,
                 ifo_name):
        """
        Cubic spline recalibration

        see https://dcc.ligo.org/LIGO-T1400682/public

        This assumes the spline points follow
        np.logspace(np.log(minimum_frequency), np.log(maximum_frequency),
                    n_points)

        minimum_frequency: float
            minimum frequency of spline points
        maximum_frequency: float
            maximum frequency of spline points
        n_points: int
            number of spline points
        """
        Recalibrate.__init__(self, ifo_name=ifo_name)
        minimum_frequency = float(minimum_frequency)
        maximum_frequency = float(maximum_frequency)
        n_points = int(n_points)
        if n_points < 4:
            raise ValueError(
                'Use at least 4 spline points for calibration model')
        self.n_points = n_points
        self.spline_points = np.logspace(np.log10(minimum_frequency),
                                         np.log10(maximum_frequency), n_points)

        This applies cubic spline calibration to the strain.

        Return
        ------
        """
        amplitude_parameters =\
            [self.params['amplitude_{}_{}'.format(self.ifo_name, ii)]
             for ii in range(self.n_points)]
        amplitude_spline = UnivariateSpline(self.spline_points,
                                            amplitude_parameters)
        delta_amplitude = amplitude_spline(strain.sample_frequencies.numpy())

        phase_parameters =\
            [self.params['phase_{}_{}'.format(self.ifo_name, ii)]
        phase_spline = UnivariateSpline(self.spline_points, phase_parameters)
        delta_phase = phase_spline(strain.sample_frequencies.numpy())

        strain_adjusted = strain * (1.0 + delta_amplitude)\
            * (2.0 + 1j * delta_phase) / (2.0 - 1j * delta_phase)

all_models = {
    CubicSpline.name: CubicSpline
}

# Copyright (C) 2016 Collin Capano
#
#
#
""" Functions for applying gates to data.
"""

from scipy import linalg
from . import strain

def _gates_from_cli(opts, gate_opt):
    """Parses the given `gate_opt` into something understandable by
    `strain.gate_data`.
    """
    gates = {}
    if getattr(opts, gate_opt) is None:
        return gates
    for gate in getattr(opts, gate_opt):
        try:
            ifo, central_time, half_dur, taper_dur = gate.split(':')
            central_time = float(central_time)
            half_dur = float(half_dur)
            taper_dur = float(taper_dur)
            raise ValueError("--gate {} not formatted correctly; ".format(
                gate) + "see help")
        try:
            gates[ifo].append((central_time, half_dur, taper_dur))
            gates[ifo] = [(central_time, half_dur, taper_dur)]
    return gates

def gates_from_cli(opts):
    """Parses the --gate option into something understandable by
    """
    return _gates_from_cli(opts, 'gate')

def psd_gates_from_cli(opts):
    """Parses the --psd-gate option into something understandable by
    """
    return _gates_from_cli(opts, 'psd_gate')

def apply_gates_to_td(strain_dict, gates):
    """Applies the given dictionary of gates to the given dictionary of
    strain.

        Dictionary of time-domain strain, keyed by the ifos.
        Dictionary of gates. Keys should be the ifo to apply the data to,
        values are a tuple giving the central time of the gate, the half
        duration, and the taper duration.

    Returns
    -------
    dict
        Dictionary of time-domain strain with the gates applied.
    """
    # copy data to new dictionary
    outdict = dict(strain_dict.items())
    for ifo in gates:
        outdict[ifo] = strain.gate_data(outdict[ifo], gates[ifo])
    return outdict

def apply_gates_to_fd(stilde_dict, gates):
    strain in the frequency domain.

    Gates are applied by IFFT-ing the strain data to the time domain, applying
    the gate, then FFT-ing back to the frequency domain.

        Dictionary of frequency-domain strain, keyed by the ifos.

    Returns
    -------
    dict
        Dictionary of frequency-domain strain with the gates applied.
    """
    outdict = dict(stilde_dict.items())
    # create a time-domin strain dictionary to apply the gates to
    strain_dict = dict([[ifo, outdict[ifo].to_timeseries()] for ifo in gates])
    # apply gates and fft back to the frequency domain
    for ifo,d in apply_gates_to_td(strain_dict, gates).items():
        outdict[ifo] = d.to_frequencyseries()

def add_gate_option_group(parser):
    """Adds the options needed to apply gates to data.

    """
    gate_group = parser.add_argument_group("Options for gating data")

    gate_group.add_argument("--gate", nargs="+", type=str,
                            metavar="IFO:CENTRALTIME:HALFDUR:TAPERDUR",
                            help="Apply one or more gates to the data before "
                                 "filtering.")
    gate_group.add_argument("--gate-overwhitened", action="store_true",
                            help="Overwhiten data first, then apply the "
                                 "gates specified in --gate. Overwhitening "
                                 "allows for sharper tapers to be used, "
                                 "since lines are not blurred.")
    gate_group.add_argument("--psd-gate", nargs="+", type=str,
                            help="Apply one or more gates to the data used "
                                 "for computing the PSD. Gates are applied "
                                 "prior to FFT-ing the data for PSD "
                                 "estimation.")
    return gate_group

def gate_and_paint(data, lindex, rindex, invpsd, copy=True):
    """Gates and in-paints data.

    data : TimeSeries
        The data to gate.
    lindex : int
        The start index of the gate.
    rindex : int
        The end index of the gate.
    invpsd : FrequencySeries
        The inverse of the PSD.
    copy : bool, optional
        Copy the data before applying the gate. Otherwise, the gate will
        be applied in-place. Default is True.

    Returns
    -------
    TimeSeries :
        The gated and in-painted time series.
    """
    # Uses the hole-filling method of
    # https://arxiv.org/pdf/1908.05644.pdf
    # Copy the data and zero inside the hole
    if copy:
        data = data.copy()
    data[lindex:rindex] = 0

    # get the over-whitened gated data
    tdfilter = invpsd.astype('complex').to_timeseries() * invpsd.delta_t
    owhgated_data = (data.to_frequencyseries() * invpsd).to_timeseries()

    # remove the projection into the null space
    proj = linalg.solve_toeplitz(tdfilter[:(rindex - lindex)],
                                 owhgated_data[lindex:rindex])
    data[lindex:rindex] -= proj

# Copyright (C) 2019 Miriam Cabero
#
#
#
""" Functions for removing frequency lines from real data.
"""

def complex_median(complex_list):
    """ Get the median value of a list of complex numbers.

    complex_list: list
        List of complex numbers to calculate the median.

    Returns
    -------
    a + 1.j*b: complex number
        The median of the real and imaginary parts.
    """
    median_real = numpy.median([complex_number.real
                     for complex_number in complex_list])
    median_imag = numpy.median([complex_number.imag
    return median_real + 1.j*median_imag

def avg_inner_product(data1, data2, bin_size):
    """ Calculate the time-domain inner product averaged over bins.

    data1: pycbc.types.TimeSeries
        First data set.
    data2: pycbc.types.TimeSeries
        Second data set, with same duration and sample rate as data1.
    bin_size: float
        Duration of the bins the data will be divided into to calculate
        the inner product.

    Returns
    -------
    inner_prod: list
        The (complex) inner product of data1 and data2 obtained in each bin.
    amp: float
        The absolute value of the median of the inner product.
    phi: float
        The angle of the median of the inner product.
    """
    assert data1.duration == data2.duration
    assert data1.sample_rate == data2.sample_rate
    seglen = int(bin_size * data1.sample_rate)
    inner_prod = []
    for idx in range(int(data1.duration / bin_size)):
        start, end = idx * seglen, (idx+1) * seglen
        norm = len(data1[start:end])
        bin_prod = 2 * sum(data1.data[start:end].real *
                           numpy.conjugate(data2.data[start:end])) / norm
        inner_prod.append(bin_prod)

    # Get the median over all bins to avoid outliers due to the presence
    # of a signal in a particular bin.
    inner_median = complex_median(inner_prod)
    return inner_prod, numpy.abs(inner_median), numpy.angle(inner_median)

def line_model(freq, data, tref, amp=1, phi=0):
    """ Simple time-domain model for a frequency line.

    freq: float
        Frequency of the line.
    data: pycbc.types.TimeSeries
        Reference data, to get delta_t, start_time, duration and sample_times.
    tref: float
        Reference time for the line model.
    amp: {1., float}, optional
        Amplitude of the frequency line.
    phi: {0. float}, optional
        Phase of the frequency line (radians).

    Returns
    -------
    freq_line: pycbc.types.TimeSeries
        A timeseries of the line model with frequency 'freq'. The returned
        data are complex to allow measuring the amplitude and phase of the
        corresponding frequency line in the strain data. For extraction, use
        only the real part of the data.
    """
    freq_line = TimeSeries(zeros(len(data)), delta_t=data.delta_t,
                           epoch=data.start_time)

    times = data.sample_times - float(tref)
    alpha = 2 * numpy.pi * freq * times + phi
    freq_line.data = amp * numpy.exp(1.j * alpha)

    return freq_line

def matching_line(freq, data, tref, bin_size=1):
    """ Find the parameter of the line with frequency 'freq' in the data.

        Frequency of the line to find in the data.
        Data from which the line wants to be measured.
        Reference time for the frequency line.
    bin_size: {1, float}, optional
        Duration of the bins the data will be divided into for averaging.

    Returns
    -------
    line_model: pycbc.types.TimeSeries
        A timeseries containing the frequency line with the amplitude
        and phase measured from the data.
    """
    template_line = line_model(freq, data, tref=tref)
    # Measure amplitude and phase of the line in the data
    _, amp, phi = avg_inner_product(data, template_line,
                                    bin_size=bin_size)
    return line_model(freq, data, tref=tref, amp=amp, phi=phi)

def calibration_lines(freqs, data, tref=None):
    """ Extract the calibration lines from strain data.

    freqs: list
        List containing the frequencies of the calibration lines.
        Strain data to extract the calibration lines from.
    tref: {None, float}, optional
        Reference time for the line. If None, will use data.start_time.

    Returns
    -------
        The strain data with the calibration lines removed.
    """
    if tref is None:
        tref = float(data.start_time)
    for freq in freqs:
        measured_line = matching_line(freq, data, tref,
                                      bin_size=data.duration)
        data -= measured_line.data.real

def clean_data(freqs, data, chunk, avg_bin):
    """ Extract time-varying (wandering) lines from strain data.

        List containing the frequencies of the wandering lines.
        Strain data to extract the wandering lines from.
    chunk: float
        Duration of the chunks the data will be divided into to account
        for the time variation of the wandering lines. Should be smaller
        than data.duration, and allow for at least a few chunks.
    avg_bin: float
        Duration of the bins each chunk will be divided into for averaging
        the inner product when measuring the parameters of the line. Should
        be smaller than chunk.

    Returns
    -------
        The strain data with the wandering lines removed.
    """
    if avg_bin >= chunk:
        raise ValueError('The bin size for averaging the inner product '
                         'must be less than the chunk size.')
    if chunk >= data.duration:
        raise ValueError('The chunk size must be less than the '
                         'data duration.')
    steps = numpy.arange(0, int(data.duration/chunk)-0.5, 0.5)
    seglen = chunk * data.sample_rate

    tref = float(data.start_time)
        for step in steps:
            start, end = int(step*seglen), int((step+1)*seglen)
            chunk_line = matching_line(freq, data[start:end],
                                       tref, bin_size=avg_bin)

            # Apply hann window on sides of chunk_line to smooth boundaries
            # and avoid discontinuities
            hann_window = numpy.hanning(len(chunk_line))
            apply_hann = TimeSeries(numpy.ones(len(chunk_line)),
                                    delta_t=chunk_line.delta_t,
                                    epoch=chunk_line.start_time)
            if step == 0:
                apply_hann.data[len(hann_window)/2:] *= \
                                hann_window[len(hann_window)/2:]
            elif step == steps[-1]:
                apply_hann.data[:len(hann_window)/2] *= \
                                hann_window[:len(hann_window)/2]
            else:
                apply_hann.data *= hann_window
            chunk_line.data *= apply_hann.data
            data.data[start:end] -= chunk_line.data.real

""" Classes and functions for adjusting strain data.
"""
# Copyright (C) 2015 Ben Lackey, Christopher M. Biwer,
#                    Daniel Finstad, Colm Talbot, Alex Nitz
#
#
#

    """ Base class for modifying calibration """
    name = None

        Return
        ------
        """
        return

        Return
        ------
        """

            be loaded.
        Return
        ------
        instance
        """
        params = {key: params[key] for key in params}
        params.pop('model')

        return cls(**params)

    """Cubic spline recalibration

    see https://dcc.ligo.org/LIGO-T1400682/public

    This assumes the spline points follow
    np.logspace(np.log(minimum_frequency), np.log(maximum_frequency),
    n_points)

    minimum_frequency: float
    minimum frequency of spline points
    maximum_frequency: float
    maximum frequency of spline points
    n_points: int
    number of spline points
    """

        Return
        ------
        """

class PhysicalModel(object):
    """ Class for adjusting time-varying calibration parameters of given
    strain data.

    name : 'physical_model'
        The name of this calibration model.

    strain : FrequencySeries
        The strain to be adjusted.
    freq : array
        The frequencies corresponding to the values of c0, d0, a0 in Hertz.
    fc0 : float
        Coupled-cavity (CC) pole at time t0, when c0=c(t0) and a0=a(t0) are
        measured.
    c0 : array
        Initial sensing function at t0 for the frequencies.
    d0 : array
        Digital filter for the frequencies.
    a_tst0 : array
        Initial actuation function for the test mass at t0 for the
        frequencies.
    a_pu0 : array
        Initial actuation function for the penultimate mass at t0 for the
    fs0 : float
        Initial spring frequency at t0 for the signal recycling cavity.
    qinv0 : float
        Initial inverse quality factor at t0 for the signal recycling
        cavity.
    """

    name = 'physical_model'
    def __init__(self, freq=None, fc0=None, c0=None, d0=None,
                 a_tst0=None, a_pu0=None, fs0=None, qinv0=None):
        self.freq = np.real(freq)
        self.c0 = c0
        self.d0 = d0
        self.a_tst0 = a_tst0
        self.a_pu0 = a_pu0
        self.fc0 = float(fc0)
        self.fs0 = float(fs0)
        self.qinv0 = float(qinv0)

        # initial detuning at time t0
        init_detuning = self.freq**2 / (self.freq**2 - 1.0j * self.freq * \
                                        self.fs0 * self.qinv0 + self.fs0**2)

        # initial open loop gain
        self.g0 = self.c0 * self.d0 * (self.a_tst0 + self.a_pu0)

        # initial response function
        self.r0 = (1.0 + self.g0) / self.c0

        # residual of c0 after factoring out the coupled cavity pole fc0
        self.c_res = self.c0 * (1 + 1.0j * self.freq / self.fc0)/init_detuning

    def update_c(self, fs=None, qinv=None, fc=None, kappa_c=1.0):
        """ Calculate the sensing function c(f,t) given the new parameters
        kappa_c(t), kappa_a(t), f_c(t), fs, and qinv.

        fc : float
            Coupled-cavity (CC) pole at time t.
        kappa_c : float
            Scalar correction factor for sensing function at time t.
        fs : float
            Spring frequency for signal recycling cavity.
        qinv : float
            Inverse quality factor for signal recycling cavity.

        Returns
        -------
        c : numpy.array
            The new sensing function c(f,t).
        """
        detuning_term = self.freq**2 / (self.freq**2 - 1.0j *self.freq*fs * \
                                        qinv + fs**2)
        return self.c_res * kappa_c / (1 + 1.0j * self.freq/fc)*detuning_term

    def update_g(self, fs=None, qinv=None, fc=None, kappa_tst_re=1.0,
                 kappa_tst_im=0.0, kappa_pu_re=1.0, kappa_pu_im=0.0,
                 kappa_c=1.0):
        """ Calculate the open loop gain g(f,t) given the new parameters

        fc : float
            Scalar correction factor for sensing function c at time t.
        kappa_tst_re : float
            Real part of scalar correction factor for actuation function
            a_tst0 at time t.
        kappa_pu_re : float
            a_pu0 at time t.
        kappa_tst_im : float
            Imaginary part of scalar correction factor for actuation function
        kappa_pu_im : float
        fs : float

        Returns
        -------
        g : numpy.array
            The new open loop gain g(f,t).
        """
        c = self.update_c(fs=fs, qinv=qinv, fc=fc, kappa_c=kappa_c)
        a_tst = self.a_tst0 * (kappa_tst_re + 1.0j * kappa_tst_im)
        a_pu = self.a_pu0 * (kappa_pu_re + 1.0j * kappa_pu_im)
        return c * self.d0 * (a_tst + a_pu)

    def update_r(self, fs=None, qinv=None, fc=None, kappa_c=1.0,
                 kappa_tst_re=1.0, kappa_tst_im=0.0, kappa_pu_re=1.0,
                 kappa_pu_im=0.0):
        """ Calculate the response function R(f,t) given the new parameters

        fc : float
        fs : float

        Returns
        -------
        r : numpy.array
            The new response function r(f,t).
        """
        g = self.update_g(fs=fs, qinv=qinv, fc=fc, kappa_c=kappa_c,
                          kappa_tst_re=kappa_tst_re,
                          kappa_tst_im=kappa_tst_im,
                          kappa_pu_re=kappa_pu_re, kappa_pu_im=kappa_pu_im)
        return (1.0 + g) / c

    def adjust_strain(self, strain, delta_fs=None, delta_qinv=None,
                      delta_fc=None, kappa_c=1.0, kappa_tst_re=1.0,
                      kappa_tst_im=0.0, kappa_pu_re=1.0, kappa_pu_im=0.0):
        """Adjust the FrequencySeries strain by changing the time-dependent
        calibration parameters kappa_c(t), kappa_a(t), f_c(t), fs, and qinv.

            The strain data to be adjusted.
        delta_fc : float
            Change in coupled-cavity (CC) pole at time t.
            Scalar correction factor for sensing function c0 at time t.
            A_{tst0} at time t.
            A_tst0 at time t.
            A_{pu0} at time t.
        fs : float

        Returns
        -------
            The adjusted strain.
        """
        fc = self.fc0 + delta_fc if delta_fc else self.fc0
        fs = self.fs0 + delta_fs if delta_fs else self.fs0
        qinv = self.qinv0 + delta_qinv if delta_qinv else self.qinv0

        # calculate adjusted response function
        r_adjusted = self.update_r(fs=fs, qinv=qinv, fc=fc, kappa_c=kappa_c,
                                   kappa_tst_re=kappa_tst_re,
                                   kappa_tst_im=kappa_tst_im,
                                   kappa_pu_re=kappa_pu_re,
                                   kappa_pu_im=kappa_pu_im)

        # calculate error function
        k = r_adjusted / self.r0

        # decompose into amplitude and unwrapped phase
        k_amp = np.abs(k)
        k_phase = np.unwrap(np.angle(k))

        # convert to FrequencySeries by interpolating then resampling
        order = 1
        k_amp_off = UnivariateSpline(self.freq, k_amp, k=order, s=0)
        k_phase_off = UnivariateSpline(self.freq, k_phase, k=order, s=0)
        freq_even = strain.sample_frequencies.numpy()
        k_even_sample = k_amp_off(freq_even) * \
                        np.exp(1.0j * k_phase_off(freq_even))
        strain_adjusted = FrequencySeries(strain.numpy() * \
                                          k_even_sample,
                                          delta_f=strain.delta_f)

    def tf_from_file(cls, path, delimiter=" "):
        """Convert the contents of a file with the columns
        [freq, real(h), imag(h)] to a numpy.array with columns
        [freq, real(h)+j*imag(h)].

        path : string
        delimiter : {" ", string}

        Return
        ------
        numpy.array
        """
        data = np.loadtxt(path, delimiter=delimiter)
        freq = data[:, 0]
        h = data[:, 1] + 1.0j * data[:, 2]
        return np.array([freq, h]).transpose()

            be loaded.

        Return
        ------
        instance
            An instance of the Recalibrate class.
        """
        # read transfer functions
        tfs = []
        tf_names = ["a-tst", "a-pu", "c", "d"]
        for tag in ['-'.join([ifo, "transfer-function", name])
                    for name in tf_names]:
            tf_path = cp.get_opt_tag(section, tag, None)
            tfs.append(cls.tf_from_file(tf_path))
        a_tst0 = tfs[0][:, 1]
        a_pu0 = tfs[1][:, 1]
        c0 = tfs[2][:, 1]
        d0 = tfs[3][:, 1]
        freq = tfs[0][:, 0]

        # if upper stage actuation is included, read that in and add it
        # to a_pu0
        uim_tag = '-'.join([ifo, 'transfer-function-a-uim'])
        if cp.has_option(section, uim_tag):
            tf_path = cp.get_opt_tag(section, uim_tag, None)
            a_pu0 += cls.tf_from_file(tf_path)[:, 1]

        # read fc0, fs0, and qinv0
        fc0 = cp.get_opt_tag(section, '-'.join([ifo, "fc0"]), None)
        fs0 = cp.get_opt_tag(section, '-'.join([ifo, "fs0"]), None)
        qinv0 = cp.get_opt_tag(section, '-'.join([ifo, "qinv0"]), None)

        return cls(freq=freq, fc0=fc0, c0=c0, d0=d0, a_tst0=a_tst0,
                   a_pu0=a_pu0, fs0=fs0, qinv0=qinv0)

    def map_to_adjust(self, strain, **params):

        Return
        ------
        """
        # calibration param names
        arg_names = ['delta_fs', 'delta_fc', 'delta_qinv', 'kappa_c',
                     'kappa_tst_re', 'kappa_tst_im', 'kappa_pu_re',
                     'kappa_pu_im']
        # calibration param labels as they exist in config files
        arg_labels = [''.join(['calib_', name]) for name in arg_names]
        # default values for calibration params
        default_values = [0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0]
        # make list of calibration param values
        calib_args = []
        for arg, val in zip(arg_labels, default_values):
            if arg in params:
                calib_args.append(params[arg])
            else:
                calib_args.append(val)
        # adjust the strain using calibration param values
        strain_adjusted = self.adjust_strain(strain, delta_fs=calib_args[0],
                              delta_fc=calib_args[1], delta_qinv=calib_args[2],
                              kappa_c=calib_args[3],
                              kappa_tst_re=calib_args[4],
                              kappa_tst_im=calib_args[5],
                              kappa_pu_re=calib_args[6],
                              kappa_pu_im=calib_args[7])

#Copyright (C) 2013 Alex Nitz
#
#
#
"""
This modules contains functions reading, generating, and segmenting strain data
"""
import logging, numpy
import pycbc.noise
import pycbc.types
from pycbc.types import Array, FrequencySeries, complex_same_precision_as
from pycbc.types import MultiDetOptionActionSpecial
from pycbc.inject import InjectionSet, SGBurstInjectionSet
from pycbc.filter import resample_to_delta_t, highpass, make_frequency_series
from pycbc.filter.zpk import filter_zpk
from pycbc.waveform.spa_tmplt import spa_distance
import pycbc.fft
import pycbc.events
import pycbc.frame
import pycbc.filter
from scipy.signal import kaiserord

def next_power_of_2(n):
    """Return the smallest integer power of 2 larger than the argument.

    n : int
        A positive integer.

    Returns
    -------
    m : int
        Smallest integer power of 2 larger than n.
    """
    return 1 << n.bit_length()

def detect_loud_glitches(strain, psd_duration=4., psd_stride=2.,
                         psd_avg_method='median', low_freq_cutoff=30.,
                         threshold=50., cluster_window=5., corrupt_time=4.,
                         high_freq_cutoff=None, output_intermediates=False):
    """Automatic identification of loud transients for gating purposes.

    This function first estimates the PSD of the input time series using the
    FindChirp Welch method. Then it whitens the time series using that
    estimate. Finally, it computes the magnitude of the whitened series,
    thresholds it and applies the FindChirp clustering over time to the
    surviving samples.

        Input strain time series to detect glitches over.
    psd_duration : {float, 4}
        Duration of the segments for PSD estimation in seconds.
    psd_stride : {float, 2}
        Separation between PSD estimation segments in seconds.
    low_freq_cutoff : {float, 30}
        Minimum frequency to include in the whitened strain.
    threshold : {float, 50}
        Minimum magnitude of whitened strain for considering a transient to
        be present.
    cluster_window : {float, 5}
        Length of time window to cluster surviving samples over, in seconds.
    corrupt_time : {float, 4}
        Amount of time to be discarded at the beginning and end of the input
        time series.
    high_frequency_cutoff : {float, None}
        Maximum frequency to include in the whitened strain. If given, the
        input series is downsampled accordingly. If omitted, the Nyquist
        frequency is used.
    output_intermediates : {bool, False}
        Save intermediate time series for debugging.
    """

    if high_freq_cutoff:
        strain = resample_to_delta_t(strain, 0.5 / high_freq_cutoff,
                                     method='ldas')
    else:
        strain = strain.copy()

    # taper strain
    corrupt_length = int(corrupt_time * strain.sample_rate)
    w = numpy.arange(corrupt_length) / float(corrupt_length)
    strain[0:corrupt_length] *= pycbc.types.Array(w, dtype=strain.dtype)
    strain[(len(strain) - corrupt_length):] *= \
        pycbc.types.Array(w[::-1], dtype=strain.dtype)

    if output_intermediates:
        strain.save_to_wav('strain_conditioned.wav')

    # zero-pad strain to a power-of-2 length
    strain_pad_length = next_power_of_2(len(strain))
    pad_start = int(strain_pad_length / 2 - len(strain) / 2)
    pad_end = pad_start + len(strain)
    pad_epoch = strain.start_time - pad_start / float(strain.sample_rate)
    strain_pad = pycbc.types.TimeSeries(
            pycbc.types.zeros(strain_pad_length, dtype=strain.dtype),
            delta_t=strain.delta_t, copy=False, epoch=pad_epoch)
    strain_pad[pad_start:pad_end] = strain[:]

    # estimate the PSD
    psd = pycbc.psd.welch(strain[corrupt_length:(len(strain)-corrupt_length)],
                          seg_len=int(psd_duration * strain.sample_rate),
                          seg_stride=int(psd_stride * strain.sample_rate),
                          avg_method=psd_avg_method,
                          require_exact_data_fit=False)
    psd = pycbc.psd.interpolate(psd, 1. / strain_pad.duration)
    psd = pycbc.psd.inverse_spectrum_truncation(
            psd, int(psd_duration * strain.sample_rate),
            low_frequency_cutoff=low_freq_cutoff,
            trunc_method='hann')
    kmin = int(low_freq_cutoff / psd.delta_f)
    psd[0:kmin] = numpy.inf
        kmax = int(high_freq_cutoff / psd.delta_f)
        psd[kmax:] = numpy.inf

    # whiten
    strain_tilde = strain_pad.to_frequencyseries()

        norm = high_freq_cutoff - low_freq_cutoff
    else:
        norm = strain.sample_rate / 2. - low_freq_cutoff
    strain_tilde *= (psd * norm) ** (-0.5)

    strain_pad = strain_tilde.to_timeseries()

        strain_pad[pad_start:pad_end].save_to_wav('strain_whitened.wav')

    mag = abs(strain_pad[pad_start:pad_end])

        mag.save('strain_whitened_mag.npy')

    mag = mag.numpy()

    # remove strain corrupted by filters at the ends
    mag[0:corrupt_length] = 0
    mag[-1:-corrupt_length-1:-1] = 0

    # find peaks and their times
    indices = numpy.where(mag > threshold)[0]
    cluster_idx = pycbc.events.findchirp_cluster_over_window(
            indices, numpy.array(mag[indices]),
            int(cluster_window*strain.sample_rate))
    times = [idx * strain.delta_t + strain.start_time \
             for idx in indices[cluster_idx]]

    return times

def from_cli(opt, dyn_range_fac=1, precision='single',
             inj_filter_rejector=None):
    """Parses the CLI options related to strain data reading and conditioning.

        required attributes  (gps-start-time, gps-end-time, strain-high-pass,
        pad-data, sample-rate, (frame-cache or frame-files), channel-name,
        fake-strain, fake-strain-seed, fake-strain-from-file, gating_file).
    dyn_range_fac : {float, 1}, optional
        A large constant to reduce the dynamic range of the strain.
    precision : string
        Precision of the returned strain ('single' or 'double').
    inj_filter_rejector : InjFilterRejector instance; optional, default=None
        If given send the InjFilterRejector instance to the inject module so
        that it can store a reduced representation of injections if
        necessary.

    Returns
    -------
        The time series containing the conditioned strain data.
    """
    gating_info = {}

    injector = InjectionSet.from_cli(opt)

    if opt.frame_cache or opt.frame_files or opt.frame_type or opt.hdf_store:
        if opt.frame_cache:
            frame_source = opt.frame_cache
        if opt.frame_files:
            frame_source = opt.frame_files

        logging.info("Reading Frames")

        if hasattr(opt, 'frame_sieve') and opt.frame_sieve:
            sieve = opt.frame_sieve
        else:
            sieve = None

        if opt.frame_type:
            strain = pycbc.frame.query_and_read_frame(
                    opt.frame_type, opt.channel_name,
                    start_time=opt.gps_start_time-opt.pad_data,
                    end_time=opt.gps_end_time+opt.pad_data,
                    sieve=sieve)
        elif opt.frame_files or opt.frame_cache:
            strain = pycbc.frame.read_frame(
                    frame_source, opt.channel_name,
        elif opt.hdf_store:
            strain = pycbc.frame.read_store(opt.hdf_store, opt.channel_name,
                                            opt.gps_start_time - opt.pad_data,
                                            opt.gps_end_time + opt.pad_data)

    elif opt.fake_strain or opt.fake_strain_from_file:
        logging.info("Generating Fake Strain")
        duration = opt.gps_end_time - opt.gps_start_time
        pdf = 1.0 / opt.fake_strain_filter_duration
        fake_flow = opt.fake_strain_flow
        fake_rate = opt.fake_strain_sample_rate
        plen = int(opt.sample_rate / pdf) // 2 + 1

        if opt.fake_strain_from_file:
            logging.info("Reading ASD from file")
            strain_psd = pycbc.psd.from_txt(opt.fake_strain_from_file,
                                            plen, pdf,
                                            fake_flow,
                                            is_asd_file=True)
        elif opt.fake_strain != 'zeroNoise':
            logging.info("Making PSD for strain")
            strain_psd = pycbc.psd.from_string(opt.fake_strain, plen, pdf,
                                               fake_flow)

        if opt.fake_strain == 'zeroNoise':
            logging.info("Making zero-noise time series")
            strain = TimeSeries(pycbc.types.zeros(duration * fake_rate),
                                delta_t=1.0 / fake_rate,
                                epoch=opt.gps_start_time)
        else:
            logging.info("Making colored noise")
            from pycbc.noise.reproduceable import colored_noise
            strain = colored_noise(strain_psd,
                                   opt.gps_start_time - opt.pad_data,
                                   opt.gps_end_time + opt.pad_data,
                                   seed=opt.fake_strain_seed,
                                   sample_rate=fake_rate,
                                   low_frequency_cutoff=fake_flow)
        if not strain.sample_rate_close(fake_rate):
            err_msg = "Actual sample rate of generated data does not match "
            err_msg += "that expected. Possible causes of this:\n"
            err_msg += "The desired duration is not a multiple of delta_t. "
            err_msg += "e.g. If using LISA with delta_t = 15 the duration "
            err_msg += "must be a multiple of 15 seconds."

    if not opt.channel_name and (opt.injection_file \
                                 or opt.sgburst_injection_file):
        raise ValueError('Please provide channel names with the format '
                         'ifo:channel (e.g. H1:CALIB-STRAIN) to inject '
                         'simulated signals into fake strain')

    if opt.zpk_z and opt.zpk_p and opt.zpk_k:
        logging.info("Highpass Filtering")
        strain = highpass(strain, frequency=opt.strain_high_pass)

        logging.info("Applying zpk filter")
        z = numpy.array(opt.zpk_z)
        p = numpy.array(opt.zpk_p)
        k = float(opt.zpk_k)
        strain = filter_zpk(strain.astype(numpy.float64), z, p, k)

    if opt.normalize_strain:
        logging.info("Dividing strain by constant")
        l = opt.normalize_strain
        strain = strain / l

    if opt.strain_high_pass:

    if opt.sample_rate:
        logging.info("Resampling data")
        strain = resample_to_delta_t(strain,
                                     1. / opt.sample_rate,

    if injector is not None:
        logging.info("Applying injections")
        injections = \
            injector.apply(strain, opt.channel_name[0:2],
                           distance_scale=opt.injection_scale_factor,
                           injection_sample_rate=opt.injection_sample_rate,
                           inj_filter_rejector=inj_filter_rejector)

    if opt.sgburst_injection_file:
        logging.info("Applying sine-Gaussian burst injections")
        injector = SGBurstInjectionSet(opt.sgburst_injection_file)
        injector.apply(strain, opt.channel_name[0:2],
                         distance_scale=opt.injection_scale_factor)

    if precision == 'single':
        logging.info("Converting to float32")
        strain = (strain * dyn_range_fac).astype(pycbc.types.float32)
    elif precision == "double":
        logging.info("Converting to float64")
        strain = (strain * dyn_range_fac).astype(pycbc.types.float64)
    else:
        raise ValueError("Unrecognized precision {}".format(precision))

    if opt.gating_file is not None:
        logging.info("Gating times contained in gating file")
        gate_params = numpy.loadtxt(opt.gating_file)
        if len(gate_params.shape) == 1:
            gate_params = [gate_params]
        strain = gate_data(strain, gate_params)
        gating_info['file'] = \
                [gp for gp in gate_params \
                 if (gp[0] + gp[1] + gp[2] >= strain.start_time) \
                 and (gp[0] - gp[1] - gp[2] <= strain.end_time)]

    if opt.autogating_threshold is not None:
        gating_info['auto'] = []
        for _ in range(opt.autogating_max_iterations):
            glitch_times = detect_loud_glitches(
                    strain, threshold=opt.autogating_threshold,
                    cluster_window=opt.autogating_cluster,
                    low_freq_cutoff=opt.strain_high_pass,
                    corrupt_time=opt.pad_data + opt.autogating_pad)
            gate_params = [[gt, opt.autogating_width, opt.autogating_taper]
                           for gt in glitch_times]
            gating_info['auto'] += gate_params
            strain = gate_data(strain, gate_params)
            if len(glitch_times) > 0:
                logging.info('Autogating at %s',
                             ', '.join(['%.3f' % gt
                                        for gt in glitch_times]))
            else:
                break

    if hasattr(opt, 'witness_frame_type') and opt.witness_frame_type:
        stilde = strain.to_frequencyseries()
        tf_file = h5py.File(opt.witness_tf_file)
        for key in tf_file:
            witness = pycbc.frame.query_and_read_frame(opt.witness_frame_type,
                   str(key),
                   start_time=strain.start_time,
                   end_time=strain.end_time)
            witness = (witness * dyn_range_fac).astype(strain.dtype)
            tf = pycbc.types.load_frequencyseries(opt.witness_tf_file,
                                                  group=key)
            tf = tf.astype(stilde.dtype)

            flen = int(opt.witness_filter_length * strain.sample_rate)
            tf = pycbc.psd.interpolate(tf, stilde.delta_f)

            tf_time = tf.to_timeseries()
            window = Array(numpy.hanning(flen * 2), dtype=strain.dtype)
            tf_time[0:flen] *= window[flen:]
            tf_time[len(tf_time)-flen:] *= window[0:flen]
            tf = tf_time.to_frequencyseries()

            kmax = min(len(tf), len(stilde) - 1)
            stilde[:kmax] -= tf[:kmax] * witness.to_frequencyseries()[:kmax]

        strain = stilde.to_timeseries()

    if opt.pad_data:
        logging.info("Remove Padding")
        start = int(opt.pad_data * strain.sample_rate)
        end = int(len(strain) - strain.sample_rate * opt.pad_data)
        strain = strain[start:end]

    if opt.taper_data:
        logging.info("Tapering data")
        # Use auto-gating, a one-sided gate is a taper
        pd_taper_window = opt.taper_data
        gate_params = [(strain.start_time, 0., pd_taper_window)]
        gate_params.append((strain.end_time, 0., pd_taper_window))
        gate_data(strain, gate_params)

        strain.injections = injections
    strain.gating_info = gating_info

    return strain

def from_cli_single_ifo(opt, ifo, inj_filter_rejector=None, **kwargs):
    """
    Get the strain for a single ifo when using the multi-detector CLI
    """
    return from_cli(single_det_opt,
        inj_filter_rejector=inj_filter_rejector, **kwargs)

def from_cli_multi_ifos(opt, ifos, inj_filter_rejector_dict=None, **kwargs):
    """
    Get the strain for all ifos when using the multi-detector CLI
    """
    strain = {}
    if inj_filter_rejector_dict is None:
        inj_filter_rejector_dict = {ifo: None for ifo in ifos}
        strain[ifo] = from_cli_single_ifo(opt, ifo,
                          inj_filter_rejector_dict[ifo], **kwargs)

def insert_strain_option_group(parser, gps_times=True):
    """ Add strain-related options to the optparser object.

    Adds the options used to call the pycbc.strain.from_cli function to an

    gps_times : bool, optional
        Include ``--gps-start-time`` and ``--gps-end-time`` options. Default
        is True.
    """

    data_reading_group = parser.add_argument_group("Options for obtaining h(t)",
                  "These options are used for generating h(t) either by "
                  "reading from a file or by generating it. This is only "
                  "needed if the PSD is to be estimated from the data, ie. "
                  " if the --psd-estimation option is given.")

    # Required options
    if gps_times:
        data_reading_group.add_argument("--gps-start-time",
                                help="The gps start time of the data "
                                     "(integer seconds)", type=int)
        data_reading_group.add_argument("--gps-end-time",
                                help="The gps end time of the data "

    data_reading_group.add_argument("--strain-high-pass", type=float,
              help="High pass frequency")
    data_reading_group.add_argument("--pad-data", default=8,
              help="Extra padding to remove highpass corruption "
                   "(integer seconds)", type=int)
    data_reading_group.add_argument("--taper-data",
              help="Taper ends of data to zero using the supplied length as a "
                   "window (integer seconds)", type=int, default=0)
    data_reading_group.add_argument("--sample-rate", type=float,
              help="The sample rate to use for h(t) generation (integer Hz)")
    data_reading_group.add_argument("--channel-name", type=str,
                   help="The channel containing the gravitational strain data")

    # Read from cache file
    data_reading_group.add_argument("--frame-cache", type=str, nargs="+",
                            help="Cache file containing the frame locations.")
    # Read from frame files
    data_reading_group.add_argument("--frame-files",
                            type=str, nargs="+",
                            help="list of frame files")
    # Read from hdf store file
    data_reading_group.add_argument("--hdf-store",
                            type=str,
                            help="Store of time series data in hdf format")
    # Use datafind to get frame files
    data_reading_group.add_argument("--frame-type",
                            type=str,
                            help="(optional), replaces frame-files. Use datafind "
                                 "to get the needed frame file(s) of this type.")
    # Filter frame files by URL
    data_reading_group.add_argument("--frame-sieve",
                            type=str,
                            help="(optional), Only use frame files where the "
                                 "URL matches the regular expression given.")

    # Generate gaussian noise with given psd
    data_reading_group.add_argument("--fake-strain",
                help="Name of model PSD for generating fake gaussian noise.",
                choices=pycbc.psd.get_lalsim_psd_list() + ['zeroNoise'])
    data_reading_group.add_argument("--fake-strain-seed", type=int, default=0,
                help="Seed value for the generation of fake colored"
                     " gaussian noise")
    data_reading_group.add_argument("--fake-strain-from-file",
                help="File containing ASD for generating fake noise from it.")
    data_reading_group.add_argument("--fake-strain-flow",
                default=1.0, type=float,
                help="Low frequency cutoff of the fake strain")
    data_reading_group.add_argument("--fake-strain-filter-duration",
                default=128.0, type=float,
                help="Duration in seconds of the fake data coloring filter")
    data_reading_group.add_argument("--fake-strain-sample-rate",
                default=16384, type=float,
                help="Sample rate of the fake data generation")

    # Injection options
    data_reading_group.add_argument("--injection-file", type=str,
                      help="(optional) Injection file containing parameters"
                           " of CBC signals to be added to the strain")
    data_reading_group.add_argument("--sgburst-injection-file", type=str,
                      "of sine-Gaussian burst signals to add to the strain")
    data_reading_group.add_argument("--injection-scale-factor", type=float,
                      default=1,
                      help="Divide injections by this factor "
                           "before adding to the strain data")
    data_reading_group.add_argument("--injection-sample-rate", type=float,
                      help="Sample rate to use for injections (integer Hz). "
                           "Typically similar to the strain data sample rate."
                           "If not provided, the strain sample rate will be "
                           "used")
    data_reading_group.add_argument("--injection-f-ref", type=float,
                      help="Reference frequency in Hz for creating CBC "
                           "injections from an XML file")
    data_reading_group.add_argument("--injection-f-final", type=float,
                      help="Override the f_final field of a CBC XML "
                           "injection file (frequency in Hz)")

    # Gating options
    data_reading_group.add_argument("--gating-file", type=str,
                    help="(optional) Text file of gating segments to apply."
                         " Format of each line is (all values in seconds):"
                         "  gps_time zeros_half_width pad_half_width")
    data_reading_group.add_argument('--autogating-threshold', type=float,
                                    metavar='SIGMA',
                                    help='If given, find and gate glitches '
                                         'producing a deviation larger than '
                                         'SIGMA in the whitened strain time '
                                         'series.')
    data_reading_group.add_argument('--autogating-max-iterations', type=int,
                                    metavar='SIGMA', default=1,
                                    help='If given, iteratively apply '
                                         'autogating')
    data_reading_group.add_argument('--autogating-cluster', type=float,
                                    metavar='SECONDS', default=5.,
                                    help='Length of clustering window for '
                                         'detecting glitches for autogating.')
    data_reading_group.add_argument('--autogating-width', type=float,
                                    metavar='SECONDS', default=0.25,
                                    help='Half-width of the gating window.')
    data_reading_group.add_argument('--autogating-taper', type=float,
                                    help='Taper the strain before and after '
                                         'each gating window over a duration '
                                         'of SECONDS.')
    data_reading_group.add_argument('--autogating-pad', type=float,
                                    metavar='SECONDS', default=16,
                                    help='Ignore the given length of whitened '
                                         'strain at the ends of a segment, to '
                                         'avoid filters ringing.')
    # Optional
    data_reading_group.add_argument("--normalize-strain", type=float,
                    help="(optional) Divide frame data by constant.")
    data_reading_group.add_argument("--zpk-z", type=float, nargs="+",
                    help="(optional) Zero-pole-gain (zpk) filter strain. "
                         "A list of zeros for transfer function")
    data_reading_group.add_argument("--zpk-p", type=float, nargs="+",
                         "A list of poles for transfer function")
    data_reading_group.add_argument("--zpk-k", type=float,
                         "Transfer function gain")

    # Options to apply to subtract noise from a witness channel and known
    # transfer function.
    data_reading_group.add_argument("--witness-frame-type", type=str,
                    help="(optional), frame type which will be use to query the"
                         " witness channel data.")
    data_reading_group.add_argument("--witness-tf-file", type=str,
                    help="an hdf file containing the transfer"
                         " functions and the associated channel names")
    data_reading_group.add_argument("--witness-filter-length", type=float,
                    help="filter length in seconds for the transfer function")

    return data_reading_group

# FIXME: This repeats almost all of the options above. Any nice way of reducing
#        this?
def insert_strain_option_group_multi_ifo(parser, gps_times=True):
    """

        is True.
    """

    data_reading_group_multi = parser.add_argument_group("Options for obtaining"
                  " h(t)",
                  "if the --psd-estimation option is given. This group "
                  "supports reading from multiple ifos simultaneously.")

        data_reading_group_multi.add_argument(
            "--gps-start-time", nargs='+', action=MultiDetOptionAction,
            metavar='IFO:TIME', type=int,
            help="The gps start time of the data (integer seconds)")
            "--gps-end-time", nargs='+', action=MultiDetOptionAction,
            help="The gps end time of the data (integer seconds)")

    data_reading_group_multi.add_argument("--strain-high-pass", nargs='+',
                            type=float, metavar='IFO:FREQUENCY',
                            help="High pass frequency")
    data_reading_group_multi.add_argument("--pad-data", nargs='+', default=8,
                            type=int, metavar='IFO:LENGTH',
                            help="Extra padding to remove highpass corruption "
                                "(integer seconds)")
    data_reading_group_multi.add_argument("--taper-data", nargs='+',
                            type=int, default=0, metavar='IFO:LENGTH',
                            help="Taper ends of data to zero using the "
                                "supplied length as a window (integer seconds)")
    data_reading_group_multi.add_argument("--sample-rate", type=float,
                            nargs='+',
                            action=MultiDetOptionAction, metavar='IFO:RATE',
                            help="The sample rate to use for h(t) generation "
                                " (integer Hz).")
    data_reading_group_multi.add_argument("--channel-name", type=str, nargs='+',
                            action=MultiDetOptionActionSpecial,
                            metavar='IFO:CHANNEL',
                            help="The channel containing the gravitational "
                                "strain data")

    data_reading_group_multi.add_argument("--frame-cache", type=str, nargs="+",
                            action=MultiDetOptionAppendAction,
                            metavar='IFO:FRAME_CACHE',
    data_reading_group_multi.add_argument("--frame-files", type=str, nargs="+",
                            metavar='IFO:FRAME_FILES',
    data_reading_group_multi.add_argument("--hdf-store", type=str, nargs='+',
                            metavar='IFO:HDF_STORE_FILE',
    data_reading_group_multi.add_argument("--frame-type", type=str, nargs="+",
                                    action=MultiDetOptionAction,
                                    metavar='IFO:FRAME_TYPE',
                                    help="(optional) Replaces frame-files. "
                                         "Use datafind to get the needed frame "
                                         "file(s) of this type.")
    data_reading_group_multi.add_argument("--frame-sieve", type=str, nargs="+",
                            metavar='IFO:FRAME_SIEVE',
    data_reading_group_multi.add_argument("--fake-strain", type=str, nargs="+",
                            action=MultiDetOptionAction, metavar='IFO:CHOICE',
                            help="Name of model PSD for generating fake "
                            "gaussian noise. Choose from %s or zeroNoise" \
                            %((', ').join(pycbc.psd.get_lalsim_psd_list()),) )
    data_reading_group_multi.add_argument("--fake-strain-seed", type=int,
                            default=0, nargs="+", action=MultiDetOptionAction,
                            metavar='IFO:SEED',
                            help="Seed value for the generation of fake "
                            "colored gaussian noise")
    data_reading_group_multi.add_argument("--fake-strain-from-file", nargs="+",
                            action=MultiDetOptionAction, metavar='IFO:FILE',
                            help="File containing ASD for generating fake "
                            "noise from it.")
    data_reading_group_multi.add_argument("--fake-strain-flow",
                nargs="+", action=MultiDetOptionAction,
    data_reading_group_multi.add_argument("--fake-strain-filter-duration",
    data_reading_group_multi.add_argument("--fake-strain-sample-rate",

    data_reading_group_multi.add_argument("--injection-file", type=str,
                    nargs="+", action=MultiDetOptionAction,
                    metavar='IFO:FILE',
                    help="(optional) Injection file containing parameters"
                         "of CBC signals to be added to the strain")
    data_reading_group_multi.add_argument("--sgburst-injection-file", type=str,
                         "of sine-Gaussian burst signals to add to the strain")
    data_reading_group_multi.add_argument("--injection-scale-factor",
                    type=float, nargs="+", action=MultiDetOptionAction,
                    metavar="IFO:VAL", default=1.,
                    help="Divide injections by this factor "
                         "before adding to the strain data")
    data_reading_group_multi.add_argument("--injection-sample-rate",
                    metavar="IFO:VAL",
                    help="Sample rate to use for injections (integer Hz). "
                         "Typically similar to the strain data sample rate."
                         "If not provided, the strain sample rate will be "
                         "used")

    data_reading_group_multi.add_argument("--injection-f-ref", type=float,
                    action=MultiDetOptionAction, metavar='IFO:VALUE',
                    help="Reference frequency in Hz for creating CBC "
                         "injections from an XML file")
    data_reading_group_multi.add_argument('--injection-f-final', type=float,
                    help="Override the f_final field of a CBC XML "
                         "injection file (frequency in Hz)")

    data_reading_group_multi.add_argument("--gating-file", nargs="+",
                      action=MultiDetOptionAction,
                      metavar='IFO:FILE',
                      help='(optional) Text file of gating segments to apply.'
                           ' Format of each line (units s) :'
                           ' gps_time zeros_half_width pad_half_width')
    data_reading_group_multi.add_argument('--autogating-threshold', type=float,
                      nargs="+", action=MultiDetOptionAction,
                      metavar='IFO:SIGMA',
                      help='If given, find and gate glitches producing a '
                           'deviation larger than SIGMA in the whitened strain'
                           ' time series')
    data_reading_group_multi.add_argument('--autogating-max-iterations', type=int,
    data_reading_group_multi.add_argument('--autogating-cluster', type=float,
                                    nargs="+", action=MultiDetOptionAction,
                                    metavar='IFO:SECONDS', default=5.,
    data_reading_group_multi.add_argument('--autogating-width', type=float,
                                    metavar='IFO:SECONDS', default=0.25,
    data_reading_group_multi.add_argument('--autogating-taper', type=float,
    data_reading_group_multi.add_argument('--autogating-pad', type=float,
                                    metavar='IFO:SECONDS', default=16,

    # Optional
    data_reading_group_multi.add_argument("--normalize-strain", type=float,
                     nargs="+", action=MultiDetOptionAction,
                     metavar='IFO:VALUE',
                     help="(optional) Divide frame data by constant.")
    data_reading_group_multi.add_argument("--zpk-z", type=float,
                     nargs="+", action=MultiDetOptionAppendAction,
                     help="(optional) Zero-pole-gain (zpk) filter strain. "
                          "A list of zeros for transfer function")
    data_reading_group_multi.add_argument("--zpk-p", type=float,
                          "A list of poles for transfer function")
    data_reading_group_multi.add_argument("--zpk-k", type=float,
                          "Transfer function gain")

    return data_reading_group_multi

ensure_one_opt_groups.append(['--frame-cache','--fake-strain',
                              '--fake-strain-from-file',
                              '--frame-files', '--frame-type',
                              '--hdf-store'])

required_opts_list = ['--gps-start-time', '--gps-end-time',
                      '--strain-high-pass', '--pad-data', '--sample-rate',
                      '--channel-name']

def verify_strain_options(opts, parser):
    """Sanity check provided strain arguments.

    Parses the strain data CLI options and verifies that they are consistent
    and reasonable.

        required attributes (gps-start-time, gps-end-time, strain-high-pass,
        pad-data, sample-rate, frame-cache, channel-name, fake-strain,
        fake-strain-seed).
    """
        ensure_one_opt(opts, parser, opt_group)
    required_opts(opts, parser, required_opts_list)

def verify_strain_options_multi_ifo(opts, parser, ifos):

    ifos : list of strings
        List of ifos for which to verify options for
    """
            ensure_one_opt_multi_ifo(opts, parser, ifo, opt_group)
        required_opts_multi_ifo(opts, parser, ifo, required_opts_list)

def gate_data(data, gate_params):
    """Apply a set of gating windows to a time series.

    Each gating window is
    defined by a central time, a given duration (centered on the given
    time) to zero out, and a given duration of smooth tapering on each side of
    the window. The window function used for tapering is a Tukey window.

        The time series to be gated.
    gate_params : list
        List of parameters for the gating windows. Each element should be a
        list or tuple with 3 elements: the central time of the gating window,
        the half-duration of the portion to zero out, and the duration of the
        Tukey tapering on each side. All times in seconds. The total duration
        of the data affected by one gating window is thus twice the second
        parameter plus twice the third parameter.

    Returns
    -------
    data: TimeSeries
        The gated time series.
    """
    def inverted_tukey(M, n_pad):
        midlen = M - 2*n_pad
        if midlen < 0:
            raise ValueError("No zeros left after applying padding.")
        padarr = 0.5*(1.+numpy.cos(numpy.pi*numpy.arange(n_pad)/n_pad))
        return numpy.concatenate((padarr,numpy.zeros(midlen),padarr[::-1]))

    sample_rate = 1./data.delta_t
    temp = data.data

    for glitch_time, glitch_width, pad_width in gate_params:
        t_start = glitch_time - glitch_width - pad_width - data.start_time
        t_end = glitch_time + glitch_width + pad_width - data.start_time
        if t_start > data.duration or t_end < 0.:
            continue # Skip gate segments that don't overlap
        win_samples = int(2*sample_rate*(glitch_width+pad_width))
        pad_samples = int(sample_rate*pad_width)
        window = inverted_tukey(win_samples, pad_samples)
        offset = int(t_start * sample_rate)
        idx1 = max(0, -offset)
        idx2 = min(len(window), len(data)-offset)
        temp[idx1+offset:idx2+offset] *= window[idx1:idx2]

class StrainSegments(object):
    """ Class for managing manipulation of strain data for the purpose of
        matched filtering. This includes methods for segmenting and
        conditioning.
    """
    def __init__(self, strain, segment_length=None, segment_start_pad=0,
                 segment_end_pad=0, trigger_start=None, trigger_end=None,
                 filter_inj_only=False, injection_window=None,
                 allow_zero_padding=False):
        """ Determine how to chop up the strain data into smaller segments
            for analysis.
        """
        self._fourier_segments = None
        self.strain = strain

        self.delta_t = strain.delta_t
        self.sample_rate = strain.sample_rate

        if segment_length:
            seg_len = segment_length
        else:
            seg_len = strain.duration

        self.delta_f = 1.0 / seg_len
        self.time_len = int(seg_len * self.sample_rate)
        self.freq_len = self.time_len // 2 + 1

        seg_end_pad = segment_end_pad
        seg_start_pad = segment_start_pad

        if not trigger_start:
            trigger_start = int(strain.start_time) + segment_start_pad
        else:
            if not allow_zero_padding:
                min_start_time = int(strain.start_time) + segment_start_pad
            else:
                min_start_time = int(strain.start_time)
            if trigger_start < min_start_time:
                err_msg = "Trigger start time must be within analysable "
                err_msg += "window. Asked to start from %d " %(trigger_start)
                err_msg += "but can only analyse from %d." %(min_start_time)

        if not trigger_end:
            trigger_end = int(strain.end_time) - segment_end_pad
        else:
                max_end_time = int(strain.end_time) - segment_end_pad
            else:
                max_end_time = int(strain.end_time)
            if trigger_end > max_end_time:
                err_msg = "Trigger end time must be within analysable "
                err_msg += "window. Asked to end at %d " %(trigger_end)
                err_msg += "but can only analyse to %d." %(max_end_time)

        throwaway_size = seg_start_pad + seg_end_pad
        seg_width = seg_len - throwaway_size

        # The amount of time we can actually analyze given the
        # amount of padding that is needed
        analyzable = trigger_end - trigger_start
        data_start = (trigger_start - segment_start_pad) - \
                       int(strain.start_time)
        data_end = trigger_end + segment_end_pad - int(strain.start_time)
        data_dur = data_end - data_start
        data_start = data_start * strain.sample_rate
        data_end = data_end * strain.sample_rate

        #number of segments we need to analyze this data
        num_segs = int(numpy.ceil(float(analyzable) / float(seg_width)))

        # The offset we will use between segments
        seg_offset = int(numpy.ceil(analyzable / float(num_segs)))
        self.segment_slices = []
        self.analyze_slices = []

        # Determine how to chop up the strain into smaller segments
        for nseg in range(num_segs-1):
            # boundaries for time slices into the strain
            seg_start = int(data_start + (nseg*seg_offset) * strain.sample_rate)
            seg_end = int(seg_start + seg_len * strain.sample_rate)
            seg_slice = slice(seg_start, seg_end)
            self.segment_slices.append(seg_slice)

            # boundaries for the analyzable portion of the segment
            ana_start = int(seg_start_pad * strain.sample_rate)
            ana_end = int(ana_start + seg_offset * strain.sample_rate)
            ana_slice = slice(ana_start, ana_end)
            self.analyze_slices.append(ana_slice)

        # The last segment takes up any integer boundary slop
        seg_end = int(data_end)
        seg_start = int(seg_end - seg_len * strain.sample_rate)
        seg_slice = slice(seg_start, seg_end)
        self.segment_slices.append(seg_slice)

        remaining = (data_dur - ((num_segs - 1) * seg_offset + seg_start_pad))
        ana_start = int((seg_len - remaining) * strain.sample_rate)
        ana_end = int((seg_len - seg_end_pad) * strain.sample_rate)
        ana_slice = slice(ana_start, ana_end)
        self.analyze_slices.append(ana_slice)

        self.full_segment_slices = copy.deepcopy(self.segment_slices)

        #Remove segments that are outside trig start and end
        segment_slices_red = []
        analyze_slices_red = []
        trig_start_idx = (trigger_start - int(strain.start_time)) * strain.sample_rate
        trig_end_idx = (trigger_end - int(strain.start_time)) * strain.sample_rate

        if filter_inj_only and hasattr(strain, 'injections'):
            end_times = strain.injections.end_times()
            end_times = [time for time in end_times if float(time) < trigger_end and float(time) > trigger_start]
            inj_idx = [(float(time) - float(strain.start_time)) * strain.sample_rate for time in end_times]

        for seg, ana in zip(self.segment_slices, self.analyze_slices):
            start = ana.start
            stop = ana.stop
            cum_start = start + seg.start
            cum_end = stop + seg.start

            # adjust first segment
            if trig_start_idx > cum_start:
                start += (trig_start_idx - cum_start)

            # adjust last segment
            if trig_end_idx < cum_end:
                stop -= (cum_end - trig_end_idx)

            if filter_inj_only and hasattr(strain, 'injections'):
                analyze_this = False
                inj_window = strain.sample_rate * 8
                for inj_id in inj_idx:
                    if inj_id < (cum_end + inj_window) and \
                            inj_id > (cum_start - inj_window):
                        analyze_this = True

                if not analyze_this:
                    continue

            if start < stop:
                segment_slices_red.append(seg)
                analyze_slices_red.append(slice(start, stop))

        self.segment_slices = segment_slices_red
        self.analyze_slices = analyze_slices_red

    def fourier_segments(self):
        """ Return a list of the FFT'd segments.
        Return the list of FrequencySeries. Additional properties are
        added that describe the strain segment. The property 'analyze'
        is a slice corresponding to the portion of the time domain equivalent
        of the segment to analyze for triggers. The value 'cumulative_index'
        indexes from the beginning of the original strain series.
        """
        if not self._fourier_segments:
            self._fourier_segments = []
            for seg_slice, ana in zip(self.segment_slices, self.analyze_slices):
                if seg_slice.start >= 0 and seg_slice.stop <= len(self.strain):
                    freq_seg = make_frequency_series(self.strain[seg_slice])
                # Assume that we cannot have a case where we both zero-pad on
                # both sides
                elif seg_slice.start < 0:
                    strain_chunk = self.strain[:seg_slice.stop]
                    strain_chunk.prepend_zeros(-seg_slice.start)
                    freq_seg = make_frequency_series(strain_chunk)
                elif seg_slice.stop > len(self.strain):
                    strain_chunk = self.strain[seg_slice.start:]
                    strain_chunk.append_zeros(seg_slice.stop - len(self.strain))
                freq_seg.analyze = ana
                freq_seg.cumulative_index = seg_slice.start + ana.start
                freq_seg.seg_slice = seg_slice
                self._fourier_segments.append(freq_seg)

        return self._fourier_segments

    def from_cli(cls, opt, strain):
        """Calculate the segmentation of the strain data for analysis from
        the command line options.
        """
        return cls(strain, segment_length=opt.segment_length,
                   segment_start_pad=opt.segment_start_pad,
                   segment_end_pad=opt.segment_end_pad,
                   trigger_start=opt.trig_start_time,
                   trigger_end=opt.trig_end_time,
                   filter_inj_only=opt.filter_inj_only,
                   injection_window=opt.injection_window,
                   allow_zero_padding=opt.allow_zero_padding)

    def insert_segment_option_group(cls, parser):
        segment_group = parser.add_argument_group(
                                  "Options for segmenting the strain",
                                  "These options are used to determine how to "
                                  "segment the strain into smaller chunks, "
                                  "and for determining the portion of each to "
                                  "analyze for triggers. ")
        segment_group.add_argument("--trig-start-time", type=int, default=0,
                    help="(optional) The gps time to start recording triggers")
        segment_group.add_argument("--trig-end-time", type=int, default=0,
                    help="(optional) The gps time to stop recording triggers")
        segment_group.add_argument("--segment-length", type=int,
                          help="The length of each strain segment in seconds.")
        segment_group.add_argument("--segment-start-pad", type=int,
                          help="The time in seconds to ignore of the "
                               "beginning of each segment in seconds. ")
        segment_group.add_argument("--segment-end-pad", type=int,
                          help="The time in seconds to ignore at the "
                               "end of each segment in seconds.")
        segment_group.add_argument("--allow-zero-padding", action='store_true',
                                   help="Allow for zero padding of data to "
                                        "analyze requested times, if needed.")
        # Injection optimization options
        segment_group.add_argument("--filter-inj-only", action='store_true',
                          help="Analyze only segments that contain an injection.")
        segment_group.add_argument("--injection-window", default=None,
                          type=float, help="""If using --filter-inj-only then
                          only search for injections within +/- injection
                          window of the injections's end time. This is useful
                          to speed up a coherent search or a search where we
                          initially filter at lower sample rate, and then
                          filter at full rate where needed. NOTE: Reverts to
                          full analysis if two injections are in the same
                          segment.""")

    def from_cli_single_ifo(cls, opt, strain, ifo):
        """
        return cls(strain, segment_length=opt.segment_length[ifo],
                   segment_start_pad=opt.segment_start_pad[ifo],
                   segment_end_pad=opt.segment_end_pad[ifo],
                   trigger_start=opt.trig_start_time[ifo],
                   trigger_end=opt.trig_end_time[ifo],

    def from_cli_multi_ifos(cls, opt, strain_dict, ifos):
        """
        strain_segments = {}
            strain_segments[ifo] = cls.from_cli_single_ifo(
                opt, strain_dict[ifo], ifo)
        return strain_segments

    def insert_segment_option_group_multi_ifo(cls, parser):
                    nargs='+', action=MultiDetOptionAction, metavar='IFO:TIME',
                    nargs='+', action=MultiDetOptionAction,
                    metavar='IFO:LENGTH',
                    help="The length of each strain segment in seconds.")
                    help="The time in seconds to ignore of the "
                        "beginning of each segment in seconds. ")
                    help="The time in seconds to ignore at the "
                         "end of each segment in seconds.")
                          help="Allow for zero padding of data to analyze "
                          "requested times, if needed.")
                                   help="Analyze only segments that contain "
                                        "an injection.")

    required_opts_list = ['--segment-length',
                   '--segment-start-pad',
                   '--segment-end-pad',
                   ]

    def verify_segment_options(cls, opt, parser):
        required_opts(opt, parser, cls.required_opts_list)

    def verify_segment_options_multi_ifo(cls, opt, parser, ifos):
            required_opts_multi_ifo(opt, parser, ifo, cls.required_opts_list)

class StrainBuffer(pycbc.frame.DataBuffer):
    def __init__(self, frame_src, channel_name, start_time,
                 max_buffer=512,
                 sample_rate=4096,
                 low_frequency_cutoff=20,
                 highpass_frequency=15.0,
                 highpass_reduction=200.0,
                 highpass_bandwidth=5.0,
                 psd_samples=30,
                 psd_segment_length=4,
                 psd_inverse_length=3.5,
                 trim_padding=0.25,
                 autogating_threshold=None,
                 autogating_cluster=None,
                 autogating_pad=None,
                 autogating_width=None,
                 autogating_taper=None,
                 state_channel=None,
                 data_quality_channel=None,
                 dyn_range_fac=pycbc.DYN_RANGE_FAC,
                 psd_abort_difference=None,
                 psd_recalculate_difference=None,
                 analyze_flags=None,
                 data_quality_flags=None,
                 dq_padding=0):
        """ Class to produce overwhitened strain incrementally

        max_buffer: {int, 512}, Optional
        sample_rate: {int, 2048}, Optional
            Rate in Hz to sample the data.
        low_frequency_cutoff: {float, 20}, Optional
            The low frequency cutoff to use for inverse spectrum truncation
        highpass_frequency: {float, 15}, Optional
            The frequency to apply a highpass filter at before downsampling.
        highpass_reduction: {float, 200}, Optional
            The amount of reduction to apply to the low frequencies.
        highpass_bandwidth: {float, 5}, Optional
            The width of the transition region for the highpass filter.
        psd_samples: {int, 30}, Optional
            The number of sample to use for psd estimation
        psd_segment_length: {float, 4}, Optional
            The number of seconds in each psd sample.
        psd_inverse_length: {float, 3.5}, Optional
            The length in seconds for fourier transform of the inverse of the
            PSD to be truncated to.
        trim_padding: {float, 0.25}, Optional
            Amount of padding in seconds to give for truncated the overwhitened
            data stream.
        autogating_threshold: float, Optional
            Sigma deviation required to cause autogating of data.
            If None, no autogating is performed.
        autogating_cluster: float, Optional
            Seconds to cluster possible gating locations.
        autogating_pad: float, Optional
            Seconds of corrupted whitened strain to ignore when generating a gate.
        autogating_width: float, Optional
            Half-duration of the zeroed-out portion of autogates.
        autogating_taper: float, Optional
            Duration of taper on either side of the gating window in seconds.
        state_channel: {str, None}, Optional
            Channel to use for state information about the strain
        data_quality_channel: {str, None}, Optional
            Channel to use for data quality information about the strain
        dyn_range_fac: {float, pycbc.DYN_RANGE_FAC}, Optional
            Scale factor to apply to strain
        psd_abort_difference: {float, None}, Optional
            The relative change in the inspiral range from the previous PSD
            estimate to trigger the data to be considered invalid.
        psd_recalculate_difference: {float, None}, Optional
            the relative change in the inspiral range from the previous PSD
            to trigger a re-estimatoin of the PSD.
        force_update_cache: {boolean, True}, Optional
            Re-check the filesystem for frame files on every attempt to
            read more data.
        analyze_flags: list of strs
            The flags that must be on to mark the current data as valid for
            *any* use.
        data_quality_flags: list of strs
            The flags used to determine if to keep triggers.
        dq_padding: {float, 0}, optional
            Extra seconds to consider invalid before/after times with bad DQ.
        increment_update_cache: {str, None}, Optional
            Pattern to look for frame files in a GPS dependent directory. This
            is an alternate to the forced updated of the frame cache, and
            apptempts to predict the next frame file name without probing the
            filesystem.
        """
        super(StrainBuffer, self).__init__(frame_src, channel_name, start_time,
                                           max_buffer=32,
                                           force_update_cache=force_update_cache,
                                           increment_update_cache=increment_update_cache)

        self.low_frequency_cutoff = low_frequency_cutoff

        # Set up status buffers
        self.analyze_flags = analyze_flags
        self.data_quality_flags = data_quality_flags
        self.state = None
        self.dq = None
        self.dq_padding = dq_padding

        # State channel
        if state_channel is not None:
            valid_mask = pycbc.frame.flag_names_to_bitmask(self.analyze_flags)
            logging.info('State channel %s interpreted as bitmask %s = good',
                         state_channel, bin(valid_mask))
            self.state = pycbc.frame.StatusBuffer(
                frame_src,
                state_channel, start_time,
                max_buffer=max_buffer,
                valid_mask=valid_mask,
                force_update_cache=force_update_cache,
                increment_update_cache=increment_update_cache)

        # low latency dq channel
        if data_quality_channel is not None:
            sb_kwargs = dict(max_buffer=max_buffer,
                             force_update_cache=force_update_cache,
                             increment_update_cache=increment_update_cache)
            if len(self.data_quality_flags) == 1 \
                    and self.data_quality_flags[0] == 'veto_nonzero':
                sb_kwargs['valid_on_zero'] = True
                logging.info('DQ channel %s interpreted as zero = good',
                             data_quality_channel)
            else:
                sb_kwargs['valid_mask'] = pycbc.frame.flag_names_to_bitmask(
                        self.data_quality_flags)
                logging.info('DQ channel %s interpreted as bitmask %s = good',
                             data_quality_channel, bin(valid_mask))
            self.dq = pycbc.frame.StatusBuffer(frame_src, data_quality_channel,
                                               start_time, **sb_kwargs)

        self.highpass_frequency = highpass_frequency
        self.highpass_reduction = highpass_reduction
        self.highpass_bandwidth = highpass_bandwidth

        self.autogating_threshold = autogating_threshold
        self.autogating_cluster = autogating_cluster
        self.autogating_pad = autogating_pad
        self.autogating_width = autogating_width
        self.autogating_taper = autogating_taper
        self.gate_params = []

        self.sample_rate = sample_rate
        self.dyn_range_fac = dyn_range_fac

        self.psd_abort_difference = psd_abort_difference
        self.psd_recalculate_difference = psd_recalculate_difference
        self.psd_segment_length = psd_segment_length
        self.psd_samples = psd_samples
        self.psd_inverse_length = psd_inverse_length
        self.psd = None
        self.psds = {}

        strain_len = int(max_buffer * self.sample_rate)
        self.strain = TimeSeries(zeros(strain_len, dtype=numpy.float32),
                                 delta_t=1.0/self.sample_rate,
                                 epoch=start_time-max_buffer)

        # Determine the total number of corrupted samples for highpass
        # and PSD over whitening
        highpass_samples, self.beta = kaiserord(self.highpass_reduction,
          self.highpass_bandwidth / self.raw_buffer.sample_rate * 2 * numpy.pi)
        self.highpass_samples =  int(highpass_samples / 2)
        resample_corruption = 10 # If using the ldas method
        self.factor = round(1.0 / self.raw_buffer.delta_t / self.sample_rate)
        self.corruption = self.highpass_samples // self.factor + resample_corruption

        self.psd_corruption =  self.psd_inverse_length * self.sample_rate
        self.total_corruption = self.corruption + self.psd_corruption

        # Determine how much padding is needed after removing the parts
        # associated with PSD over whitening and highpass filtering
        self.trim_padding = int(trim_padding * self.sample_rate)
        if self.trim_padding > self.total_corruption:
            self.trim_padding = self.total_corruption

        self.psd_duration = (psd_samples - 1) // 2 * psd_segment_length

        self.reduced_pad = int(self.total_corruption - self.trim_padding)
        self.segments = {}

        # time to ignore output of frame (for initial buffering)
        self.add_hard_count()
        self.taper_immediate_strain = True

    @property
    def start_time(self):
        """ Return the start time of the current valid segment of data """
        return self.end_time - self.blocksize

    @property
        """ Return the end time of the current valid segment of data """
        return float(self.strain.start_time + (len(self.strain) - self.total_corruption) / self.sample_rate)

    def add_hard_count(self):
        """ Reset the countdown timer, so that we don't analyze data long enough
        to generate a new PSD.
        """
        self.wait_duration = int(numpy.ceil(self.total_corruption / self.sample_rate +  self.psd_duration))
        self.invalidate_psd()

    def invalidate_psd(self):
        """ Make the current PSD invalid. A new one will be generated when
        it is next required """

    def recalculate_psd(self):
        """ Recalculate the psd
        """

        seg_len = int(self.sample_rate * self.psd_segment_length)
        e = len(self.strain)
        s = e - (self.psd_samples + 1) * seg_len // 2
        psd = pycbc.psd.welch(self.strain[s:e], seg_len=seg_len, seg_stride=seg_len//2)

        psd.dist = spa_distance(psd, 1.4, 1.4, self.low_frequency_cutoff) * pycbc.DYN_RANGE_FAC

        # If the new psd is similar to the old one, don't replace it
        if self.psd and self.psd_recalculate_difference:
            if abs(self.psd.dist - psd.dist) / self.psd.dist < self.psd_recalculate_difference:
                logging.info("Skipping recalculation of %s PSD, %s-%s",
                             self.detector, self.psd.dist, psd.dist)

        # If the new psd is *really* different than the old one, return an error
        if self.psd and self.psd_abort_difference:
            if abs(self.psd.dist - psd.dist) / self.psd.dist > self.psd_abort_difference:
                logging.info("%s PSD is CRAZY, aborting!!!!, %s-%s",
                self.psd = psd
                self.psds = {}

        # If the new estimate replaces the current one, invalide the ineterpolate PSDs
        self.psd = psd
        logging.info("Recalculating %s PSD, %s", self.detector, psd.dist)

    def overwhitened_data(self, delta_f):
        """ Return overwhitened data

        delta_f: float
            The sample step to generate overwhitened frequency domain data for

        Returns
        -------
        htilde: FrequencySeries
            Overwhited strain data
        """
        # we haven't already computed htilde for this delta_f
        if delta_f not in self.segments:
            buffer_length = int(1.0 / delta_f)
            e = len(self.strain)
            s = int(e - buffer_length * self.sample_rate - self.reduced_pad * 2)
            fseries = make_frequency_series(self.strain[s:e])

            # we haven't calculated a resample psd for this delta_f
            if delta_f not in self.psds:
                psdt = pycbc.psd.interpolate(self.psd, fseries.delta_f)
                psdt = pycbc.psd.inverse_spectrum_truncation(psdt,
                                       int(self.sample_rate * self.psd_inverse_length),
                                       low_frequency_cutoff=self.low_frequency_cutoff)
                psdt._delta_f = fseries.delta_f

                psd = pycbc.psd.interpolate(self.psd, delta_f)
                psd = pycbc.psd.inverse_spectrum_truncation(psd,

                psd.psdt = psdt
                self.psds[delta_f] = psd

            psd = self.psds[delta_f]
            fseries /= psd.psdt

            # trim ends of strain
            if self.reduced_pad  != 0:
                overwhite = TimeSeries(zeros(e-s, dtype=self.strain.dtype),
                                             delta_t=self.strain.delta_t)
                pycbc.fft.ifft(fseries, overwhite)
                overwhite2 = overwhite[self.reduced_pad:len(overwhite)-self.reduced_pad]
                taper_window = self.trim_padding / 2.0 / overwhite.sample_rate
                gate_params = [(overwhite2.start_time, 0., taper_window),
                               (overwhite2.end_time, 0., taper_window)]
                gate_data(overwhite2, gate_params)
                fseries_trimmed = FrequencySeries(zeros(len(overwhite2) // 2 + 1,
                                                  dtype=fseries.dtype), delta_f=delta_f)
                pycbc.fft.fft(overwhite2, fseries_trimmed)
                fseries_trimmed.start_time = fseries.start_time + self.reduced_pad * self.strain.delta_t
            else:
                fseries_trimmed = fseries

            fseries_trimmed.psd = psd
            self.segments[delta_f] = fseries_trimmed

        stilde = self.segments[delta_f]
        return stilde

    def near_hwinj(self):
        """Check that the current set of triggers could be influenced by
        a hardware injection.
        """
        if not self.state:

        if not self.state.is_extent_valid(self.start_time, self.blocksize, pycbc.frame.NO_HWINJ):
        return False

    def null_advance_strain(self, blocksize):
        """ Advance and insert zeros

        """
        sample_step = int(blocksize * self.sample_rate)
        csize = sample_step + self.corruption * 2
        self.strain.roll(-sample_step)

        # We should roll this off at some point too...
        self.strain[len(self.strain) - csize + self.corruption:] = 0
        self.strain.start_time += blocksize

        # The next time we need strain will need to be tapered

    def advance(self, blocksize, timeout=10):
        """Advanced buffer blocksize seconds.

        Add blocksize seconds more to the buffer, push blocksize seconds

        Returns
        -------
            Returns True if this block is analyzable.
        """
        ts = super(StrainBuffer, self).attempt_advance(blocksize, timeout=timeout)
        self.blocksize = blocksize

        # We have given up so there is no time series
            logging.info("%s frame is late, giving up", self.detector)
            self.null_advance_strain(blocksize)
            if self.state:
                self.state.null_advance(blocksize)
            if self.dq:
                self.dq.null_advance(blocksize)

        # We collected some data so we are closer to being able to analyze data
        self.wait_duration -= blocksize

        # If the data we got was invalid, reset the counter on how much to collect
        # This behavior corresponds to how we handle CAT1 vetoes
        if self.state and self.state.advance(blocksize) is False:
            self.add_hard_count()
            logging.info("%s time has invalid data, resetting buffer",
                         self.detector)

        # Also advance the dq vector in lockstep
        if self.dq:
            self.dq.advance(blocksize)

        # only condition with the needed raw data so we can continuously add
        # to the existing result

        # Precondition
        start = len(self.raw_buffer) - csize * self.factor
        strain = self.raw_buffer[start:]

        strain =  pycbc.filter.highpass_fir(strain, self.highpass_frequency,
                                       self.highpass_samples,
                                       beta=self.beta)
        strain = (strain * self.dyn_range_fac).astype(numpy.float32)

        strain = pycbc.filter.resample_to_delta_t(strain,
                                           1.0/self.sample_rate, method='ldas')

        # remove corruption at beginning
        strain = strain[self.corruption:]

        # taper beginning if needed
        if self.taper_immediate_strain:
            logging.info("Tapering start of %s strain block", self.detector)
            strain = gate_data(
                    strain, [(strain.start_time, 0., self.autogating_taper)])
            self.taper_immediate_strain = False

        # Stitch into continuous stream
        self.strain[len(self.strain) - csize + self.corruption:] = strain[:]

        # apply gating if needed
        if self.autogating_threshold is not None:
                    strain[:-self.corruption],
                    psd_duration=2., psd_stride=1.,
                    threshold=self.autogating_threshold,
                    cluster_window=self.autogating_cluster,
                    low_freq_cutoff=self.highpass_frequency,
                    corrupt_time=self.autogating_pad)
                logging.info('Autogating %s at %s', self.detector,
                             ', '.join(['%.3f' % gt for gt in glitch_times]))
                self.gate_params = \
                        [(gt, self.autogating_width, self.autogating_taper)
                         for gt in glitch_times]
                self.strain = gate_data(self.strain, self.gate_params)

        if self.psd is None and self.wait_duration <=0:
            self.recalculate_psd()

        return self.wait_duration <= 0

    def from_cli(cls, ifo, args, maxlen):
        """Initialize a StrainBuffer object (data reader) for a particular
        detector.
        """
        state_channel = analyze_flags = None
        if args.state_channel and ifo in args.state_channel \
                and args.analyze_flags and ifo in args.analyze_flags:
            state_channel = ':'.join([ifo, args.state_channel[ifo]])
            analyze_flags = args.analyze_flags[ifo].split(',')

        dq_channel = dq_flags = None
        if args.data_quality_channel and ifo in args.data_quality_channel \
                and args.data_quality_flags and ifo in args.data_quality_flags:
            dq_channel = ':'.join([ifo, args.data_quality_channel[ifo]])
            dq_flags = args.data_quality_flags[ifo].split(',')

        if args.frame_type:
            frame_src = pycbc.frame.frame_paths(args.frame_type[ifo],
                                                args.start_time,
                                                args.end_time)
        else:
            frame_src = [args.frame_src[ifo]]
        strain_channel = ':'.join([ifo, args.channel_name[ifo]])

        return cls(frame_src, strain_channel,
                   args.start_time, max_buffer=maxlen * 2,
                   state_channel=state_channel,
                   data_quality_channel=dq_channel,
                   sample_rate=args.sample_rate,
                   low_frequency_cutoff=args.low_frequency_cutoff,
                   highpass_frequency=args.highpass_frequency,
                   highpass_reduction=args.highpass_reduction,
                   highpass_bandwidth=args.highpass_bandwidth,
                   psd_samples=args.psd_samples,
                   trim_padding=args.trim_padding,
                   psd_segment_length=args.psd_segment_length,
                   psd_inverse_length=args.psd_inverse_length,
                   autogating_threshold=args.autogating_threshold,
                   autogating_cluster=args.autogating_cluster,
                   autogating_pad=args.autogating_pad,
                   autogating_width=args.autogating_width,
                   autogating_taper=args.autogating_taper,
                   psd_abort_difference=args.psd_abort_difference,
                   psd_recalculate_difference=args.psd_recalculate_difference,
                   force_update_cache=args.force_update_cache,
                   increment_update_cache=args.increment_update_cache[ifo],
                   analyze_flags=analyze_flags,
                   data_quality_flags=dq_flags,
                   dq_padding=args.data_quality_padding)

from .recalibrate import CubicSpline, PhysicalModel

from .strain import detect_loud_glitches
from .strain import from_cli, from_cli_single_ifo, from_cli_multi_ifos
from .strain import insert_strain_option_group, insert_strain_option_group_multi_ifo
from .strain import verify_strain_options, verify_strain_options_multi_ifo
from .strain import gate_data, StrainSegments, StrainBuffer

from .gate import add_gate_option_group, gates_from_cli
from .gate import apply_gates_to_td, apply_gates_to_fd, psd_gates_from_cli

models = {
    CubicSpline.name: CubicSpline,
    PhysicalModel.name: PhysicalModel
}

def read_model_from_config(cp, ifo, section="calibration"):
    """Returns an instance of the calibration model specified in the

        The detector (H1, L1)  whose model will be loaded.
    section : {"calibration", string}
        Section name from which to retrieve the model.

    Returns
    -------
    instance
        An instance of the calibration model class.
    """
    model = cp.get_opt_tag(section, "{}_model".format(ifo.lower()), None)
    recalibrator = models[model].from_config(cp, ifo.lower(), section)

    return recalibrator

from lal import PI, MTSUN_SI, TWOPI, GAMMA
from glue.ligolw import ligolw, lsctables, ilwd, utils as ligolw_utils
from pycbc.tmpltbank.lambda_mapping import ethinca_order_from_string

def return_empty_sngl(nones=False):
    """
    Function to create a SnglInspiral object where all columns are populated
    but all are set to values that test False (ie. strings to '', floats/ints
    to 0, ...). This avoids errors when you try to create a table containing
    columns you don't care about, but which still need populating. NOTE: This
    will also produce a process_id and event_id with 0 values. For most
    applications these should be set to their correct values.

    nones : bool (False)
        If True, just set all columns to None.

    Returns
    --------
    lsctables.SnglInspiral
        The "empty" SnglInspiral object.
    """

    sngl = lsctables.SnglInspiral()
    cols = lsctables.SnglInspiralTable.validcolumns
    if nones:
        for entry in cols:
            setattr(sngl, entry, None)
    else:
        for entry in cols.keys():
            if cols[entry] in ['real_4','real_8']:
                setattr(sngl,entry,0.)
            elif cols[entry] == 'int_4s':
                setattr(sngl,entry,0)
            elif cols[entry] == 'lstring':
                setattr(sngl,entry,'')
            elif entry == 'process_id':
                sngl.process_id = ilwd.ilwdchar("process:process_id:0")
            elif entry == 'event_id':
                sngl.event_id = ilwd.ilwdchar("sngl_inspiral:event_id:0")
            else:
                raise ValueError("Column %s not recognized" %(entry) )
    return sngl

def return_search_summary(start_time=0, end_time=0, nevents=0,
                          ifos=None, **kwargs):
    """
    Function to create a SearchSummary object where all columns are populated
    will also produce a process_id with 0 values. For most applications these
    should be set to their correct values.

    It then populates columns if given them as options.

    Returns
    --------
    lsctables.SeachSummary
        The "empty" SearchSummary object.
    """
    if ifos is None:
        ifos = []

    # create an empty search summary
    search_summary = lsctables.SearchSummary()
    cols = lsctables.SearchSummaryTable.validcolumns
    for entry in cols.keys():
        if cols[entry] in ['real_4','real_8']:
            setattr(search_summary,entry,0.)
        elif cols[entry] == 'int_4s':
            setattr(search_summary,entry,0)
        elif cols[entry] == 'lstring':
            setattr(search_summary,entry,'')
        elif entry == 'process_id':
            search_summary.process_id = ilwd.ilwdchar("process:process_id:0")
        else:
            raise ValueError("Column %s not recognized" %(entry) )

    # fill in columns
    if len(ifos):
        search_summary.ifos = ','.join(ifos)
    if nevents:
        search_summary.nevents = nevents
    if start_time and end_time:
        search_summary.in_start_time = int(start_time)
        search_summary.in_start_time_ns = int(start_time % 1 * 1e9)
        search_summary.in_end_time = int(end_time)
        search_summary.in_end_time_ns = int(end_time % 1 * 1e9)
        search_summary.out_start_time = int(start_time)
        search_summary.out_start_time_ns = int(start_time % 1 * 1e9)
        search_summary.out_end_time = int(end_time)
        search_summary.out_end_time_ns = int(end_time % 1 * 1e9)

    return search_summary

def convert_to_sngl_inspiral_table(params, proc_id):
    '''
    Convert a list of m1,m2,spin1z,spin2z values into a basic sngl_inspiral
    table with mass and spin parameters populated and event IDs assigned

    params : iterable
        Each entry in the params iterable should be a sequence of
        [mass1, mass2, spin1z, spin2z] in that order
    proc_id : ilwd char
        Process ID to add to each row of the sngl_inspiral table

    Returns
    SnglInspiralTable
        Bank of templates in SnglInspiralTable format
    '''
    sngl_inspiral_table = lsctables.New(lsctables.SnglInspiralTable)
    col_names = ['mass1','mass2','spin1z','spin2z']

    for values in params:
        tmplt = return_empty_sngl()

        tmplt.process_id = proc_id
        for colname, value in zip(col_names, values):
            setattr(tmplt, colname, value)
        tmplt.mtotal, tmplt.eta = pnutils.mass1_mass2_to_mtotal_eta(
            tmplt.mass1, tmplt.mass2)
        tmplt.mchirp, _ = pnutils.mass1_mass2_to_mchirp_eta(
        tmplt.template_duration = 0 # FIXME
        tmplt.event_id = sngl_inspiral_table.get_next_id()
        sngl_inspiral_table.append(tmplt)

    return sngl_inspiral_table

def calculate_ethinca_metric_comps(metricParams, ethincaParams, mass1, mass2,
                                   spin1z=0., spin2z=0., full_ethinca=True):
    """
    Calculate the Gamma components needed to use the ethinca metric.
    At present this outputs the standard TaylorF2 metric over the end time
    and chirp times \tau_0 and \tau_3.
    A desirable upgrade might be to use the \chi coordinates [defined WHERE?]
    for metric distance instead of \tau_0 and \tau_3.
    The lower frequency cutoff is currently hard-coded to be the same as the
    bank layout options fLow and f0 (which must be the same as each other).

    metricParams : metricParameters instance
        Structure holding all the options for construction of the metric
        and the eigenvalues, eigenvectors and covariance matrix
        needed to manipulate the space.
    ethincaParams : ethincaParameters instance
        Structure holding options relevant to the ethinca metric computation.
        Mass of the heavier body in the considered template.
        Mass of the lighter body in the considered template.
    spin1z : float (optional, default=0)
        Spin of the heavier body in the considered template.
    spin2z : float (optional, default=0)
        Spin of the lighter body in the considered template.
    full_ethinca : boolean (optional, default=True)
        If True calculate the ethinca components in all 3 directions (mass1,
        mass2 and time). If False calculate only the time component (which is
        stored in Gamma0).
    Returns
    --------
    fMax_theor : float
        Value of the upper frequency cutoff given by the template parameters
        and the cutoff formula requested.

    gammaVals : numpy_array
        Array holding 6 independent metric components in
        (end_time, tau_0, tau_3) coordinates to be stored in the Gamma0-5
        slots of a SnglInspiral object.
    """
    if (float(spin1z) != 0. or float(spin2z) != 0.) and full_ethinca:
        raise NotImplementedError("Ethinca cannot at present be calculated "
                                  "for nonzero component spins!")
    f0 = metricParams.f0
    if f0 != metricParams.fLow:
        raise ValueError("If calculating ethinca the bank f0 value must be "
                         "equal to f-low!")
    if ethincaParams.fLow is not None and (
        ethincaParams.fLow != metricParams.fLow):
        raise NotImplementedError("An ethinca metric f-low different from the"
                                  " bank metric f-low is not supported!")

    twicePNOrder = ethinca_order_from_string(ethincaParams.pnOrder)

    piFl = PI * f0
    totalMass, eta = pnutils.mass1_mass2_to_mtotal_eta(mass1, mass2)
    totalMass = totalMass * MTSUN_SI
    v0cube = totalMass*piFl
    v0 = v0cube**(1./3.)

    # Get theoretical cutoff frequency and work out the closest
    # frequency for which moments were calculated
    fMax_theor = pnutils.frequency_cutoff_from_name(
        ethincaParams.cutoff, mass1, mass2, spin1z, spin2z)
    fMaxes = list(metricParams.moments['J4'].keys())
    fMaxIdx = abs(numpy.array(fMaxes,dtype=float) - fMax_theor).argmin()
    fMax = fMaxes[fMaxIdx]

    # Set the appropriate moments
    Js = numpy.zeros([18,3],dtype=float)
    for i in range(18):
        Js[i,0] = metricParams.moments['J%d'%(i)][fMax]
        Js[i,1] = metricParams.moments['log%d'%(i)][fMax]
        Js[i,2] = metricParams.moments['loglog%d'%(i)][fMax]

    # Compute the time-dependent metric term.
    two_pi_flower_sq = TWOPI * f0 * TWOPI * f0
    gammaVals = numpy.zeros([6],dtype=float)
    gammaVals[0] = 0.5 * two_pi_flower_sq * \
                    ( Js[(1,0)] - (Js[(4,0)]*Js[(4,0)]) )

    # If mass terms not required stop here
    if not full_ethinca:
        return fMax_theor, gammaVals

    # 3pN is a mess, so split it into pieces
    a0 = 11583231236531/200286535680 - 5*PI*PI - 107*GAMMA/14
    a1 = (-15737765635/130056192 + 2255*PI*PI/512)*eta
    a2 = (76055/73728)*eta*eta
    a3 = (-127825/55296)*eta*eta*eta
    alog = numpy.log(4*v0) # Log terms are tricky - be careful

    # Get the Psi coefficients
    Psi = [{},{}] #Psi = numpy.zeros([2,8,2],dtype=float)
    Psi[0][0,0] = 3/5
    Psi[0][2,0] = (743/756 + 11*eta/3)*v0*v0
    Psi[0][3,0] = 0.
    Psi[0][4,0] = (-3058673/508032 + 5429*eta/504 + 617*eta*eta/24)\
                    *v0cube*v0
    Psi[0][5,1] = (-7729*PI/126)*v0cube*v0*v0/3
    Psi[0][6,0] = (128/15)*(-3*a0 - a1 + a2 + 3*a3 + 107*(1+3*alog)/14)\
                    *v0cube*v0cube
    Psi[0][6,1] = (6848/35)*v0cube*v0cube/3
    Psi[0][7,0] = (-15419335/63504 - 75703*eta/756)*PI*v0cube*v0cube*v0

    Psi[1][0,0] = 0.
    Psi[1][2,0] = (3715/12096 - 55*eta/96)/PI/v0;
    Psi[1][3,0] = -3/2
    Psi[1][4,0] = (15293365/4064256 - 27145*eta/16128 - 3085*eta*eta/384)\
                    *v0/PI
    Psi[1][5,1] = (193225/8064)*v0*v0/3
    Psi[1][6,0] = (4/PI)*(2*a0 + a1/3 - 4*a2/3 - 3*a3 -107*(1+6*alog)/42)\
                    *v0cube
    Psi[1][6,1] = (-428/PI/7)*v0cube/3
    Psi[1][7,0] = (77096675/1161216 + 378515*eta/24192 + 74045*eta*eta/8064)\

    # Calculate the g matrix
    PNterms = [(0,0),(2,0),(3,0),(4,0),(5,1),(6,0),(6,1),(7,0)]
    PNterms = [term for term in PNterms if term[0] <= twicePNOrder]

    # Now can compute the mass-dependent gamma values
    for m in [0, 1]:
        for k in PNterms:
            gammaVals[1+m] += 0.5 * two_pi_flower_sq * Psi[m][k] * \
                                ( Js[(9-k[0],k[1])]
                                - Js[(12-k[0],k[1])] * Js[(4,0)] )

    g = numpy.zeros([2,2],dtype=float)
    for (m,n) in [(0,0),(0,1),(1,1)]:
            for l in PNterms:
                g[m,n] += Psi[m][k] * Psi[n][l] * \
                        ( Js[(17-k[0]-l[0], k[1]+l[1])]
                        - Js[(12-k[0],k[1])] * Js[(12-l[0],l[1])] )
        g[m,n] = 0.5 * two_pi_flower_sq * g[m,n]
        g[n,m] = g[m,n]

    gammaVals[3] = g[0,0]
    gammaVals[4] = g[0,1]
    gammaVals[5] = g[1,1]

    return fMax_theor, gammaVals

def output_sngl_inspiral_table(outputFile, tempBank, metricParams,
                               ethincaParams, programName="", optDict = None,
                               outdoc=None, **kwargs):
    """
    Function that converts the information produced by the various pyCBC bank
    generation codes into a valid LIGOLW xml file containing a sngl_inspiral
    table and outputs to file.

    outputFile : string
        Name of the file that the bank will be written to
    tempBank : iterable
        Each entry in the tempBank iterable should be a sequence of
        [mass1,mass2,spin1z,spin2z] in that order.
    ethincaParams: {ethincaParameters instance, None}
        Structure holding options relevant to the ethinca metric computation
        including the upper frequency cutoff to be used for filtering.
        NOTE: The computation is currently only valid for non-spinning systems
        and uses the TaylorF2 approximant.
    programName (key-word-argument) : string
        Name of the executable that has been run
    optDict (key-word argument) : dictionary
        Dictionary of the command line arguments passed to the program
    outdoc (key-word argument) : ligolw xml document
        If given add template bank to this representation of a xml document and
        write to disk. If not given create a new document.
    kwargs : key-word arguments
        All other key word arguments will be passed directly to
        ligolw_process.register_to_xmldoc
    """
    if optDict is None:
        optDict = {}
    if outdoc is None:

    # get IFO to put in search summary table
    ifos = []
    if 'channel_name' in optDict.keys():
        if optDict['channel_name'] is not None:
            ifos = [optDict['channel_name'][0:2]]

    proc_id = ligolw_process.register_to_xmldoc(outdoc, programName, optDict,
                                                ifos=ifos, **kwargs).process_id
    sngl_inspiral_table = convert_to_sngl_inspiral_table(tempBank, proc_id)
    # Calculate Gamma components if needed
    if ethincaParams is not None:
        if ethincaParams.doEthinca:
            for sngl in sngl_inspiral_table:
                # Set tau_0 and tau_3 values needed for the calculation of
                # ethinca metric distances
                (sngl.tau0,sngl.tau3) = pnutils.mass1_mass2_to_tau0_tau3(
                    sngl.mass1, sngl.mass2, metricParams.f0)
                fMax_theor, GammaVals = calculate_ethinca_metric_comps(
                    metricParams, ethincaParams,
                    sngl.mass1, sngl.mass2, spin1z=sngl.spin1z,
                    spin2z=sngl.spin2z, full_ethinca=ethincaParams.full_ethinca)
                # assign the upper frequency cutoff and Gamma0-5 values
                sngl.f_final = fMax_theor
                for i in range(len(GammaVals)):
                    setattr(sngl, "Gamma"+str(i), GammaVals[i])
        # If Gamma metric components are not wanted, assign f_final from an
        # upper frequency cutoff specified in ethincaParams
        elif ethincaParams.cutoff is not None:
                sngl.f_final = pnutils.frequency_cutoff_from_name(
                    ethincaParams.cutoff,
                    sngl.mass1, sngl.mass2, sngl.spin1z, sngl.spin2z)

    # set per-template low-frequency cutoff
    if 'f_low_column' in optDict and 'f_low' in optDict and \
            optDict['f_low_column'] is not None:
            setattr(sngl, optDict['f_low_column'], optDict['f_low'])

    outdoc.childNodes[0].appendChild(sngl_inspiral_table)

    # get times to put in search summary table
    start_time = 0
    end_time = 0
    if 'gps_start_time' in optDict.keys() and 'gps_end_time' in optDict.keys():
        start_time = optDict['gps_start_time']
        end_time = optDict['gps_end_time']

    # make search summary table
    search_summary_table = lsctables.New(lsctables.SearchSummaryTable)
    search_summary = return_search_summary(start_time, end_time,
                               len(sngl_inspiral_table), ifos, **kwargs)
    search_summary_table.append(search_summary)
    outdoc.childNodes[0].appendChild(search_summary_table)

    # write the xml doc to disk
    ligolw_utils.write_filename(outdoc, outputFile,
                                gz=outputFile.endswith('.gz'))

from pycbc.tmpltbank.coord_utils import get_cov_params

def get_physical_covaried_masses(xis, bestMasses, bestXis, req_match,
                                 massRangeParams, metricParams, fUpper,
                                 giveUpThresh = 5000):
    """
    This function takes the position of a point in the xi parameter space and
    iteratively finds a close point in the physical coordinate space (masses
    and spins).
 
    xis : list or array
        Desired position of the point in the xi space. If only N values are
        provided and the xi space's dimension is larger then it is assumed that
        *any* value in the remaining xi coordinates is acceptable.
    bestMasses : list
        Contains [totalMass, eta, spin1z, spin2z]. Is a physical position
        mapped to xi coordinates in bestXis that is close to the desired point.
        This is aimed to give the code a starting point.
    bestXis : list
        Contains the position of bestMasses in the xi coordinate system.
    req_match : float
        Desired maximum mismatch between xis and the obtained point. If a point
        is found with mismatch < req_match immediately stop and return that
        point. A point with this mismatch will not always be found.
    massRangeParams : massRangeParameters instance
        Instance holding all the details of mass ranges and spin ranges.
    fUpper : float
        The value of fUpper that was used when obtaining the xi_i
        coordinates. This lets us know how to rotate potential physical points
        into the correct xi_i space. This must be a key in metricParams.evals,
        metricParams.evecs and metricParams.evecsCV
        (ie. we must know how to do the transformation for
        the given value of fUpper)
    giveUpThresh : int, optional (default = 5000)
        The program will try this many iterations. If no close matching point
        has been found after this it will give up.

    Returns
    --------
        The heavier mass of the obtained point.
        The smaller mass of the obtained point
    spin1z : float
        The heavier bodies spin of the obtained point.
    spin2z : float
        The smaller bodies spin of the obtained point.
    count : int
        How many iterations it took to find the point. For debugging.
    mismatch : float
        The mismatch between the obtained point and the input xis.
    new_xis : list
        The position of the point in the xi space
    """
    # TUNABLE PARAMETERS GO HERE!
    # This states how far apart to scatter test points in the first proposal
    origScaleFactor = 1

    # Set up
    xi_size = len(xis)
    scaleFactor = origScaleFactor
    bestChirpmass = bestMasses[0] * (bestMasses[1])**(3./5.)
    count = 0
    unFixedCount = 0
    currDist = 100000000000000000
    while(1):
        # If we are a long way away we use larger jumps
        if count:
            if currDist > 1 and scaleFactor == origScaleFactor:
                scaleFactor = origScaleFactor*10
        # Get a set of test points with mass -> xi mappings
        totmass, eta, spin1z, spin2z, mass1, mass2, new_xis = \
            get_mass_distribution([bestChirpmass, bestMasses[1], bestMasses[2],
                                   bestMasses[3]],
                                  scaleFactor, massRangeParams, metricParams,
                                  fUpper)
        cDist = (new_xis[0] - xis[0])**2
        for j in range(1,xi_size):
            cDist += (new_xis[j] - xis[j])**2
        if (cDist.min() < req_match):
            idx = cDist.argmin()
            scaleFactor = origScaleFactor
            new_xis_list = [new_xis[ldx][idx] for ldx in range(len(new_xis))]
            return mass1[idx], mass2[idx], spin1z[idx], spin2z[idx], count, \
                   cDist.min(), new_xis_list
        if (cDist.min() < currDist):
            bestMasses[0] = totmass[idx]
            bestMasses[1] = eta[idx]
            bestMasses[2] = spin1z[idx]
            bestMasses[3] = spin2z[idx]
            bestChirpmass = bestMasses[0] * (bestMasses[1])**(3./5.)
            currDist = cDist.min()
            unFixedCount = 0
        count += 1
        unFixedCount += 1
        if unFixedCount > giveUpThresh:
            # Stop at this point
            diff = (bestMasses[0]*bestMasses[0] * (1-4*bestMasses[1]))**0.5
            mass1 = (bestMasses[0] + diff)/2.
            mass2 = (bestMasses[0] - diff)/2.
            new_xis_list = [new_xis[ldx][0] for ldx in range(len(new_xis))]
            return mass1, mass2, bestMasses[2], bestMasses[3], count, \
                   currDist, new_xis_list
        if not unFixedCount % 100:
            scaleFactor *= 2
        if scaleFactor > 64:
            scaleFactor = 1
    # Shouldn't be here!
    raise RuntimeError

def get_mass_distribution(bestMasses, scaleFactor, massRangeParams,
                          metricParams, fUpper,
                          numJumpPoints=100, chirpMassJumpFac=0.0001,
                          etaJumpFac=0.01, spin1zJumpFac=0.01,
                          spin2zJumpFac=0.01):
    """
    Given a set of masses, this function will create a set of points nearby
    in the mass space and map these to the xi space.

        Contains [ChirpMass, eta, spin1z, spin2z]. Points will be placed around
        tjos
    scaleFactor : float
        This parameter describes the radius away from bestMasses that points
        will be placed in.
    numJumpPoints : int, optional (default = 100)
        The number of points that will be generated every iteration
    chirpMassJumpFac : float, optional (default=0.0001)
        The jump points will be chosen with fractional variation in chirpMass
        up to this multiplied by scaleFactor.
    etaJumpFac : float, optional (default=0.01)
        The jump points will be chosen with fractional variation in eta
    spin1zJumpFac : float, optional (default=0.01)
        The jump points will be chosen with absolute variation in spin1z up to
        this multiplied by scaleFactor.
    spin2zJumpFac : float, optional (default=0.01)
        The jump points will be chosen with absolute variation in spin2z up to

    Returns 
    --------
    Totmass : numpy.array
        Total mass of the resulting points
    Eta : numpy.array
        Symmetric mass ratio of the resulting points
    Spin1z : numpy.array
        Spin of the heavier body of the resulting points
    Spin2z : numpy.array
        Spin of the smaller body of the resulting points
    Diff : numpy.array
        Mass1 - Mass2 of the resulting points
    Mass1 : numpy.array
        Mass1 (mass of heavier body) of the resulting points
    Mass2 : numpy.array
        Mass2 (mass of smaller body) of the resulting points
    new_xis : list of numpy.array
        Position of points in the xi coordinates
    """
    # FIXME: It would be better if rejected values could be drawn from the 
    # full possible mass/spin distribution. However speed in this function is
    # a major factor and must be considered.
    bestChirpmass = bestMasses[0]
    bestEta = bestMasses[1]
    bestSpin1z = bestMasses[2]
    bestSpin2z = bestMasses[3]

    # Firstly choose a set of values for masses and spins
    chirpmass = bestChirpmass * (1 - (numpy.random.random(numJumpPoints)-0.5) \
                                       * chirpMassJumpFac * scaleFactor )
    etaRange = massRangeParams.maxEta - massRangeParams.minEta
    currJumpFac = etaJumpFac * scaleFactor
    if currJumpFac > etaRange:
        currJumpFac = etaRange
    eta = bestEta * ( 1 - (numpy.random.random(numJumpPoints) - 0.5) \
                           * currJumpFac)

    maxSpinMag = max(massRangeParams.maxNSSpinMag, massRangeParams.maxBHSpinMag)
    minSpinMag = min(massRangeParams.maxNSSpinMag, massRangeParams.maxBHSpinMag)
    # Note that these two are cranged by spinxzFac, *not* spinxzFac/spinxz
    currJumpFac = spin1zJumpFac * scaleFactor
    if currJumpFac > maxSpinMag:
        currJumpFac = maxSpinMag

    # Actually set the new spin trial points
    if massRangeParams.nsbhFlag or (maxSpinMag == minSpinMag):
        curr_spin_1z_jump_fac = currJumpFac
        curr_spin_2z_jump_fac = currJumpFac
        # Check spins aren't going to be unphysical
        if currJumpFac > massRangeParams.maxBHSpinMag:
            curr_spin_1z_jump_fac = massRangeParams.maxBHSpinMag
        if currJumpFac > massRangeParams.maxNSSpinMag:
            curr_spin_2z_jump_fac = massRangeParams.maxNSSpinMag
        spin1z = bestSpin1z + ( (numpy.random.random(numJumpPoints) - 0.5) \
                            * curr_spin_1z_jump_fac)
        spin2z = bestSpin2z + ( (numpy.random.random(numJumpPoints) - 0.5) \
                            * curr_spin_2z_jump_fac)
    else:
        # If maxNSSpinMag is very low (0) and maxBHSpinMag is high we can
        # find it hard to place any points. So mix these when
        # masses are swapping between the NS and BH.
        curr_spin_bh_jump_fac = currJumpFac
        curr_spin_ns_jump_fac = currJumpFac
            curr_spin_bh_jump_fac = massRangeParams.maxBHSpinMag
            curr_spin_ns_jump_fac = massRangeParams.maxNSSpinMag
        spin1z = numpy.zeros(numJumpPoints, dtype=float)
        spin2z = numpy.zeros(numJumpPoints, dtype=float)
        split_point = int(numJumpPoints/2)
        # So set the first half to be at least within the BH range and the
        # second half to be at least within the NS range
        spin1z[:split_point] = bestSpin1z + \
                            ( (numpy.random.random(split_point) - 0.5)\
                              * curr_spin_bh_jump_fac)
        spin1z[split_point:] = bestSpin1z + \
                      ( (numpy.random.random(numJumpPoints-split_point) - 0.5)\
                        * curr_spin_ns_jump_fac)
        spin2z[:split_point] = bestSpin2z + \
        spin2z[split_point:] = bestSpin2z + \

    # Point[0] is always set to the original point
    chirpmass[0] = bestChirpmass
    eta[0] = bestEta
    spin1z[0] = bestSpin1z
    spin2z[0] = bestSpin2z

    # Remove points where eta becomes unphysical
    eta[eta > massRangeParams.maxEta] = massRangeParams.maxEta
    if massRangeParams.minEta:
        eta[eta < massRangeParams.minEta] = massRangeParams.minEta
    else:
        eta[eta < 0.0001] = 0.0001

    # Total mass, masses and mass diff
    totmass = chirpmass / (eta**(3./5.))
    diff = (totmass*totmass * (1-4*eta))**0.5
    mass1 = (totmass + diff)/2.
    mass2 = (totmass - diff)/2.

    # Check the validity of the spin values
    # Do the first spin

    if maxSpinMag == 0:
        # Shortcut if non-spinning
        pass
    elif massRangeParams.nsbhFlag or (maxSpinMag == minSpinMag):
        # Simple case where I don't have to worry about correlation with mass
        numploga = abs(spin1z) > massRangeParams.maxBHSpinMag
        spin1z[numploga] = 0
    else:
        # Do have to consider masses
        boundary_mass = massRangeParams.ns_bh_boundary_mass
        numploga1 = numpy.logical_and(mass1 >= boundary_mass,
                                   abs(spin1z) <= massRangeParams.maxBHSpinMag)
        numploga2 = numpy.logical_and(mass1 < boundary_mass,
                                   abs(spin1z) <= massRangeParams.maxNSSpinMag)
        numploga = numpy.logical_or(numploga1, numploga2)
        numploga = numpy.logical_not(numploga)

    # Same for the second spin

        pass
        numplogb = abs(spin2z) > massRangeParams.maxNSSpinMag
        spin2z[numplogb] = 0
    else:
        numplogb1 = numpy.logical_and(mass2 >= boundary_mass,
                                   abs(spin2z) <= massRangeParams.maxBHSpinMag)
        numplogb2 = numpy.logical_and(mass2 < boundary_mass,
                                   abs(spin2z) <= massRangeParams.maxNSSpinMag)
        numplogb = numpy.logical_or(numplogb1, numplogb2)
        numplogb = numpy.logical_not(numplogb)

    if (maxSpinMag) and (numploga[0] or numplogb[0]):
        raise ValueError("Cannot remove the guide point!")

    # And remove points where the individual masses are outside of the physical
    # range. Or the total masses are.
    # These "removed" points will have metric distances that will be much, much
    # larger than any thresholds used in the functions in brute_force_utils.py
    # and will always be rejected. An unphysical value cannot be used as it
    # would result in unphysical metric distances and cause failures.
    totmass[mass1 < massRangeParams.minMass1] = 0.0001
    totmass[mass1 > massRangeParams.maxMass1] = 0.0001
    totmass[mass2 < massRangeParams.minMass2] = 0.0001
    totmass[mass2 > massRangeParams.maxMass2] = 0.0001
    # There is some numerical error which can push this a bit higher. We do
    # *not* want to reject the initial guide point. This error comes from
    # Masses -> totmass, eta -> masses conversion, we will have points pushing
    # onto the boudaries of the space.
    totmass[totmass > massRangeParams.maxTotMass*1.0001] = 0.0001
    totmass[totmass < massRangeParams.minTotMass*0.9999] = 0.0001
    if massRangeParams.max_chirp_mass:
        totmass[chirpmass > massRangeParams.max_chirp_mass*1.0001] = 0.0001
    if massRangeParams.min_chirp_mass:
        totmass[chirpmass < massRangeParams.min_chirp_mass*0.9999] = 0.0001

    if totmass[0] < 0.00011:

    mass1[totmass < 0.00011] = 0.0001
    mass2[totmass < 0.00011] = 0.0001

    # Then map to xis
    new_xis = get_cov_params(mass1, mass2, spin1z, spin2z,
                             metricParams, fUpper)
    return totmass, eta, spin1z, spin2z, mass1, mass2, new_xis

def stack_xi_direction_brute(xis, bestMasses, bestXis, direction_num,
                             req_match, massRangeParams, metricParams, fUpper,
                             scaleFactor=0.8, numIterations=3000):
    """
    This function is used to assess the depth of the xi_space in a specified
    dimension at a specified point in the higher dimensions. It does this by
    iteratively throwing points at the space to find maxima and minima.

        Position in the xi space at which to assess the depth. This can be only
        a subset of the higher dimensions than that being sampled.
        mapped to xi coordinates in bestXis that is close to the xis point.
    direction_num : int
        The dimension that you want to assess the depth of (0 = 1, 1 = 2 ...)
        When considering points to assess the depth with, only consider points
        with a mismatch that is smaller than this with xis.
    scaleFactor : float, optional (default = 0.8)
        The value of the scale factor that is used when calling
        pycbc.tmpltbank.get_mass_distribution.
    numIterations : int, optional (default = 3000)
        The number of times to make calls to get_mass_distribution when
        assessing the maximum/minimum of this parameter space. Making this
        smaller makes the code faster, but at the cost of accuracy.   
 
    Returns
    --------
    xi_min : float
        The minimal value of the specified dimension at the specified point in
    xi_max : float
       The maximal value of the specified dimension at the specified point in
    """

    # Find minimum
    ximin = find_xi_extrema_brute(xis, bestMasses, bestXis, direction_num, \
                                  req_match, massRangeParams, metricParams, \
                                  fUpper, find_minimum=True, \
                                  scaleFactor=scaleFactor, \
                                  numIterations=numIterations)
 
    # Find maximum
    ximax = find_xi_extrema_brute(xis, bestMasses, bestXis, direction_num, \
                                  fUpper, find_minimum=False, \

    return ximin, ximax

def find_xi_extrema_brute(xis, bestMasses, bestXis, direction_num, req_match, \
                          massRangeParams, metricParams, fUpper, \
                          find_minimum=False, scaleFactor=0.8, \
                          numIterations=3000):   
    """
    This function is used to find the largest or smallest value of the xi
    space in a specified
    iteratively throwing points at the space to find extrema.

    find_minimum : boolean, optional (default = False)
        If True, find the minimum value of the xi direction. If False find the
        maximum value.

    Returns
    --------
    xi_extent : float
        The extremal value of the specified dimension at the specified point in
    """

    # Setup
    if find_minimum:
        xiextrema = 10000000000
    else:
        xiextrema = -100000000000

    for _ in range(numIterations):
        # Evaluate extrema of the xi direction specified
        totmass, eta, spin1z, spin2z, _, _, new_xis = \
            get_mass_distribution([bestChirpmass,bestMasses[1],bestMasses[2],
                                  fUpper)
        for j in range(1, xi_size):
        redCDist = cDist[cDist < req_match]
        if len(redCDist):
            if not find_minimum:
                new_xis[direction_num][cDist > req_match] = -10000000
                currXiExtrema = (new_xis[direction_num]).max()
                idx = (new_xis[direction_num]).argmax()
            else:
                new_xis[direction_num][cDist > req_match] = 10000000
                currXiExtrema = (new_xis[direction_num]).min()
                idx = (new_xis[direction_num]).argmin()
            if ( ((not find_minimum) and (currXiExtrema > xiextrema)) or \
                         (find_minimum and (currXiExtrema < xiextrema)) ):
                xiextrema = currXiExtrema
                bestMasses[0] = totmass[idx]
                bestMasses[1] = eta[idx]
                bestMasses[2] = spin1z[idx]
                bestMasses[3] = spin2z[idx]
                bestChirpmass = bestMasses[0] * (bestMasses[1])**(3./5.)
    return xiextrema

# Copyright (C) 2013 Ian W. Harry
#
#
#

from pycbc.tmpltbank.lambda_mapping import generate_mapping

def determine_eigen_directions(metricParams, preserveMoments=False,
                               vary_fmax=False, vary_density=None):
    """
    This function will calculate the coordinate transfomations that are needed
    to rotate from a coordinate system described by the various Lambda
    components in the frequency expansion, to a coordinate system where the
    metric is Cartesian.

        Structure holding all the options for construction of the metric.
    preserveMoments : boolean, optional (default False)
        Currently only used for debugging.
        If this is given then if the moments structure is already set
        within metricParams then they will not be recalculated.
    vary_fmax : boolean, optional (default False)
        If set to False the metric and rotations are calculated once, for the
        full range of frequency [f_low,f_upper).
        If set to True the metric and rotations are calculated multiple times,
        for frequency ranges [f_low,f_low + i*vary_density), where i starts at
        1 and runs up until f_low + (i+1)*vary_density > f_upper.
        Thus values greater than f_upper are *not* computed.
        The calculation for the full range [f_low,f_upper) is also done.
    vary_density : float, optional
        If vary_fmax is True, this will be used in computing the frequency
        ranges as described for vary_fmax.

    Returns
    --------
        **THIS FUNCTION ONLY RETURNS THE CLASS**
        The following will be **added** to this structure
    metricParams.evals : Dictionary of numpy.array
        Each entry in the dictionary corresponds to the different frequency
        ranges described in vary_fmax. If vary_fmax = False, the only entry
        will be f_upper, this corresponds to integrals in [f_low,f_upper). This
        entry is always present. Each other entry will use floats as keys to
        the dictionary. These floats give the upper frequency cutoff when it is
        varying.
        Each numpy.array contains the eigenvalues which, with the eigenvectors
        in evecs, are needed to rotate the
        coordinate system to one in which the metric is the identity matrix.
    metricParams.evecs : Dictionary of numpy.matrix
        Each entry in the dictionary is as described under evals.
        Each numpy.matrix contains the eigenvectors which, with the eigenvalues
        in evals, are needed to rotate the
    metricParams.metric : Dictionary of numpy.matrix
        Each numpy.matrix contains the metric of the parameter space in the
        Lambda_i coordinate system.
    metricParams.moments : Moments structure
        See the structure documentation for a description of this. This
        contains the result of all the integrals used in computing the metrics
        above. It can be used for the ethinca components calculation, or other
        similar calculations.
    """

    evals = {}
    evecs = {}
    metric = {}
    unmax_metric = {}

    # First step is to get the moments needed to calculate the metric
    if not (metricParams.moments and preserveMoments):
        get_moments(metricParams, vary_fmax=vary_fmax,
                    vary_density=vary_density)

    # What values are going to be in the moments
    # J7 is the normalization factor so it *MUST* be present
    list = metricParams.moments['J7'].keys()

    # We start looping over every item in the list of metrics
    for item in list:
        # Here we convert the moments into a form easier to use here
        Js = {}
        for i in range(-7,18):
            Js[i] = metricParams.moments['J%d'%(i)][item]

        logJs = {}
        for i in range(-1,18):
            logJs[i] = metricParams.moments['log%d'%(i)][item]

        loglogJs = {}
            loglogJs[i] = metricParams.moments['loglog%d'%(i)][item]

        logloglogJs = {}
            logloglogJs[i] = metricParams.moments['logloglog%d'%(i)][item]

        loglogloglogJs = {}
            loglogloglogJs[i] = metricParams.moments['loglogloglog%d'%(i)][item]

        mapping = generate_mapping(metricParams.pnOrder)

        # Calculate the metric
        gs, unmax_metric_curr = calculate_metric(Js, logJs, loglogJs,
                                          logloglogJs, loglogloglogJs, mapping)
        metric[item] = numpy.matrix(gs)
        unmax_metric[item] = unmax_metric_curr

        # And the eigenvalues
        evals[item],evecs[item] = numpy.linalg.eig(gs)

        # Numerical error can lead to small negative eigenvalues.
        for i in range(len(evals[item])):
            if evals[item][i] < 0:
                # Due to numerical imprecision the very small eigenvalues can
                # be negative. Make these positive.
                evals[item][i] = -evals[item][i]
            if evecs[item][i,i] < 0:
                # We demand a convention that all diagonal terms in the matrix
                # of eigenvalues are positive.
                # This is done to help visualization of the spaces (increasing
                # mchirp always goes the same way)
                evecs[item][:,i] = - evecs[item][:,i]

    metricParams.evals = evals
    metricParams.evecs = evecs
    metricParams.metric = metric
    metricParams.time_unprojected_metric = unmax_metric

    return metricParams

def get_moments(metricParams, vary_fmax=False, vary_density=None):
    """
    This function will calculate the various integrals (moments) that are
    needed to compute the metric used in template bank placement and
    coincidence.

    Returns
    --------
    None : None
        **THIS FUNCTION RETURNS NOTHING**
        The following will be **added** to the metricParams structure
        This contains the result of all the integrals used in computing the
        metrics above. It can be used for the ethinca components calculation,
        or other similar calculations. This is composed of two compound
        dictionaries. The first entry indicates which moment is being
        calculated and the second entry indicates the upper frequency cutoff
        that was used.

        In all cases x = f/f0.

        For the first entries the options are:

        moments['J%d' %(i)][f_cutoff]
        This stores the integral of
        x**((-i)/3.) * delta X / PSD(x)

        moments['log%d' %(i)][f_cutoff]
        (numpy.log(x**(1./3.))) x**((-i)/3.) * delta X / PSD(x)

        moments['loglog%d' %(i)][f_cutoff]
        (numpy.log(x**(1./3.)))**2 x**((-i)/3.) * delta X / PSD(x)

        (numpy.log(x**(1./3.)))**3 x**((-i)/3.) * delta X / PSD(x)

        (numpy.log(x**(1./3.)))**4 x**((-i)/3.) * delta X / PSD(x)

        The second entry stores the frequency cutoff used when computing
        the integral. See description of the vary_fmax option above.

        All of these values are nomralized by a factor of

        x**((-7)/3.) * delta X / PSD(x)

        The normalization factor can be obtained in

        moments['I7'][f_cutoff]
    """
    # NOTE: Unless the TaylorR2F4 metric is used the log^3 and log^4 terms are
    # not needed. As this calculation is not too slow compared to bank
    # placement we just do this anyway.

    psd_amp = metricParams.psd.data
    psd_f = numpy.arange(len(psd_amp), dtype=float) * metricParams.deltaF
    new_f, new_amp = interpolate_psd(psd_f, psd_amp, metricParams.deltaF)

    # Need I7 first as this is the normalization factor
    funct = lambda x,f0: 1
    I7 = calculate_moment(new_f, new_amp, metricParams.fLow, \
                          metricParams.fUpper, metricParams.f0, funct,\
                          vary_fmax=vary_fmax, vary_density=vary_density)

    # Do all the J moments
    moments = {}
    moments['I7'] = I7
    for i in range(-7,18):
        funct = lambda x,f0: x**((-i+7)/3.)
        moments['J%d' %(i)] = calculate_moment(new_f, new_amp, \
                                metricParams.fLow, metricParams.fUpper, \
                                metricParams.f0, funct, norm=I7, \
                                vary_fmax=vary_fmax, vary_density=vary_density)

    # Do the logx multiplied by some power terms
    for i in range(-1,18):
        funct = lambda x,f0: (numpy.log((x*f0)**(1./3.))) * x**((-i+7)/3.)
        moments['log%d' %(i)] = calculate_moment(new_f, new_amp, \

    # Do the loglog term
        funct = lambda x,f0: (numpy.log((x*f0)**(1./3.)))**2 * x**((-i+7)/3.)
        moments['loglog%d' %(i)] = calculate_moment(new_f, new_amp, \

    # Do the logloglog term
        funct = lambda x,f0: (numpy.log((x*f0)**(1./3.)))**3 * x**((-i+7)/3.)
        moments['logloglog%d' %(i)] = calculate_moment(new_f, new_amp, \

        funct = lambda x,f0: (numpy.log((x*f0)**(1./3.)))**4 * x**((-i+7)/3.)
        moments['loglogloglog%d' %(i)] = calculate_moment(new_f, new_amp, \

    metricParams.moments = moments

def interpolate_psd(psd_f, psd_amp, deltaF):
    """
    Function to interpolate a PSD to a different value of deltaF. Uses linear
    interpolation.

    psd_f : numpy.array or list or similar
        List of the frequencies contained within the PSD.
    psd_amp : numpy.array or list or similar
        List of the PSD values at the frequencies in psd_f.
    deltaF : float
        Value of deltaF to interpolate the PSD to.

    Returns
    --------
    new_psd_f : numpy.array
       Array of the frequencies contained within the interpolated PSD
    new_psd_amp : numpy.array
       Array of the interpolated PSD values at the frequencies in new_psd_f.
    """
    # In some cases this will be a no-op. I thought about removing this, but
    # this function can take unequally sampled PSDs and it is difficult to
    # check for this. As this function runs quickly anyway (compared to the
    # moment calculation) I decided to always interpolate.

    new_psd_f = []
    new_psd_amp = []
    fcurr = psd_f[0]

    for i in range(len(psd_f) - 1):
        f_low = psd_f[i]
        f_high = psd_f[i+1]
        amp_low = psd_amp[i]
        amp_high = psd_amp[i+1]
        while(1):
            if fcurr > f_high:
                break
            new_psd_f.append(fcurr)
            gradient = (amp_high - amp_low) / (f_high - f_low)
            fDiff = fcurr - f_low
            new_psd_amp.append(amp_low + fDiff * gradient)
            fcurr = fcurr + deltaF
    return numpy.asarray(new_psd_f), numpy.asarray(new_psd_amp)

def calculate_moment(psd_f, psd_amp, fmin, fmax, f0, funct,
                     norm=None, vary_fmax=False, vary_density=None):
    """
    Function for calculating one of the integrals used to construct a template
    bank placement metric. The integral calculated will be

    \int funct(x) * (psd_x)**(-7./3.) * delta_x / PSD(x)

    where x = f / f0. The lower frequency cutoff is given by fmin, see
    the parameters below for details on how the upper frequency cutoff is
    chosen

    psd_f : numpy.array
       numpy array holding the set of evenly spaced frequencies used in the PSD
    psd_amp : numpy.array
       numpy array holding the PSD values corresponding to the psd_f
       frequencies
    fmin : float
        The lower frequency cutoff used in the calculation of the integrals
        used to obtain the metric.
    fmax : float
        The upper frequency cutoff used in the calculation of the integrals
        used to obtain the metric. This can be varied (see the vary_fmax
        option below).
    f0 : float
        This is an arbitrary scaling factor introduced to avoid the potential
        for numerical overflow when calculating this. Generally the default
        value (70) is safe here. **IMPORTANT, if you want to calculate the
        ethinca metric components later this MUST be set equal to f_low.**
    funct : Lambda function
        The function to use when computing the integral as described above.
    norm : Dictionary of floats
        If given then moment[f_cutoff] will be divided by norm[f_cutoff]

    Returns
    --------
    moment : Dictionary of floats
        moment[f_cutoff] will store the value of the moment at the frequency
        cutoff given by f_cutoff.
    """

    # Must ensure deltaF in psd_f is constant
    psd_x = psd_f / f0
    deltax = psd_x[1] - psd_x[0]

    mask = numpy.logical_and(psd_f > fmin, psd_f < fmax)
    psdf_red = psd_f[mask]
    comps_red = psd_x[mask] ** (-7./3.) * funct(psd_x[mask], f0) * deltax / \
                psd_amp[mask]
    moment = {}
    moment[fmax] = comps_red.sum()
    if norm:
        moment[fmax] = moment[fmax] / norm[fmax]
    if vary_fmax:
        for t_fmax in numpy.arange(fmin + vary_density, fmax, vary_density):
            moment[t_fmax] = comps_red[psdf_red < t_fmax].sum()
            if norm:
                moment[t_fmax] = moment[t_fmax] / norm[t_fmax]
    return moment

def calculate_metric(Js, logJs, loglogJs, logloglogJs, loglogloglogJs, \
                     mapping):
    """
    This function will take the various integrals calculated by get_moments and
    convert this into a metric for the appropriate parameter space.

    Js : Dictionary
        The list of (log^0 x) * x**(-i/3) integrals computed by get_moments()
        The index is Js[i]
    logJs : Dictionary
        The list of (log^1 x) * x**(-i/3) integrals computed by get_moments()
        The index is logJs[i]
    loglogJs : Dictionary
        The list of (log^2 x) * x**(-i/3) integrals computed by get_moments()
        The index is loglogJs[i]
    logloglogJs : Dictionary
        The list of (log^3 x) * x**(-i/3) integrals computed by get_moments()
        The index is logloglogJs[i]
    loglogloglogJs : Dictionary
        The list of (log^4 x) * x**(-i/3) integrals computed by get_moments()
        The index is loglogloglogJs[i]
    mapping : dictionary
        Used to identify which Lambda components are active in this parameter
        space and map these to entries in the metric matrix.

    Returns
    --------
    metric : numpy.matrix
        The resulting metric.
    """

    # How many dimensions in the parameter space?
    maxLen = len(mapping.keys())

    metric = numpy.matrix(numpy.zeros(shape=(maxLen,maxLen),dtype=float))
    unmax_metric = numpy.matrix(numpy.zeros(shape=(maxLen+1,maxLen+1),
                                                                  dtype=float))

    for i in range(16):
        for j in range(16):
            calculate_metric_comp(metric, unmax_metric, i, j, Js,
                                           logJs, loglogJs, logloglogJs,
                                           loglogloglogJs, mapping)
    return metric, unmax_metric

def calculate_metric_comp(gs, unmax_metric, i, j, Js, logJs, loglogJs,
                          logloglogJs, loglogloglogJs, mapping):
    """
    Used to compute part of the metric. Only call this from within
    calculate_metric(). Please see the documentation for that function.
    """
    # Time term in unmax_metric. Note that these terms are recomputed a bunch
    # of time, but this cost is insignificant compared to computing the moments
    unmax_metric[-1,-1] = (Js[1] - Js[4]*Js[4])

    # Normal terms
    if 'Lambda%d'%i in mapping and 'Lambda%d'%j in mapping:
        gammaij = Js[17-i-j] - Js[12-i]*Js[12-j]
        gamma0i = (Js[9-i] - Js[4]*Js[12-i])
        gamma0j = (Js[9-j] - Js[4] * Js[12-j])
        gs[mapping['Lambda%d'%i],mapping['Lambda%d'%j]] = \
            0.5 * (gammaij - gamma0i*gamma0j/(Js[1] - Js[4]*Js[4]))
        unmax_metric[mapping['Lambda%d'%i], -1] = gamma0i
        unmax_metric[-1, mapping['Lambda%d'%j]] = gamma0j
        unmax_metric[mapping['Lambda%d'%i],mapping['Lambda%d'%j]] = gammaij
    # Normal,log cross terms
    if 'Lambda%d'%i in mapping and 'LogLambda%d'%j in mapping:
        gammaij = logJs[17-i-j] - logJs[12-j] * Js[12-i]
        gamma0i = (Js[9-i] - Js[4] * Js[12-i])
        gamma0j = logJs[9-j] - logJs[12-j] * Js[4]
        gs[mapping['Lambda%d'%i],mapping['LogLambda%d'%j]] = \
            gs[mapping['LogLambda%d'%j],mapping['Lambda%d'%i]] = \
        unmax_metric[-1, mapping['Lambda%d'%i]] = gamma0i
        unmax_metric[-1, mapping['LogLambda%d'%j]] = gamma0j
        unmax_metric[mapping['LogLambda%d'%j], -1] = gamma0j
        unmax_metric[mapping['Lambda%d'%i],mapping['LogLambda%d'%j]] = gammaij
        unmax_metric[mapping['LogLambda%d'%j],mapping['Lambda%d'%i]] = gammaij
    # Log,log terms
    if 'LogLambda%d'%i in mapping and 'LogLambda%d'%j in mapping:
        gammaij = loglogJs[17-i-j] - logJs[12-j] * logJs[12-i]
        gamma0i = (logJs[9-i] - Js[4] * logJs[12-i])
        gs[mapping['LogLambda%d'%i],mapping['LogLambda%d'%j]] = \
        unmax_metric[mapping['LogLambda%d'%i], -1] = gamma0i
        unmax_metric[mapping['LogLambda%d'%i],mapping['LogLambda%d'%j]] =\
            gammaij

    # Normal,loglog cross terms
    if 'Lambda%d'%i in mapping and 'LogLogLambda%d'%j in mapping:
        gammaij = loglogJs[17-i-j] - loglogJs[12-j] * Js[12-i]
        gamma0j = loglogJs[9-j] - loglogJs[12-j] * Js[4]
        gs[mapping['Lambda%d'%i],mapping['LogLogLambda%d'%j]] = \
            gs[mapping['LogLogLambda%d'%j],mapping['Lambda%d'%i]] = \
        unmax_metric[-1, mapping['LogLogLambda%d'%j]] = gamma0j
        unmax_metric[mapping['LogLogLambda%d'%j], -1] = gamma0j
        unmax_metric[mapping['Lambda%d'%i],mapping['LogLogLambda%d'%j]] = \
            gammaij
        unmax_metric[mapping['LogLogLambda%d'%j],mapping['Lambda%d'%i]] = \
            gammaij

    # log,loglog cross terms
    if 'LogLambda%d'%i in mapping and 'LogLogLambda%d'%j in mapping:
        gammaij = logloglogJs[17-i-j] - loglogJs[12-j] * logJs[12-i]
        gs[mapping['LogLambda%d'%i],mapping['LogLogLambda%d'%j]] = \
            gs[mapping['LogLogLambda%d'%j],mapping['LogLambda%d'%i]] = \
        unmax_metric[-1, mapping['LogLambda%d'%i]] = gamma0i
        unmax_metric[mapping['LogLambda%d'%i],mapping['LogLogLambda%d'%j]] = \
            gammaij
        unmax_metric[mapping['LogLogLambda%d'%j],mapping['LogLambda%d'%i]] = \
            gammaij

    # Loglog,loglog terms
    if 'LogLogLambda%d'%i in mapping and 'LogLogLambda%d'%j in mapping:
        gammaij = loglogloglogJs[17-i-j] - loglogJs[12-j] * loglogJs[12-i]
        gamma0i = (loglogJs[9-i] - Js[4] * loglogJs[12-i])
        gs[mapping['LogLogLambda%d'%i],mapping['LogLogLambda%d'%j]] = \
        unmax_metric[mapping['LogLogLambda%d'%i], -1] = gamma0i
        unmax_metric[mapping['LogLogLambda%d'%i],mapping['LogLogLambda%d'%j]] =\
            gammaij

#
#
#

from pycbc.tmpltbank.lambda_mapping import get_chirp_params
from pycbc.tmpltbank.em_progenitors import load_ns_sequence, remnant_masses

def estimate_mass_range(numPoints, massRangeParams, metricParams, fUpper,\
                        covary=True):
    """
    This function will generate a large set of points with random masses and
    spins (using pycbc.tmpltbank.get_random_mass) and translate these points
    into the xi_i coordinate system for the given upper frequency cutoff.

    numPoints : int
        Number of systems to simulate
        The value of fUpper to use when getting the mu coordinates from the
        lambda coordinates. This must be a key in metricParams.evals and
        metricParams.evecs (ie. we must know how to do the transformation for
        the given value of fUpper). It also must be a key in
        metricParams.evecsCV if covary=True.
    covary : boolean, optional (default = True)
        If this is given then evecsCV will be used to rotate from the Cartesian
        coordinate system into the principal coordinate direction (xi_i). If
        not given then points in the original Cartesian coordinates are
        returned.

    Returns
    -------
    xis : numpy.array
        A list of the positions of each point in the xi_i coordinate system.
    """
    vals_set = get_random_mass(numPoints, massRangeParams)
    mass1 = vals_set[0]
    mass2 = vals_set[1]
    spin1z = vals_set[2]
    spin2z = vals_set[3]
    if covary:
        lambdas = get_cov_params(mass1, mass2, spin1z, spin2z, metricParams,
                                 fUpper)
    else:
        lambdas = get_conv_params(mass1, mass2, spin1z, spin2z, metricParams,
                                  fUpper)

    return numpy.array(lambdas)

def get_random_mass_point_particles(numPoints, massRangeParams):
    """
    This function will generate a large set of points within the chosen mass
    and spin space. It will also return the corresponding PN spin coefficients
    for ease of use later (though these may be removed at some future point).

    Returns
    --------
        Mass of heavier body.
        Mass of lighter body.
        Spin of body 1.
        Spin of body 2.
    """

    # WARNING: We expect mass1 > mass2 ALWAYS

    # First we choose the total masses from a unifrom distribution in mass
    # to the -5/3. power.
    mass = numpy.random.random(numPoints) * \
           (massRangeParams.minTotMass**(-5./3.) \
            - massRangeParams.maxTotMass**(-5./3.)) \
           + massRangeParams.maxTotMass**(-5./3.)
    mass = mass**(-3./5.)

    # Next we choose the mass ratios, this will take different limits based on
    # the value of total mass
    maxmass2 = numpy.minimum(mass/2., massRangeParams.maxMass2)
    minmass1 = numpy.maximum(massRangeParams.minMass1, mass/2.)
    mineta = numpy.maximum(massRangeParams.minCompMass \
                            * (mass-massRangeParams.minCompMass)/(mass*mass), \
                           massRangeParams.maxCompMass \
                            * (mass-massRangeParams.maxCompMass)/(mass*mass))
    # Note that mineta is a numpy.array because mineta depends on the total
    # mass. Therefore this is not precomputed in the massRangeParams instance
        mineta = numpy.maximum(massRangeParams.minEta, mineta)
    # Eta also restricted by chirp mass restrictions
        eta_val_at_min_chirp = massRangeParams.min_chirp_mass / mass
        eta_val_at_min_chirp = eta_val_at_min_chirp**(5./3.)
        mineta = numpy.maximum(mineta, eta_val_at_min_chirp)

    maxeta = numpy.minimum(massRangeParams.maxEta, maxmass2 \
                             * (mass - maxmass2) / (mass*mass))
    maxeta = numpy.minimum(maxeta, minmass1 \
                             * (mass - minmass1) / (mass*mass))
    # max eta also affected by chirp mass restrictions
        eta_val_at_max_chirp = massRangeParams.max_chirp_mass / mass
        eta_val_at_max_chirp = eta_val_at_max_chirp**(5./3.)
        maxeta = numpy.minimum(maxeta, eta_val_at_max_chirp)

    if (maxeta < mineta).any():
        errMsg = "ERROR: Maximum eta is smaller than minimum eta!!"
        raise ValueError(errMsg)
    eta = numpy.random.random(numPoints) * (maxeta - mineta) + mineta

    # Also calculate the component masses; mass1 > mass2
    diff = (mass*mass * (1-4*eta))**0.5
    mass1 = (mass + diff)/2.
    mass2 = (mass - diff)/2.
    # Check the masses are where we want them to be (allowing some floating
    # point rounding error).
    if (mass1 > massRangeParams.maxMass1*1.001).any() \
          or (mass1 < massRangeParams.minMass1*0.999).any():
        errMsg = "Mass1 is not within the specified mass range."
    if (mass2 > massRangeParams.maxMass2*1.001).any() \
          or (mass2 < massRangeParams.minMass2*0.999).any():
        errMsg = "Mass2 is not within the specified mass range."

    # Next up is the spins. First check if we have non-zero spins
    if massRangeParams.maxNSSpinMag == 0 and massRangeParams.maxBHSpinMag == 0:
        spin1z = numpy.zeros(numPoints,dtype=float)
        spin2z = numpy.zeros(numPoints,dtype=float)
    elif massRangeParams.nsbhFlag:
        # Spin 1 first
        mspin = numpy.zeros(len(mass1))
        mspin += massRangeParams.maxBHSpinMag
        spin1z = (2*numpy.random.random(numPoints) - 1) * mspin
        # Then spin2
        mspin = numpy.zeros(len(mass2))
        mspin += massRangeParams.maxNSSpinMag
        spin2z = (2*numpy.random.random(numPoints) - 1) * mspin
    else:        
        mspin[mass1 > boundary_mass] = massRangeParams.maxBHSpinMag
        # Then spin 2
        mspin[mass2 > boundary_mass] = massRangeParams.maxBHSpinMag

    return mass1, mass2, spin1z, spin2z

def get_random_mass(numPoints, massRangeParams):
    """
    and spin space, and with the desired minimum remnant disk mass (this applies
    to NS-BH systems only). It will also return the corresponding PN spin
    coefficients for ease of use later (though these may be removed at some
    future point).

    Returns
    --------
    """

    # Check if EM contraints are required, i.e. if the systems must produce
    # a minimum remnant disk mass.  If this is not the case, proceed treating
    # the systems as point particle binaries
    if massRangeParams.remnant_mass_threshold is None:
        mass1, mass2, spin1z, spin2z = \
            get_random_mass_point_particles(numPoints, massRangeParams)
    # otherwise, load EOS dependent data, generate the EM constraint
    # (i.e. compute the minimum symmetric mass ratio needed to
    # generate a given remnant disk mass as a function of the NS
    # mass and the BH spin along z) and then proceed by accepting
    # only systems that can yield (at least) the desired remnant
    # disk mass and that pass the mass and spin range cuts.
    else:
        ns_sequence, max_ns_g_mass = load_ns_sequence(massRangeParams.ns_eos)
        if max_ns_g_mass < boundary_mass:
            warn_msg = "WARNING: "
            warn_msg += "Option of ns-bh-boundary-mass is  %s " %(boundary_mass)
            warn_msg += "which is higher than the maximum NS gravitational "
            warn_msg += "mass admitted by the EOS that was prescribed "
            warn_msg += "(%s). " %(max_ns_g_mass)
            warn_msg += "The code will proceed using the latter value "
            warn_msg += "as the boundary mass."
            logging.warn(warn_msg)
            boundary_mass = max_ns_g_mass

        # Empty arrays to store points that pass all cuts
        mass1_out = []
        mass2_out = []
        spin1z_out = []
        spin2z_out = []

        # As the EM cut can remove several randomly generated
        # binaries, track the number of accepted points that pass
        # all cuts and stop only once enough of them are generated
        numPointsFound = 0
        while numPointsFound < numPoints:
            # Generate the random points within the required mass
            # and spin cuts
            mass1, mass2, spin1z, spin2z = \
                get_random_mass_point_particles(numPoints-numPointsFound,
                                                massRangeParams)

            # Now proceed with cutting out EM dim systems
            # Use a logical mask to track points that do not correspond to
            # BBHs. The remaining points will be BNSs and NSBHs.
            # Further down, EM-dim NSBHs will also be removed.
            mask_not_bbh = numpy.zeros(len(mass1), dtype=bool)

            # Keep a point if:
            # 1) the secondary object is a not a BH (mass2 < boundary mass)
            #    [Store masses and spins of non BBH]
            mask_not_bbh[mass2 < boundary_mass] = True
            mass1_not_bbh = mass1[mask_not_bbh]
            mass2_not_bbh = mass2[mask_not_bbh]
            spin1z_not_bbh = spin1z[mask_not_bbh]
            spin2z_not_bbh = spin2z[mask_not_bbh]
            # 2) and if the primary mass is a NS (i.e., it is a BNS), or...
            mask_nsbh = numpy.zeros(len(mass1_not_bbh), dtype=bool)
            #    [mask_nsbh identifies NSBH systems]
            mask_nsbh[mass1_not_bbh > boundary_mass] = True
            #    [mask_bns identifies BNS systems]
            mask_bns = ~mask_nsbh
            #    [Store masses and spins of BNSs]
            mass1_bns = mass1_not_bbh[mask_bns]
            mass2_bns = mass2_not_bbh[mask_bns]
            spin1z_bns = spin1z_not_bbh[mask_bns]
            spin2z_bns = spin2z_not_bbh[mask_bns]
            # 3) ...it is an NS-BH with remnant mass greater than the threshold
            #    required to have a counterpart
            #    [Store masses and spins of all NSBHs]
            mass1_nsbh = mass1_not_bbh[mask_nsbh]
            mass2_nsbh = mass2_not_bbh[mask_nsbh]
            spin1z_nsbh = spin1z_not_bbh[mask_nsbh]
            spin2z_nsbh = spin2z_not_bbh[mask_nsbh]
            #    [Store etas of all NSBHs]
            eta_nsbh = conversions.eta_from_mass1_mass2(mass1_nsbh, mass2_nsbh)
            #    [mask_bright_nsbh will identify NSBH systems with high enough
            #     threshold mass]
            mask_bright_nsbh = numpy.zeros(len(mass1_nsbh), dtype=bool)
            if eta_nsbh.size != 0:
                remnant = remnant_masses(eta_nsbh, mass2_nsbh, ns_sequence,
                                         spin1z_nsbh, 0.)
                mask_bright_nsbh[remnant
                                 >
                                 massRangeParams.remnant_mass_threshold] = True

            # Keep only points that correspond to binaries that can produce an
            # EM counterpart (i.e., BNSs and EM-bright NSBHs) and add their
            # properties to the pile of accpeted points to output
            mass1_out = numpy.concatenate((mass1_out, mass1_bns,
                                           mass1_nsbh[mask_bright_nsbh]))
            mass2_out = numpy.concatenate((mass2_out, mass2_bns,
                                           mass2_nsbh[mask_bright_nsbh]))
            spin1z_out = numpy.concatenate((spin1z_out, spin1z_bns,
                                            spin1z_nsbh[mask_bright_nsbh]))
            spin2z_out = numpy.concatenate((spin2z_out, spin2z_bns,
                                            spin2z_nsbh[mask_bright_nsbh]))

            # Number of points that survived all cuts
            numPointsFound = len(mass1_out)

        # Ready to go
        mass1 = mass1_out
        mass2 = mass2_out
        spin1z = spin1z_out
        spin2z = spin2z_out

def get_cov_params(mass1, mass2, spin1z, spin2z, metricParams, fUpper,
                   lambda1=None, lambda2=None, quadparam1=None,
                   quadparam2=None):
    """
    Function to convert between masses and spins and locations in the xi
    parameter space. Xi = Cartesian metric and rotated to principal components.

        lambda coordinates. This must be a key in metricParams.evals,

    Returns
    --------
    xis : list of floats or numpy.arrays
        Position of the system(s) in the xi coordinate system
    """

    # Do this by doing masses - > lambdas -> mus
    mus = get_conv_params(mass1, mass2, spin1z, spin2z, metricParams, fUpper,
                          lambda1=lambda1, lambda2=lambda2,
                          quadparam1=quadparam1, quadparam2=quadparam2)
    # and then mus -> xis
    xis = get_covaried_params(mus, metricParams.evecsCV[fUpper])
    return xis

def get_conv_params(mass1, mass2, spin1z, spin2z, metricParams, fUpper,
                    lambda1=None, lambda2=None, quadparam1=None,
                    quadparam2=None):
    """
    Function to convert between masses and spins and locations in the mu
    parameter space. Mu = Cartesian metric, but not principal components.

    Returns
    --------
    mus : list of floats or numpy.arrays
        Position of the system(s) in the mu coordinate system
    """

    # Do this by masses -> lambdas
    lambdas = get_chirp_params(mass1, mass2, spin1z, spin2z,
                               metricParams.f0, metricParams.pnOrder,
                               lambda1=lambda1, lambda2=lambda2,
                               quadparam1=quadparam1, quadparam2=quadparam2)
    # and lambdas -> mus
    mus = get_mu_params(lambdas, metricParams, fUpper)
    return mus

def get_mu_params(lambdas, metricParams, fUpper):
    """
    Function to rotate from the lambda coefficients into position in the mu
    coordinate system. Mu = Cartesian metric, but not principal components.

    lambdas : list of floats or numpy.arrays
        Position of the system(s) in the lambda coefficients

    Returns
    --------
    """
    lambdas = numpy.array(lambdas, copy=False)
    # If original inputs were floats we need to make this a 2D array
    if len(lambdas.shape) == 1:
        resize_needed = True
        lambdas = lambdas[:,None]
    else:
        resize_needed = False

    evecs = metricParams.evecs[fUpper]
    evals = metricParams.evals[fUpper]

    evecs = numpy.array(evecs, copy=False)

    mus = ((lambdas.T).dot(evecs)).T
    mus = mus * numpy.sqrt(evals)[:,None]

    if resize_needed:
        mus = numpy.ndarray.flatten(mus)

    return mus

def get_covaried_params(mus, evecsCV):
    """
    Function to rotate from position(s) in the mu_i coordinate system into the
    position(s) in the xi_i coordinate system

    evecsCV : numpy.matrix
        This matrix is used to perform the rotation to the xi_i
        coordinate system.

    Returns
    --------
    """
    mus = numpy.array(mus, copy=False)
    if len(mus.shape) == 1:
        mus = mus[:,None]
    else:

    xis = ((mus.T).dot(evecsCV)).T

        xis = numpy.ndarray.flatten(xis)

    return xis

def rotate_vector(evecs, old_vector, rescale_factor, index):
    """
    Function to find the position of the system(s) in one of the xi_i or mu_i
    directions.

    evecs : numpy.matrix
        Matrix of the eigenvectors of the metric in lambda_i coordinates. Used
        to rotate to a Cartesian coordinate system.
    old_vector : list of floats or numpy.arrays
        The position of the system(s) in the original coordinates
    rescale_factor : float
        Scaling factor to apply to resulting position(s)
    index : int
        The index of the final coordinate system that is being computed. Ie.
        if we are going from mu_i -> xi_j, this will give j.

    Returns
    --------
    positions : float or numpy.array
        Position of the point(s) in the resulting coordinate.
    """
    temp = 0
    for i in range(len(evecs)):
        temp += (evecs[i,index] * rescale_factor) * old_vector[i]
    return temp

def get_point_distance(point1, point2, metricParams, fUpper):
    """
    Function to calculate the mismatch between two points, supplied in terms
    of the masses and spins, using the xi_i parameter space metric to
    approximate the mismatch of the two points. Can also take one of the points
    as an array of points and return an array of mismatches (but only one can
    be an array!)

    point1 : List of floats or numpy.arrays
        point1[0] contains the mass(es) of the heaviest body(ies).
        point1[1] contains the mass(es) of the smallest body(ies).
        point1[2] contains the spin(es) of the heaviest body(ies).
        point1[3] contains the spin(es) of the smallest body(ies).
    point2 : List of floats
        point2[0] contains the mass of the heaviest body.
        point2[1] contains the mass of the smallest body.
        point2[2] contains the spin of the heaviest body.
        point2[3] contains the spin of the smallest body.

    Returns
    --------
    dist : float or numpy.array
        Distance between the point2 and all points in point1
    xis1 : List of floats or numpy.arrays
        Position of the input point1(s) in the xi_i parameter space
    xis2 : List of floats
        Position of the input point2 in the xi_i parameter space
    """
    aMass1 = point1[0]
    aMass2 = point1[1]
    aSpin1 = point1[2]
    aSpin2 = point1[3]

    bMass1 = point2[0]
    bMass2 = point2[1]
    bSpin1 = point2[2]
    bSpin2 = point2[3]

    aXis = get_cov_params(aMass1, aMass2, aSpin1, aSpin2, metricParams, fUpper)

    bXis = get_cov_params(bMass1, bMass2, bSpin1, bSpin2, metricParams, fUpper)

    dist = (aXis[0] - bXis[0])**2
    for i in range(1,len(aXis)):
        dist += (aXis[i] - bXis[i])**2

    return dist, aXis, bXis

def calc_point_dist(vsA, entryA):
    """
    This function is used to determine the distance between two points.

    vsA : list or numpy.array or similar
        An array of point 1's position in the \chi_i coordinate system
    entryA : list or numpy.array or similar
        An array of point 2's position in the \chi_i coordinate system
    MMdistA : float
        The minimal mismatch allowed between the points

    Returns
    --------
    val : float
        The metric distance between the two points.
    """
    chi_diffs = vsA - entryA
    val = ((chi_diffs)*(chi_diffs)).sum()
    return val 

def test_point_dist(point_1_chis, point_2_chis, distance_threshold):
    """
    This function tests if the difference between two points in the chi
    parameter space is less than a distance threshold. Returns True if it is
    and False if it is not.   

    point_1_chis : numpy.array
    point_2_chis : numpy.array
    distance_threshold : float
        The distance threshold to use.
    """
    return calc_point_dist(point_1_chis, point_2_chis) < distance_threshold

def calc_point_dist_vary(mus1, fUpper1, mus2, fUpper2, fMap, norm_map, MMdistA):
    """
    Function to determine if two points, with differing upper frequency cutoffs
    have a mismatch < MMdistA for *both* upper frequency cutoffs.

    mus1 : List of numpy arrays
        mus1[i] will give the array of point 1's position in the \chi_j
        coordinate system. The i element corresponds to varying values of the
        upper frequency cutoff. fMap is used to map between i and actual
        frequencies
    fUpper1 : float
        The upper frequency cutoff of point 1.
    mus2 : List of numpy arrays
        mus2[i] will give the array of point 2's position in the \chi_j
    fUpper2 : float
        The upper frequency cutoff of point 2.
    fMap : dictionary
        fMap[fUpper] will give the index needed to get the \chi_j coordinates
        in the two sets of mus
    norm_map : dictionary
        norm_map[fUpper] will give the relative frequency domain template
        amplitude (sigma) at the given value of fUpper.
    MMdistA

    Returns
    --------
    Boolean
        True if the points have a mismatch < MMdistA
        False if the points have a mismatch > MMdistA
    """
    f_upper = min(fUpper1, fUpper2)
    f_other = max(fUpper1, fUpper2)
    idx = fMap[f_upper]
    vecs1 = mus1[idx]
    vecs2 = mus2[idx]
    val = ((vecs1 - vecs2)*(vecs1 - vecs2)).sum()
    if (val > MMdistA):
    # Reduce match to account for normalization.
    norm_fac = norm_map[f_upper] / norm_map[f_other]
    val = 1 - (1 - val)*norm_fac
    return (val < MMdistA)

def find_max_and_min_frequencies(name, mass_range_params, freqs):
    """
    ADD DOCS
    """

    cutoff_fns = pnutils.named_frequency_cutoffs
    if name not in cutoff_fns.keys():
        err_msg = "%s not recognized as a valid cutoff frequency choice." %name
        err_msg += "Recognized choices: " + " ".join(cutoff_fns.keys())

    # Can I do this quickly?
    total_mass_approxs = {
        "SchwarzISCO": pnutils.f_SchwarzISCO,
        "LightRing"  : pnutils.f_LightRing,
        "ERD"        : pnutils.f_ERD
    }
    
    if name in total_mass_approxs.keys():
        # This can be done quickly if the cutoff only depends on total mass
        # Assumes that lower total mass = higher cutoff frequency
        upper_f_cutoff = total_mass_approxs[name](mass_range_params.minTotMass)
        lower_f_cutoff = total_mass_approxs[name](mass_range_params.maxTotMass)
    else:
        # Do this numerically
        # FIXME: Is 1000000 the right choice? I think so, but just highlighting
                get_random_mass(1000000, mass_range_params)
        mass_dict = {}
        mass_dict['mass1'] = mass1
        mass_dict['mass2'] = mass2
        mass_dict['spin1z'] = spin1z
        mass_dict['spin2z'] = spin2z
        tmp_freqs = cutoff_fns[name](mass_dict)
        upper_f_cutoff = tmp_freqs.max()
        lower_f_cutoff = tmp_freqs.min()

    cutoffs = numpy.array([lower_f_cutoff,upper_f_cutoff])
    if lower_f_cutoff < freqs.min():
        warn_msg = "WARNING: "
        warn_msg += "Lowest frequency cutoff is %s Hz " %(lower_f_cutoff,)
        warn_msg += "which is lower than the lowest frequency calculated "
        warn_msg += "for the metric: %s Hz. " %(freqs.min())
        warn_msg += "Distances for these waveforms will be calculated at "
        warn_msg += "the lowest available metric frequency."
        logging.warn(warn_msg)
    if upper_f_cutoff > freqs.max():
        warn_msg += "Highest frequency cutoff is %s Hz " %(upper_f_cutoff,)
        warn_msg += "which is larger than the highest frequency calculated "
        warn_msg += "for the metric: %s Hz. " %(freqs.max())
        warn_msg += "the largest available metric frequency."
    return find_closest_calculated_frequencies(cutoffs, freqs)

def return_nearest_cutoff(name, mass_dict, freqs):
    """
    Given an array of total mass values and an (ascending) list of
    frequencies, this will calculate the specified cutoff formula for each
    mtotal and return the nearest frequency to each cutoff from the input
    list.
    Currently only supports cutoffs that are functions of the total mass
    and no other parameters (SchwarzISCO, LightRing, ERD)

        Name of the cutoff formula to be approximated
    mass_dict : Dictionary where the keys are used to call the functions
        returned by tmpltbank.named_frequency_cutoffs. The values can be
        numpy arrays or single values.
    freqs : list of floats
        A list of frequencies (must be sorted ascending)

    Returns
    -------
        The frequencies closest to the cutoff for each value of totmass.
    """
    # A bypass for the redundant case
    if len(freqs) == 1:
        return numpy.zeros(len(mass_dict['m1']), dtype=float) + freqs[0]
    f_cutoff = cutoff_fns[name](mass_dict)
    return find_closest_calculated_frequencies(f_cutoff, freqs)

def find_closest_calculated_frequencies(input_freqs, metric_freqs):
    """
    Given a value (or array) of input frequencies find the closest values in
    the list of frequencies calculated in the metric.

    input_freqs : numpy.array or float
        The frequency(ies) that you want to find the closest value in
        metric_freqs
    metric_freqs : numpy.array
        The list of frequencies calculated by the metric

    Returns
    --------
    output_freqs : numpy.array or float
        The list of closest values to input_freqs for which the metric was
        computed
    """
    try:
        refEv = numpy.zeros(len(input_freqs),dtype=float)
        refEv = numpy.zeros(1, dtype=float)
        input_freqs = numpy.array([input_freqs])

    if len(metric_freqs) == 1:
        refEv[:] = metric_freqs[0]
        return refEv

    # FIXME: This seems complicated for what is a simple operation. Is there
    #        a simpler *and* faster way of doing this?
    # NOTE: This function assumes a sorted list of frequencies
    # NOTE: totmass and f_cutoff are both numpy arrays as this function is
    #       designed so that the cutoff can be calculated for many systems
    #       simulataneously
    for i in range(len(metric_freqs)):
        if i == 0:
            # If frequency is lower than halfway between the first two entries
            # use the first (lowest) value
            logicArr = input_freqs < ((metric_freqs[0] + metric_freqs[1])/2.)
        elif i == (len(metric_freqs)-1):
            # If frequency is larger than halfway between the last two entries
            # use the last (highest) value
            logicArr = input_freqs > ((metric_freqs[-2] + metric_freqs[-1])/2.)
        else:
            # For frequencies within the range in freqs, check which points
            # should use the frequency corresponding to index i.
            logicArrA = input_freqs > ((metric_freqs[i-1] + metric_freqs[i])/2.)
            logicArrB = input_freqs < ((metric_freqs[i] + metric_freqs[i+1])/2.)
            logicArr = numpy.logical_and(logicArrA,logicArrB)
        if logicArr.any():
            refEv[logicArr] = metric_freqs[i]
    return refEv

def outspiral_loop(N):
    """
    Return a list of points that will loop outwards in a 2D lattice in terms
    of distance from a central point. So if N=2 this will be [0,0], [0,1],
    [0,-1],[1,0],[-1,0],[1,1] .... This is useful when you want to loop over
    a number of bins, but want to start in the center and work outwards.
    """
    # Create a 2D lattice of all points
    X,Y = numpy.meshgrid(numpy.arange(-N,N+1), numpy.arange(-N,N+1))

    # Flatten it
    X = numpy.ndarray.flatten(X)
    Y = numpy.ndarray.flatten(Y)

    # Force to an integer
    X = numpy.array(X, dtype=int)
    Y = numpy.array(Y, dtype=int)
   
    # Calculate distances
    G = numpy.sqrt(X**2+Y**2)

    # Combine back into an array
    out_arr = numpy.array([X,Y,G])
   
    # And order correctly
    sorted_out_arr = out_arr[:,out_arr[2].argsort()]

    return sorted_out_arr[:2,:].T

# Copyright (C) 2015 Francesco Pannarale
#
#
#

import scipy.optimize
import sys

##############################################################################
# Innermost Stable Spherical Orbit (ISSO) solver in the Perez-Giz (PG)       #
# formalism [see Stone, Loeb, Berger, PRD 87, 084053 (2013)].                #

# Equation that determines the ISCO radius (in BH mass units)
def ISCO_eq(r, chi):
    """
    Polynomial that enables the calculation of the Kerr
    inntermost stable circular orbit (ISCO) radius via its
    roots.

    r: float
        the radial coordinate in BH mass units
    chi: float
        the BH dimensionless spin parameter

    Returns
    float
        (r*(r-6))**2-chi**2*(2*r*(3*r+14)-9*chi**2)
    """
    return (r*(r-6))**2-chi**2*(2*r*(3*r+14)-9*chi**2)

# Equation that determines the ISSO radius (in BH mass units) at one of the
# poles
def ISSO_eq_at_pole(r, chi):
    """
    Polynomial that enables the calculation of the Kerr polar
    (inclination = +/- pi/2) innermost stable spherical orbit
    (ISSO) radius via its roots.  Physical solutions are
    between 6 and 1+sqrt[3]+sqrt[3+2sqrt[3]].

    r: float
    chi: float

    Returns
    float
        r**3*(r**2*(r-6)+chi**2*(3*r+4))+chi**4*(3*r*(r-2)+chi**2)
    """
    return r**3*(r**2*(r-6)+chi**2*(3*r+4))+chi**4*(3*r*(r-2)+chi**2)

# Equation that determines the ISSO radius (in BH mass units) for a generic
# orbital inclination
def PG_ISSO_eq(r, chi, incl):
    """
    Polynomial that enables the calculation of a generic innermost
    stable spherical orbit (ISSO) radius via its roots.  Physical
    solutions are between the equatorial ISSO (aka the ISCO) radius
    and the polar ISSO radius.
    [See Stone, Loeb, Berger, PRD 87, 084053 (2013)]

    r: float
    chi: float
    incl: float
        inclination angle between the BH spin and the orbital angular
        momentum in radians

    Returns
    float
        r**8*Z+chi**2*(1-cos_incl**2)*(chi**2*(1-cos_incl**2)*Y-2*r**4*X)
        where
        X=chi**2*(chi**2*(3*chi**2+4*r*(2*r-3))+r**2*(15*r*(r-4)+28))-
          6*r**4*(r**2-4)
        Y=chi**4*(chi**4+r**2*(7*r*(3*r-4)+36))+6*r*(r-2)*(chi**6+2*r**3*
          (chi**2*(3*r+2)+3*r**2*(r-2)))
        Z=ISCO_eq(r,chi)
    """
    chi2 = chi*chi
    chi4 = chi2*chi2
    r2 = r*r
    r4 = r2*r2
    three_r = 3*r
    r_minus_2 = r - 2
    sin_incl2 = (math.sin(incl))**2

    X = chi2*(chi2*(3*chi2+4*r*(2*r-3))+r2*(15*r*(r-4)+28))-6*r4*(r2-4)
    Y = chi4 * (chi4+r2*(7 * r * (three_r-4)+36))+6 * r * r_minus_2 * \
        (chi4*chi2+2*r2*r*(chi2*(three_r+2)+3*r2*r_minus_2))
    Z = ISCO_eq(r, chi)

    return r4*r4*Z+chi2*sin_incl2*(chi2*sin_incl2*Y-2*r4*X)

# ISSO radius solver
def PG_ISSO_solver(chi,incl):
    """
    Function that determines the radius of the innermost stable
    spherical orbit (ISSO) for a Kerr BH and a generic inclination
    angle between the BH spin and the orbital angular momentum.
    This function finds the appropriat root of PG_ISSO_eq.

    chi: float
        the inclination angle between the BH spin and the orbital
        angular momentum in radians

    Returns
    solution: float
        the radius of the orbit in BH mass units
    """
    # Auxiliary variables
    cos_incl=math.cos(incl)
    sgnchi = np.sign(cos_incl)*chi

    # ISCO radius for the given spin magnitude
    if sgnchi > 0.99:
        initial_guess = 2 # Works better than 6 for chi=1
    elif sgnchi < 0:
        initial_guess = 9
    else:
        initial_guess = 5 # Works better than 6 for chi=0.5
    rISCO_limit = scipy.optimize.fsolve(ISCO_eq, initial_guess, args=sgnchi)

    # ISSO radius for an inclination of pi/2
    if chi < 0:
    else:
        initial_guess = 6
    rISSO_at_pole_limit = scipy.optimize.fsolve(ISSO_eq_at_pole, initial_guess,
                                                args=chi)

    # If the inclination is 0 or pi, just output the ISCO radius
    if incl in [0, math.pi]:
        solution = rISCO_limit
    # If the inclination is pi/2, just output the ISSO radius at the pole(s)
    elif incl == math.pi/2:
        solution = rISSO_at_pole_limit
    # Otherwise, find the ISSO radius for a generic inclination
    else:
        initial_guess = max(rISCO_limit,rISSO_at_pole_limit)
        solution = scipy.optimize.fsolve(PG_ISSO_eq, initial_guess,
                                         args=(chi, incl))
        if solution < 1 or solution > 9:
            initial_guess = min(rISCO_limit,rISSO_at_pole_limit)
            solution = scipy.optimize.fsolve(PG_ISSO_eq, initial_guess,
                                             args=(chi, incl))

    return solution

# 2H 2-piecewise polytropic EOS, NS non-rotating equilibrium sequence        #
# File format is: grav mass (Msun)   baryonic mass (Msun)    compactness     #
#                                                                            #
# Eventually, the function should call an NS sequence generator within LAL   #
# the EOS prescribed by the user and store it.                               #
def load_ns_sequence(eos_name):
    """
    Load the data of an NS non-rotating equilibrium sequence
    generated using the equation of state (EOS) chosen by the
    user.  [Only the 2H 2-piecewise polytropic EOS is currently
    supported.  This yields NSs with large radiss (15-16km).]

    eos_name: string
        NS equation of state label ('2H' is the only supported
        choice at the moment)

    Returns
    ns_sequence: 3D-array
        contains the sequence data in the form NS gravitational
         mass (in solar masses), NS baryonic mass (in solar
         masses), NS compactness (dimensionless)
    max_ns_g_mass: float
        the maximum NS gravitational mass (in solar masses) in
        the sequence (this is the mass of the most massive stable
        NS)
    """
    ns_sequence = []

    if eos_name == '2H':
        ns_sequence_path = os.path.join(pycbc.tmpltbank.NS_SEQUENCE_FILE_DIRECTORY, 'equil_2H.dat')
        ns_sequence = np.loadtxt(ns_sequence_path)
    else:
        err_msg = "Only the 2H EOS is currently supported."
        err_msg += "If you plan to use a different NS EOS,"
        err_msg += "be sure not to filter too many templates!\n"
        logging.error(err_msg)
        sys.exit(1)
        raise Exception('Unsupported EOS!')

    max_ns_g_mass = max(ns_sequence[:,0])

    return (ns_sequence, max_ns_g_mass)

# Given an NS equilibrium sequence and gravitational mass (in Msun), return  #
# the NS baryonic mass (in Msun).                                            #
def ns_g_mass_to_ns_b_mass(ns_g_mass, ns_sequence):
    """
    Determines the baryonic mass of an NS given its gravitational
    mass and an NS equilibrium sequence.

    ns_g_mass: float
        NS gravitational mass (in solar masses)

    Returns
    float
        The NS baryonic mass (in solar massesr**3*(r**2*(r-6)+chi**2*(3*r+4))+
        chi**4*(3*r*(r-2)+chi**2))
    """
    x = ns_sequence[:,0]
    y = ns_sequence[:,1]
    f = scipy.interpolate.interp1d(x, y)

    return f(ns_g_mass)

# the NS compactness.                                                        #
def ns_g_mass_to_ns_compactness(ns_g_mass, ns_sequence):
    """
    Determines the compactness of an NS given its
    gravitational mass and an NS equilibrium sequence.

    Returns
    float
        The NS compactness (dimensionless)
    """
    y = ns_sequence[:,2]

# NS-BH merger remnant mass [Foucart, Hinderer, Nissanke PRD 98, 081501(R)   #
# (2018)].                                                                   #
#                                                                            #
# * THIS ASSUMES THE NS SPIN IS 0 (the user is warned about this).           #
# * Physical parameters passed to remnant_mass (see sanity checks below)     #
#   must not be unphysical.                                                  #
# * Tilted BH spin cases are handled by using rISSO(chi,incl) where rISCO    #
#   appears in the formula. This approximation was suggested in Stone, Loeb, #
#   Berger, PRD 87, 084053 (2013) for the previous NS-BH remnant mass fit of #
#   Foucart, PRD 86, 124007 (2012).                                          #
def remnant_mass(eta, ns_g_mass, ns_sequence, chi, incl):
    """
    Function that determines the remnant disk mass of
    an NS-BH system using the fit to numerical-relativity
    results discussed in Foucart, Hinderer, Nissanke PRD 98, 081501(R) (2018).

    eta: float
        the symmetric mass ratio of the binary
    chi: float

    Returns
    remnant_mass: float
        The remnant mass in solar masses
    """

    # Sanity checks on eta and chi
    if not (eta>0. and eta<=0.25 and abs(chi)<=1 and ns_g_mass>0):
        err_msg = "The BH spin magnitude must be <= 1,"
        err_msg += "eta must be between 0 and 0.25,"
        err_msg += "and the NS mass must be positive."
        info_msg = "The function remnant_mass was launched with"
        info_msg += "ns_mass={0}, eta={1}, chi={2},"
        info_msg += "inclination={3}\n'.format(ns_g_mass, eta, chi, incl)"
        logging.info(info_msg)
        raise Exception('Unphysical parameters!')

    # NS compactness and rest mass
    ns_compactness = ns_g_mass_to_ns_compactness(ns_g_mass, ns_sequence)
    ns_b_mass = ns_g_mass_to_ns_b_mass(ns_g_mass, ns_sequence)

    # Fit parameters and tidal correction
    alpha = 0.406
    beta  = 0.139
    gamma = 0.255
    delta = 1.761
    # The remnant mass over the NS rest mass
    remnant_mass = (max(alpha/eta**(1./3.)*(1 - 2*ns_compactness)
                        - beta*ns_compactness / eta*PG_ISSO_solver(chi, incl)
                        + gamma, 0.)) ** delta
    # Convert to solar masses
    remnant_mass = remnant_mass*ns_b_mass

    return remnant_mass

#############################################################################
# Vectorized version of remnant_mass. The third argument (NS equilibrium    #
# sequence) is excluded from vectorisation.                                 #
remnant_masses = np.vectorize(remnant_mass)
remnant_masses.excluded.add(2)

#
#
#
import re
from lal import MTSUN_SI, PI, CreateREAL8Vector

# PLEASE ENSURE THESE ARE KEPT UP TO DATE WITH THE REST OF THIS FILE
pycbcValidTmpltbankOrders = ['zeroPN','onePN','onePointFivePN','twoPN',\
      'twoPointFivePN','threePN','threePointFivePN']

pycbcValidOrdersHelpDescriptions="""
     * zeroPN: Will only include the dominant term (proportional to chirp mass)
     * onePN: Will only the leading orbit term and first correction at 1PN
     * onePointFivePN: Will include orbit and spin terms to 1.5PN.
     * twoPN: Will include orbit and spin terms to 2PN.
     * twoPointFivePN: Will include orbit and spin terms to 2.5PN.
     * threePN: Will include orbit terms to 3PN and spin terms to 2.5PN.
     * threePointFivePN: Include orbit terms to 3.5PN and spin terms to 2.5PN
"""

def generate_mapping(order):
    """
    This function will take an order string and return a mapping between
    components in the metric and the various Lambda components. This must be
    used (and consistently used) when generating the metric *and* when
    transforming to/from the xi_i coordinates to the lambda_i coordinates.

    NOTE: This is not a great way of doing this. It would be nice to clean
    this up. Hence pulling this function out. The valid PN orders are
    {}

    order : string
        A string containing a PN order. Valid values are given above.

    Returns
    --------
        A mapping between the active Lambda terms and index in the metric
    """
    mapping = {}
    mapping['Lambda0'] = 0
    if order == 'zeroPN':
        return mapping
    mapping['Lambda2'] = 1
    if order == 'onePN':
    mapping['Lambda3'] = 2
    if order == 'onePointFivePN':
    mapping['Lambda4'] = 3
    if order == 'twoPN':
    mapping['LogLambda5'] = 4
    if order == 'twoPointFivePN':
    mapping['Lambda6'] = 5
    mapping['LogLambda6'] = 6
    if order == 'threePN':
    mapping['Lambda7'] = 7
    if order == 'threePointFivePN':
    # For some as-of-yet unknown reason, the tidal terms are not giving correct
    # match estimates when enabled. So, for now, this order is commented out.
    #if order == 'tidalTesting':
    #    mapping['Lambda10'] = 8
    #    mapping['Lambda12'] = 9
    #    return mapping
    raise ValueError("Order %s is not understood." %(order))

# Override doc so the PN orders are added automatically to online docs
generate_mapping.__doc__ = \
    generate_mapping.__doc__.format(pycbcValidOrdersHelpDescriptions)

def generate_inverse_mapping(order):
    """Genereate a lambda entry -> PN order map.

    This function will generate the opposite of generate mapping. So where
    generate_mapping gives dict[key] = item this will give
    dict[item] = key. Valid PN orders are:
    {}

    Returns
    --------
        An inverse mapping between the active Lambda terms and index in the
        metric
    """
    mapping = generate_mapping(order)
    inv_mapping = {}
    for key,value in mapping.items():
        inv_mapping[value] = key

    return inv_mapping

generate_inverse_mapping.__doc__ = \
    generate_inverse_mapping.__doc__.format(pycbcValidOrdersHelpDescriptions)

def get_ethinca_orders():
    """
    Returns the dictionary mapping TaylorF2 PN order names to twice-PN
    orders (powers of v/c)
    """
    ethinca_orders = {"zeroPN"           : 0,
                      "onePN"            : 2,
                      "onePointFivePN"   : 3,
                      "twoPN"            : 4,
                      "twoPointFivePN"   : 5,
                      "threePN"          : 6,
                      "threePointFivePN" : 7
                     }
    return ethinca_orders

def ethinca_order_from_string(order):
    """
    Returns the integer giving twice the post-Newtonian order
    used by the ethinca calculation. Currently valid only for TaylorF2 metric

    Returns
    -------
    int
    """
    if order in get_ethinca_orders().keys():
        return get_ethinca_orders()[order]
    else: raise ValueError("Order "+str(order)+" is not valid for ethinca"
                           "calculation! Valid orders: "+
                           str(get_ethinca_orders().keys()))

def get_chirp_params(mass1, mass2, spin1z, spin2z, f0, order,
                     quadparam1=None, quadparam2=None, lambda1=None,
                     lambda2=None):
    """
    Take a set of masses and spins and convert to the various lambda
    coordinates that describe the orbital phase. Accepted PN orders are:
    {}

    mass1 : float or array
        Mass1 of input(s).
    mass2 : float or array
        Mass2 of input(s).
    spin1z : float or array
        Parallel spin component(s) of body 1.
    spin2z : float or array
        Parallel spin component(s) of body 2.
    f0 : float
        This value must also be used consistently (ie. don't change its value
        when calling different functions!).
        The Post-Newtonian order that is used to translate from masses and
        spins to the lambda_i coordinate system. Valid orders given above.

    Returns
    --------
        The lambda coordinates for the input system(s)
    """

    # Determine whether array or single value input
    sngl_inp = False
    try:
        num_points = len(mass1)
        sngl_inp = True
        # If you care about speed, you aren't calling this function one entry
        # at a time.
        mass1 = numpy.array([mass1])
        mass2 = numpy.array([mass2])
        spin1z = numpy.array([spin1z])
        spin2z = numpy.array([spin2z])
        if quadparam1 is not None:
            quadparam1 = numpy.array([quadparam1])
        if quadparam2 is not None:
            quadparam2 = numpy.array([quadparam2])
        if lambda1 is not None:
            lambda1 = numpy.array([lambda1])
        if lambda2 is not None:
            lambda2 = numpy.array([lambda2])
        num_points = 1

    if quadparam1 is None:
        quadparam1 = numpy.ones(len(mass1), dtype=float)
    if quadparam2 is None:
        quadparam2 = numpy.ones(len(mass1), dtype=float)
    if lambda1 is None:
        lambda1 = numpy.zeros(len(mass1), dtype=float)
    if lambda2 is None:
        lambda2 = numpy.zeros(len(mass1), dtype=float)

    mass1_v = CreateREAL8Vector(len(mass1))
    mass1_v.data[:] = mass1[:]
    mass2_v = CreateREAL8Vector(len(mass1))
    mass2_v.data[:] = mass2[:]
    spin1z_v = CreateREAL8Vector(len(mass1))
    spin1z_v.data[:] = spin1z[:]
    spin2z_v = CreateREAL8Vector(len(mass1))
    spin2z_v.data[:] = spin2z[:]
    lambda1_v = CreateREAL8Vector(len(mass1))
    lambda1_v.data[:] = lambda1[:]
    lambda2_v = CreateREAL8Vector(len(mass1))
    lambda2_v.data[:] = lambda2[:]
    dquadparam1_v = CreateREAL8Vector(len(mass1))
    dquadparam1_v.data[:] = quadparam1[:] - 1.
    dquadparam2_v = CreateREAL8Vector(len(mass1))
    dquadparam2_v.data[:] = quadparam2[:] - 1.

    phasing_arr = lalsimulation.SimInspiralTaylorF2AlignedPhasingArray\
        (mass1_v, mass2_v, spin1z_v, spin2z_v, lambda1_v, lambda2_v,
         dquadparam1_v, dquadparam2_v)

    vec_len = lalsimulation.PN_PHASING_SERIES_MAX_ORDER + 1;
    phasing_vs = numpy.zeros([num_points, vec_len])
    phasing_vlogvs = numpy.zeros([num_points, vec_len])
    phasing_vlogvsqs = numpy.zeros([num_points, vec_len])

    lng = len(mass1)
    jmp = lng * vec_len
    for idx in range(vec_len):
        phasing_vs[:,idx] = phasing_arr.data[lng*idx : lng*(idx+1)]
        phasing_vlogvs[:,idx] = \
            phasing_arr.data[jmp + lng*idx : jmp + lng*(idx+1)]
        phasing_vlogvsqs[:,idx] = \
            phasing_arr.data[2*jmp + lng*idx : 2*jmp + lng*(idx+1)]

    pim = PI * (mass1 + mass2)*MTSUN_SI
    pmf = pim * f0
    pmf13 = pmf**(1./3.)
    logpim13 = numpy.log((pim)**(1./3.))

    mapping = generate_inverse_mapping(order)
    lambdas = []
    lambda_str = '^Lambda([0-9]+)'
    loglambda_str = '^LogLambda([0-9]+)'
    logloglambda_str = '^LogLogLambda([0-9]+)'
    for idx in range(len(mapping.keys())):
        # RE magic engage!
        rematch = re.match(lambda_str, mapping[idx])
        if rematch:
            pn_order = int(rematch.groups()[0])
            term = phasing_vs[:,pn_order]
            term = term + logpim13 * phasing_vlogvs[:,pn_order]
            lambdas.append(term * pmf13**(-5+pn_order))
            continue
        rematch = re.match(loglambda_str, mapping[idx])
            lambdas.append((phasing_vlogvs[:,pn_order]) * pmf13**(-5+pn_order))
            continue
        rematch = re.match(logloglambda_str, mapping[idx])
            raise ValueError("LOGLOG terms are not implemented")
            #pn_order = int(rematch.groups()[0])
            #lambdas.append(phasing_vlogvsqs[:,pn_order] * pmf13**(-5+pn_order))
            #continue
        err_msg = "Failed to parse " +  mapping[idx]

    if sngl_inp:
        return [l[0] for l in lambdas]
    else:
        return lambdas

get_chirp_params.__doc__ = \
    get_chirp_params.__doc__.format(pycbcValidOrdersHelpDescriptions)

#
#
#

import lal

def generate_hexagonal_lattice(maxv1, minv1, maxv2, minv2, mindist):
    """
    This function generates a 2-dimensional lattice of points using a hexagonal
    lattice.

    maxv1 : float
        Largest value in the 1st dimension to cover
    minv1 : float
        Smallest value in the 1st dimension to cover
    maxv2 : float
        Largest value in the 2nd dimension to cover
    minv2 : float
        Smallest value in the 2nd dimension to cover
    mindist : float
        Maximum allowed mismatch between a point in the parameter space and the
        generated bank of points.

    Returns
    --------
    v1s : numpy.array
        Array of positions in the first dimension
    v2s : numpy.array
        Array of positions in the second dimension
    """
    if minv1 > maxv1:
        raise ValueError("Invalid input to function.")
    if minv2 > maxv2:
    # Place first point
    v1s = [minv1]
    v2s = [minv2]
    initPoint = [minv1,minv2]
    # Place first line
    initLine = [initPoint]
    tmpv1 = minv1
    while (tmpv1 < maxv1):
        tmpv1 = tmpv1 + (3 * mindist)**(0.5)
        initLine.append([tmpv1,minv2])
        v1s.append(tmpv1)
        v2s.append(minv2)
    initLine = numpy.array(initLine)
    initLine2 = copy.deepcopy(initLine)
    initLine2[:,0] += 0.5 * (3*mindist)**0.5
    initLine2[:,1] += 1.5 * (mindist)**0.5
    for i in range(len(initLine2)):
        v1s.append(initLine2[i,0])
        v2s.append(initLine2[i,1])
    tmpv2_1 = initLine[0,1]
    tmpv2_2 = initLine2[0,1]
    while tmpv2_1 < maxv2 and tmpv2_2 < maxv2:
        tmpv2_1 = tmpv2_1 + 3.0 * (mindist)**0.5
        tmpv2_2 = tmpv2_2 + 3.0 * (mindist)**0.5
        initLine[:,1] = tmpv2_1
        initLine2[:,1] = tmpv2_2
        for i in range(len(initLine)):
            v1s.append(initLine[i,0])
            v2s.append(initLine[i,1])
        for i in range(len(initLine2)):
            v1s.append(initLine2[i,0])
            v2s.append(initLine2[i,1])
    v1s = numpy.array(v1s)
    v2s = numpy.array(v2s)
    return v1s, v2s

def generate_anstar_3d_lattice(maxv1, minv1, maxv2, minv2, maxv3, minv3, \
                               mindist):
    """
    This function calls into LAL routines to generate a 3-dimensional array
    of points using the An^* lattice.

    maxv3 : float
        Largest value in the 3rd dimension to cover
    minv3 : float
        Smallest value in the 3rd dimension to cover

    Returns
    --------
    v3s : numpy.array
    """
    # Lalpulsar not a requirement for the rest of pycbc, so check if we have it
    # here in this function.
    try:
        import lalpulsar
    except:
        raise ImportError("A SWIG-wrapped install of lalpulsar is needed to use the anstar tiling functionality.")

    tiling = lalpulsar.CreateLatticeTiling(3)
    lalpulsar.SetLatticeTilingConstantBound(tiling, 0, minv1, maxv1)
    lalpulsar.SetLatticeTilingConstantBound(tiling, 1, minv2, maxv2)
    lalpulsar.SetLatticeTilingConstantBound(tiling, 2, minv3, maxv3)
    # Make a 3x3 Euclidean lattice
    a = lal.gsl_matrix(3,3)
    a.data[0,0] = 1
    a.data[1,1] = 1
    a.data[2,2] = 1
    try:
        # old versions of lalpulsar used an enumeration
        lattice = lalpulsar.TILING_LATTICE_ANSTAR
        # newer versions of lalpulsar use a string
        lattice = 'An-star'
    lalpulsar.SetTilingLatticeAndMetric(tiling, lattice, a, mindist)
    try:
        iterator = lalpulsar.CreateLatticeTilingIterator(tiling, 3)
        # old versions of lalpulsar required the flags argument
        # (set to 0 for defaults)
        iterator = lalpulsar.CreateLatticeTilingIterator(tiling, 3, 0)

    vs1 = []
    vs2 = []
    vs3 = []
    curr_point = lal.gsl_vector(3)
    while (lalpulsar.NextLatticeTilingPoint(iterator, curr_point) > 0):
        vs1.append(curr_point.data[0])
        vs2.append(curr_point.data[1])
        vs3.append(curr_point.data[2])
    return vs1, vs2, vs3

#
#
#

import os
from pycbc.tmpltbank.lambda_mapping import get_ethinca_orders, pycbcValidOrdersHelpDescriptions
from pycbc.tmpltbank.em_progenitors import load_ns_sequence
from pycbc.types import positive_float, nonnegative_float

class IndentedHelpFormatterWithNL(argparse.ArgumentDefaultsHelpFormatter):
    """
    This class taken from 
    https://groups.google.com/forum/#!topic/comp.lang.python/bfbmtUGhW8I
    and is used to format the argparse help messages to deal with line breaking
    nicer. Specfically the pn-order help is large and looks crappy without this.
    This function is (C) Tim Chase
    """
    def format_description(self, description):
        """
        No documentation
        """
        if not description: return ""
        desc_width = self.width - self.current_indent
        indent = " "*self.current_indent
        # the above is still the same
        bits = description.split('\n')
        formatted_bits = [
            textwrap.fill(bit,
                desc_width,
                initial_indent=indent,
                subsequent_indent=indent)
            for bit in bits]
        result = "\n".join(formatted_bits) + "\n"

    def format_option(self, option):
        """
        """
        # The help for each option consists of two parts:
        #   * the opt strings and metavars
        #   eg. ("-x", or "-fFILENAME, --file=FILENAME")
        #   * the user-supplied help string
        #   eg. ("turn on expert mode", "read data from FILENAME")
        #
        # If possible, we write both of these on the same line:
        #   -x    turn on expert mode
        #
        # But if the opt string list is too long, we put the help
        # string on a second line, indented to the same column it would
        # start in if it fit on the first line.
        #   -fFILENAME, --file=FILENAME
        #       read data from FILENAME
        result = []
        opts = self.option_strings[option]
        opt_width = self.help_position - self.current_indent - 2
        if len(opts) > opt_width:
            opts = "%*s%s\n" % (self.current_indent, "", opts)
            indent_first = self.help_position
        else: # start help on same line as opts
            opts = "%*s%-*s  " % (self.current_indent, "", opt_width, opts)
            indent_first = 0
        result.append(opts)
        if option.help:
            help_text = self.expand_default(option)
            # Everything is the same up through here
            help_lines = []
            for para in help_text.split("\n"):
                help_lines.extend(textwrap.wrap(para, self.help_width))
            # Everything is the same after here
            result.append("%*s%s\n" % (
                indent_first, "", help_lines[0]))
            result.extend(["%*s%s\n" % (self.help_position, "", line)
                for line in help_lines[1:]])
        elif opts[-1] != "\n":
            result.append("\n")
        return "".join(result)

def get_options_from_group(option_group):
    """
    Take an option group and return all the options that are defined in that
    group.
    """
    option_list = option_group._group_actions
    command_lines = []
    for option in option_list:
        option_strings = option.option_strings
        for string in option_strings:
            if string.startswith('--'):
                command_lines.append(string)
    return command_lines

def insert_base_bank_options(parser):
    """
    Adds essential common options for template bank generation to an
    ArgumentParser instance.
    """

    def match_type(s):
        err_msg = "must be a number between 0 and 1 excluded, not %r" % s
        try:
            value = float(s)
            raise argparse.ArgumentTypeError(err_msg)
        if value <= 0 or value >= 1:

    parser.add_argument(
            '-m', '--min-match', type=match_type, required=True,
            help="Generate bank with specified minimum match. Required.")
            '-O', '--output-file', required=True,
            help="Output file name. Required.")
    parser.add_argument('--f-low-column', type=str, metavar='NAME',
                        help='If given, store the lower frequency cutoff into '
                             'column NAME of the single-inspiral table.')

def insert_metric_calculation_options(parser):
    """
    Adds the options used to obtain a metric in the bank generation codes to an
    argparser as an OptionGroup. This should be used if you want to use these
    options in your code.
    """
    metricOpts = parser.add_argument_group(
                "Options related to calculating the parameter space metric")
    metricOpts.add_argument("--pn-order", action="store", type=str,
                required=True,
                help="Determines the PN order to use.  For a bank of "
                     "non-spinning templates, spin-related terms in the "
                     "metric will be zero.  REQUIRED. "
                     "Choices: %s" %(pycbcValidOrdersHelpDescriptions))
    metricOpts.add_argument("--f0", action="store", type=positive_float,
                default=70.,\
                help="f0 is used as a dynamic scaling factor when "
                     "calculating integrals used in metric construction.  "
                     "I.e. instead of integrating F(f) we integrate F(f/f0) "
                     "then rescale by powers of f0.  The default value 70Hz "
                     "should be fine for most applications.  OPTIONAL. "
                     "UNITS=Hz. **WARNING: If the ethinca metric is to be "
                     "calculated, f0 must be set equal to f-low**")
    metricOpts.add_argument("--f-low", action="store", type=positive_float,
                help="Lower frequency cutoff used in computing the "
                     "parameter space metric.  REQUIRED. UNITS=Hz")
    metricOpts.add_argument("--f-upper", action="store", type=positive_float,
                help="Upper frequency cutoff used in computing the "
    metricOpts.add_argument("--delta-f", action="store", type=positive_float,
                help="Frequency spacing used in computing the parameter "
                     "space metric:  integrals of the form \int F(f) df "
                     "are approximated as \sum F(f) delta_f.  REQUIRED. "
                     "UNITS=Hz")
    metricOpts.add_argument("--write-metric", action="store_true",
                default=False, help="If given write the metric components "
                     "to disk as they are calculated.")
    return metricOpts

def verify_metric_calculation_options(opts, parser):
    """
    Parses the metric calculation options given and verifies that they are
    correct.

    opts : argparse.Values instance
        Result of parsing the input options with OptionParser
        The OptionParser instance.
    """
    if not opts.pn_order:
        parser.error("Must supply --pn-order")

class metricParameters(object):
    """
    This class holds all of the options that are parsed in the function
    insert_metric_calculation_options
    and all products produced using these options. It can also be initialized
    from the __init__ function, providing directly the options normally
    provided on the command line.
    """
    _psd = None
    _metric = None
    _evals = None
    _evecs = None
    _evecsCV = None
    def __init__(self, pnOrder, fLow, fUpper, deltaF, f0=70,
                 write_metric=False):
        """
        Initialize an instance of the metricParameters by providing all
        options directly. See the help message associated with any code
        that uses the metric options for more details of how to set each of
        these, e.g. pycbc_aligned_stoch_bank --help
        """
        self.pnOrder=pnOrder
        self.fLow=fLow
        self.fUpper=fUpper
        self.deltaF=deltaF
        self.f0=f0
        self._moments=None
        self.write_metric=write_metric

    def from_argparse(cls, opts):
        """
        Initialize an instance of the metricParameters class from an
        argparse.OptionParser instance. This assumes that
        insert_metric_calculation_options
        and
        verify_metric_calculation_options
        have already been called before initializing the class.
        """
        return cls(opts.pn_order, opts.f_low, opts.f_upper, opts.delta_f,\
                   f0=opts.f0, write_metric=opts.write_metric)

    @property
    def psd(self):
        """
        A pyCBC FrequencySeries holding the appropriate PSD.
        Return the PSD used in the metric calculation.
        """
        if not self._psd:
            errMsg = "The PSD has not been set in the metricParameters "
            errMsg += "instance."
            raise ValueError(errMsg)
        return self._psd

    @psd.setter
    def psd(self, inPsd):
        self._psd = inPsd

    @property
    def moments(self):
        """
        Moments structure

        This stores the integral of 

        The second entry stores the frequency cutoff that was used when
        computing the integral.
        """
        return self._moments

    @moments.setter
    def moments(self, inMoments):
        self._moments=inMoments

    @property
    def evals(self):
        """
        The eigenvalues of the parameter space.
        This is a Dictionary of numpy.array
        varying.
        """
        if self._evals is None:
            errMsg = "The metric eigenvalues have not been set in the "
            errMsg += "metricParameters instance."
        return self._evals

    @evals.setter
    def evals(self, inEvals):
        if self.write_metric:
            for frequency in inEvals.keys():
                numpy.savetxt("metric_evals_%d.dat" %(frequency),
                              inEvals[frequency])
        self._evals = inEvals

    @property
    def evecs(self):
        """
        The eigenvectors of the parameter space.
        This is a Dictionary of numpy.matrix
        """
        if self._evecs is None:
            errMsg = "The metric eigenvectors have not been set in the "
        return self._evecs

    @evecs.setter
    def evecs(self, inEvecs):
            for frequency in inEvecs.keys():
                numpy.savetxt("metric_evecs_%d.dat" %(frequency),
                              inEvecs[frequency])
        self._evecs = inEvecs

    @property
    def metric(self):
        """
        The metric of the parameter space.
        """
        if self._metric is None:
        return self._metric

    @metric.setter
    def metric(self, inMetric):
            for frequency in inMetric.keys():
                numpy.savetxt("metric_components_%d.dat" %(frequency),
                              inMetric[frequency])
        self._metric = inMetric

    @property
    def time_unprojected_metric(self):
        """
        The metric of the parameter space with the time dimension unprojected.
        Lambda_i, t coordinate system. The time components are always in the
        last [-1] position in the matrix.
        """
        if self._time_unprojected_metric is None:
            err_msg = "The time unprojected metric has not been set in the "
            err_msg += "metricParameters instance."
        return self._time_unprojected_metric

    @time_unprojected_metric.setter
    def time_unprojected_metric(self, inMetric):
                numpy.savetxt("metric_timeunprojected_%d.dat" %(frequency),
        self._time_unprojected_metric = inMetric

    @property
    def evecsCV(self):
        """
        The eigenvectors of the principal directions of the mu space.
        """
        if self._evecsCV is None:
            errMsg = "The covariance eigenvectors have not been set in the "
        return self._evecsCV

    @evecsCV.setter
    def evecsCV(self, inEvecs):
                numpy.savetxt("covariance_evecs_%d.dat" %(frequency),
        self._evecsCV = inEvecs

def insert_mass_range_option_group(parser,nonSpin=False):
    """
    Adds the options used to specify mass ranges in the bank generation codes
    to an argparser as an OptionGroup. This should be used if you
 
    nonSpin : boolean, optional (default=False)
        If this is provided the spin-related options will not be added.
    """
    massOpts = parser.add_argument_group("Options related to mass and spin "
                  "limits for bank generation")
    massOpts.add_argument("--min-mass1", action="store", type=positive_float,
                  required=True, 
                  help="Minimum mass1: must be >= min-mass2. "
                       "REQUIRED. UNITS=Solar mass")
    massOpts.add_argument("--max-mass1", action="store", type=positive_float,
                  required=True,
                  help="Maximum mass1: must be >= max-mass2. "
    massOpts.add_argument("--min-mass2", action="store", type=positive_float,
                  help="Minimum mass2. REQUIRED. UNITS=Solar mass")
    massOpts.add_argument("--max-mass2", action="store", type=positive_float,
                  help="Maximum mass2. REQUIRED. UNITS=Solar mass")
    massOpts.add_argument("--max-total-mass", action="store",
                          type=positive_float, default=None, 
                          help="Maximum total mass. OPTIONAL, if not provided "
                          "the max total mass is determined by the component "
                          "masses. UNITS=Solar mass")
    massOpts.add_argument("--min-total-mass", action="store",
                  help="Minimum total mass. OPTIONAL, if not provided the "
                       "min total mass is determined by the component masses."
                       " UNITS=Solar mass")
    massOpts.add_argument("--max-chirp-mass", action="store",
                          type=positive_float, default=None,
                  help="Maximum chirp mass. OPTIONAL, if not provided the "
                       "max chirp mass is determined by the component masses."
    massOpts.add_argument("--min-chirp-mass", action="store",
                       "min chirp mass is determined by the component masses."
    massOpts.add_argument("--max-eta", action="store", type=positive_float,
                  default=0.25,
                  help="Maximum symmetric mass ratio. OPTIONAL, no upper bound"
                       " on eta will be imposed if not provided. "
                       "UNITS=Solar mass.")
    massOpts.add_argument("--min-eta", action="store", type=nonnegative_float,
                  default=0.,
                  help="Minimum symmetric mass ratio. OPTIONAL, no lower bound"
    massOpts.add_argument("--ns-eos", action="store",
                  default=None,
                  help="Select the EOS to be used for the NS when calculating "
                       "the remnant disk mass. Only 2H is currently supported. "
                       "OPTIONAL")
    massOpts.add_argument("--remnant-mass-threshold", action="store",
                          type=nonnegative_float, default=None,
                  help="Setting this filters EM dim NS-BH binaries: if the "
                       "remnant disk mass does not exceed this value, the NS-BH "
                       "binary is dropped from the target parameter space. "
                       "When it is set to None (default value) the EM dim "
                       "filter is not activated. OPTIONAL")
    massOpts.add_argument("--use-eos-max-ns-mass", action="store_true", default=False,
                  help="Cut the mass range of the smaller object to the maximum "
                       "mass allowed by EOS. "
    massOpts.add_argument("--delta-bh-spin", action="store",
                  help="Grid spacing used for the BH spin z component when "
                       "generating the surface of the minumum minimum symmetric "
                       "mass ratio as a function of BH spin and NS mass required "
                       "to produce a remnant disk mass that exceeds the threshold "
                       "specificed in --remnant-mass-threshold. "
                       "OPTIONAL (0.1 by default) ")
    massOpts.add_argument("--delta-ns-mass", action="store",
                  help="Grid spacing used for the NS mass when generating the "
                       "surface of the minumum minimum symmetric mass ratio as "
                       "a function of BH spin and NS mass required to produce "
                       "a remnant disk mass that exceeds the thrsehold specified "
                       "in --remnant-mass-threshold. "
    if nonSpin:
        parser.add_argument_group(massOpts)
        return massOpts

    massOpts.add_argument("--max-ns-spin-mag", action="store",
                  help="Maximum neutron star spin magnitude.  Neutron stars "
                       "are defined as components lighter than the NS-BH "
                       "boundary (3 Msun by default). REQUIRED if min-mass2 "
                       "< ns-bh-boundary-mass")
    massOpts.add_argument("--max-bh-spin-mag", action="store",
                  help="Maximum black hole spin magnitude.  Black holes are "
                       "defined as components at or above the NS-BH boundary "
                       "(3 Msun by default). REQUIRED if max-mass1 >= "
                       "ns-bh-boundary-mass")
    # Mutually exclusive group prevents both options being set on command line
    # If --nsbh-flag is True then spinning bank generation must ignore the
    # default value of ns-bh-boundary-mass.
    action = massOpts.add_mutually_exclusive_group(required=False)
    action.add_argument("--ns-bh-boundary-mass", action='store',
                        type=positive_float,
                  help="Mass boundary between neutron stars and black holes. "
                       "Components below this mass are considered neutron "
                       "stars and are subject to the neutron star spin limits. "
                       "Components at/above are subject to the black hole spin "
                       "limits.  OPTIONAL, default=%f.  UNITS=Solar mass" \
                       % massRangeParameters.default_nsbh_boundary_mass)
    action.add_argument("--nsbh-flag", action="store_true", default=False,
                  help="Set this flag if generating a bank that contains only "
                       "systems with 1 black hole and 1 neutron star. With "
                       "this flag set the heavier body will always be subject "
                       "to the black hole spin restriction and the lighter "
                       "to the neutron star spin restriction, regardless of "
                       "mass.  OPTIONAL.  If set, the value of "
                       "--ns-bh-boundary-mass will be ignored.")
    return massOpts

def verify_mass_range_options(opts, parser, nonSpin=False):
    """
    correct.

        If this is provided the spin-related options will not be checked.
    """
    # Mass1 must be the heavier!
    if opts.min_mass1 < opts.min_mass2:
        parser.error("min-mass1 cannot be less than min-mass2!")
    if opts.max_mass1 < opts.max_mass2:
        parser.error("max-mass1 cannot be less than max-mass2!")
    # If given are min/max total mass/chirp mass possible?
    if opts.min_total_mass \
            and (opts.min_total_mass > opts.max_mass1 + opts.max_mass2):
        err_msg = "Supplied minimum total mass %f " %(opts.min_total_mass,)
        err_msg += "greater than the sum of the two max component masses "
        err_msg += " %f and %f." %(opts.max_mass1,opts.max_mass2)
        parser.error(err_msg)
    if opts.max_total_mass \
            and (opts.max_total_mass < opts.min_mass1 + opts.min_mass2):
        err_msg = "Supplied maximum total mass %f " %(opts.max_total_mass,)
        err_msg += "smaller than the sum of the two min component masses "
        err_msg += " %f and %f." %(opts.min_mass1,opts.min_mass2)
    if opts.max_total_mass and opts.min_total_mass \
            and (opts.max_total_mass < opts.min_total_mass):
        parser.error("Min total mass must be larger than max total mass.")
    # Warn the user that his/her setup is such that EM dim NS-BH binaries
    # will not be targeted by the template bank that is being built.  Also
    # inform him/her about the caveats involved in this.
    if hasattr(opts, 'remnant_mass_threshold') \
            and opts.remnant_mass_threshold is not None:
        logging.info("""You have asked to exclude EM dim NS-BH systems from the
                        target parameter space. The script will assume that m1 is
                        the BH and m2 is the NS: make sure that your settings
                        respect this convention. The script will also treat the
                        NS as non-spinning: use NS spins in the template bank
                        at your own risk!""")
        if opts.use_eos_max_ns_mass:
            logging.info("""You have asked to take into account the maximum NS
                        mass value for the EOS in use.""")
        # Find out if the EM constraint surface data already exists or not
        # and inform user whether this will be read from file or generated.
        # This is the minumum eta as a function of BH spin and NS mass
        # required to produce an EM counterpart
        if os.path.isfile('constraint_em_bright.npz'):
            logging.info("""The constraint surface for EM bright binaries
                        will be read in from constraint_em_bright.npz.""")

    # Assign min/max total mass from mass1, mass2 if not specified
    if (not opts.min_total_mass) or \
            ((opts.min_mass1 + opts.min_mass2) > opts.min_total_mass):
        opts.min_total_mass = opts.min_mass1 + opts.min_mass2
    if (not opts.max_total_mass) or \
            ((opts.max_mass1 + opts.max_mass2) < opts.max_total_mass):
        opts.max_total_mass = opts.max_mass1 + opts.max_mass2

    # It is vital that min and max total mass be set correctly.
    # This is becasue the heavily-used function get_random_mass will place
    # points first in total mass (to some power), and then in eta. If the total
    # mass limits are not well known ahead of time it will place unphysical
    # points and fail.
    # This test is a bit convoluted as we identify the maximum and minimum
    # possible total mass from chirp mass and/or eta restrictions.
    if opts.min_chirp_mass is not None:
        # Need to get the smallest possible min_tot_mass from this chirp mass
        # There are 4 possibilities for where the min_tot_mass is found on the
        # line of min_chirp_mass that interacts with the component mass limits.
        # Either it is found at max_m2, or at min_m1, or it starts on the equal
        # mass line within the parameter space, or it doesn't intersect
        # at all.
        # First let's get the masses at both of these possible points
        m1_at_max_m2 = pnutils.mchirp_mass1_to_mass2(opts.min_chirp_mass,
                                                     opts.max_mass2)
        if m1_at_max_m2 < opts.max_mass2:
            # Unphysical, remove
            m1_at_max_m2 = -1
        m2_at_min_m1 = pnutils.mchirp_mass1_to_mass2(opts.min_chirp_mass,
                                                      opts.min_mass1)
        if m2_at_min_m1 > opts.min_mass1:
            m2_at_min_m1 = -1
        # Get the values on the equal mass line
        m1_at_equal_mass, m2_at_equal_mass = pnutils.mchirp_eta_to_mass1_mass2(
                                                     opts.min_chirp_mass, 0.25)

        # Are any of these possible?
        if m1_at_max_m2 <= opts.max_mass1 and m1_at_max_m2 >= opts.min_mass1:
            min_tot_mass = opts.max_mass2 + m1_at_max_m2
        elif m2_at_min_m1 <= opts.max_mass2 and m2_at_min_m1 >= opts.min_mass2:
            min_tot_mass = opts.min_mass1 + m2_at_min_m1
        elif m1_at_equal_mass <= opts.max_mass1 and \
                 m1_at_equal_mass >= opts.min_mass1 and \
                 m2_at_equal_mass <= opts.max_mass2 and \
                 m2_at_equal_mass >= opts.min_mass2:
            min_tot_mass = m1_at_equal_mass + m2_at_equal_mass
        # So either the restriction is low enough to be redundant, or is
        # removing all the parameter space
        elif m2_at_min_m1 < opts.min_mass2:
            # This is the redundant case, ignore
            min_tot_mass = opts.min_total_mass
        else:
            # And this is the bad case
            err_msg = "The minimum chirp mass provided is not possible given "
            err_msg += "restrictions on component masses."
        # Is there also an eta restriction?
        if opts.max_eta:
            # Get the value of m1,m2 at max_eta, min_chirp_mass
            max_eta_m1, max_eta_m2 = pnutils.mchirp_eta_to_mass1_mass2(
                                         opts.min_chirp_mass, opts.max_eta)
            max_eta_min_tot_mass = max_eta_m1 + max_eta_m2
            if max_eta_min_tot_mass > min_tot_mass:
                # Okay, eta does restrict this further. Still physical?
                min_tot_mass = max_eta_min_tot_mass
                if max_eta_m1 > opts.max_mass1:
                    err_msg = "The combination of component mass, chirp "
                    err_msg += "mass, eta and (possibly) total mass limits "
                    err_msg += "have precluded all systems."
                    raise ValueError(err_msg)
        # Update min_tot_mass if needed
        if min_tot_mass > opts.min_total_mass:
            opts.min_total_mass = float(min_tot_mass)

    # Then need to do max_chirp_mass and min_eta
    if opts.max_chirp_mass is not None:
        # Need to get the largest possible maxn_tot_mass from this chirp mass
        # There are 3 possibilities for where the max_tot_mass is found on the
        # line of max_chirp_mass that interacts with the component mass limits.
        # Either it is found at min_m2, or at max_m1, or it doesn't intersect
        # at all.
        m1_at_min_m2 = pnutils.mchirp_mass1_to_mass2(opts.max_chirp_mass,
                                                     opts.min_mass2)
        m2_at_max_m1 = pnutils.mchirp_mass1_to_mass2(opts.max_chirp_mass,
                                                      opts.max_mass1)
        # Are either of these possible?
        if m1_at_min_m2 <= opts.max_mass1 and m1_at_min_m2 >= opts.min_mass1:
            max_tot_mass = opts.min_mass2 + m1_at_min_m2
        elif m2_at_max_m1 <= opts.max_mass2 and m2_at_max_m1 >= opts.min_mass2:
            max_tot_mass = opts.max_mass1 + m2_at_max_m1
        # removing all the paramter space
        elif m2_at_max_m1 > opts.max_mass2:
            max_tot_mass = opts.max_total_mass
        else:
            err_msg = "The maximum chirp mass provided is not possible given "
        if opts.min_eta:
            min_eta_m1, min_eta_m2 = pnutils.mchirp_eta_to_mass1_mass2(
                                         opts.max_chirp_mass, opts.min_eta)
            min_eta_max_tot_mass = min_eta_m1 + min_eta_m2
            if min_eta_max_tot_mass < max_tot_mass:
                max_tot_mass = min_eta_max_tot_mass
                if min_eta_m1 < opts.min_mass1:
        if max_tot_mass < opts.max_total_mass:
            opts.max_total_mass = float(max_tot_mass)

    # Need to check max_eta alone for minimum and maximum mass
    if opts.max_eta:
        # Similar to above except this can affect both the minimum and maximum
        # total mass. Need to identify where the line of max_eta intersects
        # the parameter space, and if it affects mass restrictions.
        m1_at_min_m2 = pnutils.eta_mass1_to_mass2(opts.max_eta, opts.min_mass2,
                                                      return_mass_heavier=True)
        m2_at_min_m1 = pnutils.eta_mass1_to_mass2(opts.max_eta, opts.min_mass1,
                                                     return_mass_heavier=False)
        m1_at_max_m2 = pnutils.eta_mass1_to_mass2(opts.max_eta, opts.max_mass2,
        m2_at_max_m1 = pnutils.eta_mass1_to_mass2(opts.max_eta, opts.max_mass1,
                                                      return_mass_heavier=False)
        # Check for restrictions on the minimum total mass
            min_tot_mass = opts.min_mass2 + m1_at_min_m2
            # This case doesn't change the minimal total mass
        elif m2_at_min_m1 > opts.max_mass2:
        elif opts.max_eta == 0.25 and (m1_at_min_m2 < opts.min_mass2 or \
                                                m2_at_min_m1 > opts.min_mass1): 
            # This just catches potential roundoff issues in the case that
            # max-eta is not used
        else:
            err_msg = "The maximum eta provided is not possible given "
            print(m1_at_min_m2, m2_at_min_m1, m1_at_max_m2, m2_at_max_m1)
            print(opts.min_mass1, opts.max_mass1, opts.min_mass2, opts.max_mass2)

        # Check for restrictions on the maximum total mass
        if m2_at_max_m1 <= opts.max_mass2 and m2_at_max_m1 >= opts.min_mass2:
        elif m1_at_max_m2 <= opts.max_mass1 and m1_at_max_m2 >= opts.min_mass1:
            # This case doesn't change the maximal total mass
        # removing all the paramter space, the latter case is already tested
        else:

    # Need to check min_eta alone for maximum and minimum total mass
    if opts.min_eta:
        # Same as max_eta.
        # Need to identify where the line of max_eta intersects
        m1_at_min_m2 = pnutils.eta_mass1_to_mass2(opts.min_eta, opts.min_mass2,
        m2_at_min_m1 = pnutils.eta_mass1_to_mass2(opts.min_eta, opts.min_mass1,
        m1_at_max_m2 = pnutils.eta_mass1_to_mass2(opts.min_eta, opts.max_mass2,
        m2_at_max_m1 = pnutils.eta_mass1_to_mass2(opts.min_eta, opts.max_mass1,

            max_tot_mass = opts.max_mass2 + m1_at_max_m2

            # This case doesn't affect the maximum total mass
        elif m2_at_max_m1 < opts.min_mass2:
        else:
            err_msg = "The minimum eta provided is not possible given "

        if m2_at_min_m1 <= opts.max_mass2 and m2_at_min_m1 >= opts.min_mass2:
        elif m1_at_min_m2 <= opts.max_mass1 and m1_at_min_m2 >= opts.min_mass1:
        # removing all the paramter space, which is tested above
        else:

    if opts.max_total_mass < opts.min_total_mass:
        err_msg = "After including restrictions on chirp mass, component mass, "
        err_msg += "eta and total mass, no physical systems are possible."

    if opts.max_eta and opts.min_eta and (opts.max_eta < opts.min_eta):
        parser.error("--max-eta must be larger than --min-eta.")
        return

    if opts.max_ns_spin_mag is None:
        if opts.nsbh_flag:
            parser.error("Must supply --max_ns_spin_mag with --nsbh-flag")
        # Can ignore this if no NSs will be generated
        elif opts.min_mass2 < (opts.ns_bh_boundary_mass or
                massRangeParameters.default_nsbh_boundary_mass):
            parser.error("Must supply --max-ns-spin-mag for the chosen"
                         " value of --min_mass2")
        else:
            opts.max_ns_spin_mag = opts.max_bh_spin_mag
    if opts.max_bh_spin_mag is None:
            parser.error("Must supply --max_bh_spin_mag with --nsbh-flag")
        # Can ignore this if no BHs will be generated
        if opts.max_mass1 >= (opts.ns_bh_boundary_mass or
            parser.error("Must supply --max-bh-spin-mag for the chosen"
                         " value of --max_mass1")
        else:
            opts.max_bh_spin_mag = opts.max_ns_spin_mag

class massRangeParameters(object):
    """
    insert_mass_range_option_group
    from the __init__ function providing directly the options normally
    provided on the command line
    """

    default_nsbh_boundary_mass = 3.
    default_ns_eos = '2H'
    default_delta_bh_spin = 0.1
    default_delta_ns_mass = 0.1

    def __init__(self, minMass1, maxMass1, minMass2, maxMass2,
                 maxNSSpinMag=0, maxBHSpinMag=0, maxTotMass=None,
                 minTotMass=None, maxEta=None, minEta=0, 
                 max_chirp_mass=None, min_chirp_mass=None, 
                 ns_bh_boundary_mass=None, nsbhFlag=False,
                 remnant_mass_threshold=None, ns_eos=None, use_eos_max_ns_mass=False,
                 delta_bh_spin=None, delta_ns_mass=None):
        """
        Initialize an instance of the massRangeParameters by providing all
        these. For e.g. pycbc_aligned_stoch_bank --help
        """
        self.minMass1=minMass1
        self.maxMass1=maxMass1
        self.minMass2=minMass2
        self.maxMass2=maxMass2
        self.maxNSSpinMag=maxNSSpinMag
        self.maxBHSpinMag=maxBHSpinMag
        self.minTotMass = minMass1 + minMass2
        if minTotMass and (minTotMass > self.minTotMass):
            self.minTotMass = minTotMass
        self.maxTotMass = maxMass1 + maxMass2
        if maxTotMass and (maxTotMass < self.maxTotMass):
            self.maxTotMass = maxTotMass
        self.maxTotMass=maxTotMass
        self.minTotMass=minTotMass
        if maxEta:
            self.maxEta=maxEta
        else:
            self.maxEta=0.25
        self.max_chirp_mass = max_chirp_mass
        self.min_chirp_mass = min_chirp_mass
        self.minEta=minEta
        self.ns_bh_boundary_mass = (
            ns_bh_boundary_mass or self.default_nsbh_boundary_mass)
        self.nsbhFlag=nsbhFlag
        self.remnant_mass_threshold = remnant_mass_threshold
        self.ns_eos = (
            ns_eos or self.default_ns_eos)
        self.delta_bh_spin = (
            delta_bh_spin or self.default_delta_bh_spin)
        self.delta_ns_mass = (
            delta_ns_mass or self.default_delta_ns_mass)
        self.use_eos_max_ns_mass = use_eos_max_ns_mass
        if self.remnant_mass_threshold is not None:
            if self.ns_eos is not '2H':
                errMsg = """
                         By setting a value for --remnant-mass-threshold
                         you have asked to filter out EM dim NS-BH templates.
                         The EOS you chose is not supported currently: please
                         remove the --ns-eos option from your command line or
                         set it to '2H'.
                         """
                raise ValueError(errMsg)
            if use_eos_max_ns_mass:
                _, max_ns_g_mass = load_ns_sequence(self.ns_eos)
                if(self.maxMass2 > max_ns_g_mass):
                    errMsg = """
                             The maximum NS mass supported by this EOS is
                             {0}. Please set --max-mass2 to this value or run
                             without the --use-eos-max-ns-mass flag.
                             """.format(max_ns_g_mass-0.0000000001)
                    raise ValueError(errMsg)
            self.delta_bh_spin = (
                delta_bh_spin or self.default_delta_bh_spin)
            self.delta_ns_mass = (
                delta_ns_mass or self.default_delta_ns_mass)

        # FIXME: This may be inaccurate if Eta limits are given
        # This will not cause any problems, but maybe could be fixed.
        self.minCompMass = self.minMass2
        self.maxCompMass = self.maxMass1

        # WARNING: We expect mass1 > mass2 ALWAYS
        # Check input:
        if (minMass2 > minMass1) or (maxMass2 > maxMass1):
            errMsg = "Mass1 must be larger than mass2. Check input options."

        if (minMass2 > maxMass2) or (minMass1 > maxMass1):
            errMsg = "Minimum masses cannot be larger than maximum masses."
            errMsg += "Check input options."

    def from_argparse(cls, opts, nonSpin=False):
        """
        Initialize an instance of the massRangeParameters class from an
        insert_mass_range_option_group
        and
        verify_mass_range_options
        """
        if nonSpin:
            return cls(opts.min_mass1, opts.max_mass1, opts.min_mass2,
                       opts.max_mass2, maxTotMass=opts.max_total_mass,
                       minTotMass=opts.min_total_mass, maxEta=opts.max_eta,
                       minEta=opts.min_eta, max_chirp_mass=opts.max_chirp_mass,
                       min_chirp_mass=opts.min_chirp_mass,
                       remnant_mass_threshold=opts.remnant_mass_threshold,
                       ns_eos=opts.ns_eos, use_eos_max_ns_mass=opts.use_eos_max_ns_mass,
                       delta_bh_spin=opts.delta_bh_spin, delta_ns_mass=opts.delta_ns_mass)
        else:
                       minEta=opts.min_eta, maxNSSpinMag=opts.max_ns_spin_mag,
                       maxBHSpinMag=opts.max_bh_spin_mag,
                       nsbhFlag=opts.nsbh_flag,
                       max_chirp_mass=opts.max_chirp_mass,
                       ns_bh_boundary_mass=opts.ns_bh_boundary_mass,

    def is_outside_range(self, mass1, mass2, spin1z, spin2z):
        """
        Test if a given location in mass1, mass2, spin1z, spin2z is within the
        range of parameters allowed by the massParams object.
        """
        # Mass1 test
        if mass1 * 1.001 < self.minMass1:
            return 1
        if mass1 > self.maxMass1 * 1.001:
            return 1
        # Mass2 test
        if mass2 * 1.001 < self.minMass2:
            return 1
        if mass2 > self.maxMass2 * 1.001:
            return 1
        # Spin1 test
        if self.nsbhFlag:
            if (abs(spin1z) > self.maxBHSpinMag * 1.001):
                return 1
        else:
            spin1zM = abs(spin1z)
            if not( (mass1 * 1.001 > self.ns_bh_boundary_mass \
                     and spin1zM <= self.maxBHSpinMag * 1.001) \
                 or (mass1 < self.ns_bh_boundary_mass * 1.001 \
                     and spin1zM <= self.maxNSSpinMag * 1.001)):
                return 1
        # Spin2 test
            if (abs(spin2z) > self.maxNSSpinMag * 1.001):
                return 1
        else:
            spin2zM = abs(spin2z)
            if not( (mass2 * 1.001 > self.ns_bh_boundary_mass \
                     and spin2zM <= self.maxBHSpinMag * 1.001) \
                 or (mass2 < self.ns_bh_boundary_mass * 1.001 and \
                     spin2zM <= self.maxNSSpinMag * 1.001)):
                return 1
        # Total mass test
        mTot = mass1 + mass2
        if mTot > self.maxTotMass * 1.001:
            return 1
        if mTot * 1.001 < self.minTotMass:
            return 1

        # Eta test
        eta = mass1 * mass2 / (mTot * mTot)
        if eta > self.maxEta * 1.001:
            return 1
        if eta * 1.001 < self.minEta:
            return 1

        # Chirp mass test
        chirp_mass = mTot * eta**(3./5.)
        if self.min_chirp_mass is not None \
                and chirp_mass * 1.001 < self.min_chirp_mass:
            return 1
        if self.max_chirp_mass is not None \
                and chirp_mass > self.max_chirp_mass * 1.001:
            return 1

        return 0

class ethincaParameters(object):
    """
    insert_ethinca_metric_options
    """
    def __init__(self, pnOrder, cutoff, freqStep, fLow=None, full_ethinca=False,
                 time_ethinca=False):
        """
        Initialize an instance of ethincaParameters by providing all
        options directly.  See the insert_ethinca_metric_options() function
        for explanation or e.g. run pycbc_geom_nonspinbank --help
        """
        self.full_ethinca=full_ethinca
        self.time_ethinca=time_ethinca
        self.doEthinca= self.full_ethinca or self.time_ethinca
        self.cutoff=cutoff
        self.freqStep=freqStep
        # independent fLow for ethinca metric is currently not used
        # check that ethinca options make sense
        if self.full_ethinca and self.time_ethinca:
            err_msg = "It does not make sense to ask me to do the time "
            err_msg += "restricted ethinca and also the full ethinca."
        if self.doEthinca and not (
                cutoff in pnutils.named_frequency_cutoffs.keys()):
            raise ValueError("Need a valid cutoff formula to calculate "
                             "ethinca! Possible values are "+
                             str(tuple(pnutils.named_frequency_cutoffs.keys())))
        if self.doEthinca and not freqStep:
            raise ValueError("Need to specify a cutoff frequency step to "
                             "calculate ethinca! (ethincaFreqStep)")

        """
        Initialize an instance of the ethincaParameters class from an
        insert_ethinca_metric_options
        and
        verify_ethinca_metric_options
        """
        return cls(opts.ethinca_pn_order, opts.filter_cutoff,
            opts.ethinca_frequency_step, fLow=None,
            full_ethinca=opts.calculate_ethinca_metric,
            time_ethinca=opts.calculate_time_metric_components)

def insert_ethinca_metric_options(parser):
    """
    Adds the options used to calculate the ethinca metric, if required.
 
    """
    ethincaGroup = parser.add_argument_group("Ethinca metric options",
                    "Options used in the calculation of Gamma metric "
                    "components for the ethinca coincidence test and for "
                    "assigning high-frequency cutoffs to templates.")
    ethinca_methods = ethincaGroup.add_mutually_exclusive_group()
    ethinca_methods.add_argument("--calculate-time-metric-components",
                    action="store_true", default=False,
                    help="If given, the ethinca metric will be calculated "
                    "for only the time component, and stored in the Gamma0 "
                    "entry of the sngl_inspiral table. OPTIONAL, default=False")
    ethinca_methods.add_argument("--calculate-ethinca-metric",
                    action="store_true", default=False, 
                    "and stored in the Gamma entries of the sngl_inspiral "
                    "table. OPTIONAL, default=False")
    ethincaGroup.add_argument("--ethinca-pn-order",
                    default=None, choices=get_ethinca_orders(), 
                    help="Specify a PN order to be used in calculating the "
                    "ethinca metric. OPTIONAL: if not specified, the same "
                    "order will be used as for the bank metric.")
    ethincaGroup.add_argument("--filter-cutoff",
                    default=None, 
                    choices=tuple(pnutils.named_frequency_cutoffs.keys()),
                    help="Specify an upper frequency cutoff formula for the "
                    "ethinca metric calculation, and for the values of f_final"
                    " assigned to the templates.  REQUIRED if the "
                    "calculate-ethinca-metric option is given.")
    ethincaGroup.add_argument("--ethinca-frequency-step", action="store",
                    type=float, default=10.,
                    help="Control the precision of the upper frequency cutoff."
                    " For speed, the metric is calculated only for discrete "
                    "f_max values with a spacing given by this option. Each "
                    "template is assigned the metric for the f_max closest to "
                    "its analytical cutoff formula. OPTIONAL, default=10. "
                    "UNITS=Hz")

    return ethincaGroup

def verify_ethinca_metric_options(opts, parser):
    """
    Checks that the necessary options are given for the ethinca metric

    """
    if opts.filter_cutoff is not None and not (opts.filter_cutoff in
              pnutils.named_frequency_cutoffs.keys()):
        parser.error("Need a valid cutoff formula to calculate ethinca or "
                     "assign filter f_final values! Possible values are "
                     +str(tuple(pnutils.named_frequency_cutoffs.keys())))
    if (opts.calculate_ethinca_metric or opts.calculate_time_metric_components)\
                                           and not opts.ethinca_frequency_step:
        parser.error("Need to specify a cutoff frequency step to calculate "
                     "ethinca!")
    if not (opts.calculate_ethinca_metric or\
              opts.calculate_time_metric_components) and opts.ethinca_pn_order:
        parser.error("Can't specify an ethinca PN order if not "
                     "calculating ethinca metric!")

def check_ethinca_against_bank_params(ethincaParams, metricParams):
    """
    Cross-check the ethinca and bank layout metric calculation parameters
    and set the ethinca metric PN order equal to the bank PN order if not
    previously set.

    ethincaParams: instance of ethincaParameters
    metricParams: instance of metricParameters
    """
    if ethincaParams.doEthinca:
        if metricParams.f0 != metricParams.fLow:
            raise ValueError("If calculating ethinca metric, f0 and f-low "
                             "must be equal!")
        if ethincaParams.fLow is not None and (
                ethincaParams.fLow != metricParams.fLow):
            raise ValueError("Ethinca metric calculation does not currently "
                             "support a f-low value different from the bank "
                             "metric!")
        if ethincaParams.pnOrder is None:
            ethincaParams.pnOrder = metricParams.pnOrder
    else: pass

#
#
#

from pycbc.tmpltbank import coord_utils

class PartitionedTmpltbank(object):
    """
    This class is used to hold a template bank partitioned into numerous bins
    based on position in the Cartesian parameter space where the axes are the
    principal components. It can also be used to hold intermediary
    products used while constructing (e.g.) a stochastic template bank.
    """
    def __init__(self, mass_range_params, metric_params, ref_freq,
                 bin_spacing, bin_range_check=1):
        """
        Set up the partitioned template bank class. The combination of the
        reference frequency, the bin spacing and the metric dictates how the
        parameter space will be partitioned.

        mass_range_params : massRangeParameters object
            An initialized massRangeParameters object holding the details of
            the mass and spin ranges being considered.
        metric_params : metricParameters object
            An initialized metricParameters object holding the details of the
            parameter space metric that is being used.
        ref_freq : float
            The reference frequency to use as the upper frequency cutoff of
            the metric when partitioning the bank. In general this would be
            set to the *smallest* upper frequency cutoff that is possible in
            the given parameter space. However, in some cases this can lead
            to only a small number of partitions and the computational cost
            will increase dramatically. NOTE: when using the vary-fupper
            option this upper frequency cutoff is only used to determine which
            points should be matched against each other, it is *not* used in
            the actual metric-based calculation of the distance (which uses the
            frequency cutoffs of the points being considered).
        bin_spacing : float
            The metric distance to space the bins by. NOTE: If you want to
            place the bins to have a width corresponding to a minimal match of
            0.97 you would set this to :math:`(1 - 0.97)^{0.5}`.
            Note the square root,
            matches correspond to the square of parameter space distance.
        bin_range_check : int
            When computing matches consider points in the corresponding bin and
            all bins +/- this value in both chi_1 and chi_2 directions.
            DEFAULT = 1.
        """
        # Flags to be used in other methods of this class. Initialized here for
        # simplicity
        self.spin_warning_given = False

        # These will probably be used a lot, so add to object
        self.mass_range_params = mass_range_params
        self.metric_params = metric_params
        self.ref_freq = ref_freq
        self.bin_spacing = bin_spacing

        # Get parameter space extent
        vals = coord_utils.estimate_mass_range(1000000, mass_range_params,
                                          metric_params, ref_freq, covary=True)
        chi1_max = vals[0].max()
        chi1_min = vals[0].min()
        chi1_diff = chi1_max - chi1_min
        chi2_max = vals[1].max()
        chi2_min = vals[1].min()
        chi2_diff = chi2_max - chi2_min
        # Add a little bit extra as we may not have reached the edges.
        # FIXME: Maybe better to use the numerical code to find maxima here?
        chi1_min = chi1_min - 0.1*chi1_diff
        chi1_max = chi1_max + 0.1*chi1_diff
        chi2_min = chi2_min - 0.1*chi2_diff
        chi2_max = chi2_max + 0.1*chi2_diff

        massbank = {}
        bank = {}
        # Also add a little bit here
        for i in range(-2, int((chi1_max - chi1_min) // bin_spacing + 2)):
            bank[i] = {}
            massbank[i] = {}
            for j in range(-2, int((chi2_max - chi2_min) // bin_spacing + 2)):
                bank[i][j] = []
                massbank[i][j] = {}
                massbank[i][j]['mass1s'] = numpy.array([])

        self.massbank = massbank
        self.bank = bank
        # Record minimum and maximum bins
        self.min_chi1_bin = -2
        self.min_chi2_bin = -2
        self.max_chi1_bin = int((chi1_max - chi1_min) // bin_spacing + 1)
        self.max_chi2_bin = int((chi2_max - chi2_min) // bin_spacing + 1)
        self.chi1_min = chi1_min
        self.chi1_max = chi1_max
        self.chi2_min = chi2_min
        self.chi2_max = chi2_max

        # How many adjacent bins should we check?
        self.bin_range_check = 1
        self.bin_loop_order = coord_utils.outspiral_loop(self.bin_range_check)

    def get_point_from_bins_and_idx(self, chi1_bin, chi2_bin, idx):
        """Find masses and spins given bin numbers and index.

        Given the chi1 bin, chi2 bin and an index, return the masses and spins
        of the point at that index. Will fail if no point exists there.

        chi1_bin : int
            The bin number for chi1.
        chi2_bin : int
            The bin number for chi2.
        idx : int
            The index within the chi1, chi2 bin.

        Returns
        --------
        mass1 : float
            Mass of heavier body.
        mass2 : float
            Mass of lighter body.
        spin1z : float
            Spin of heavier body.
        spin2z : float
            Spin of lighter body.
        """
        mass1 = self.massbank[chi1_bin][chi2_bin]['mass1s'][idx]
        mass2 = self.massbank[chi1_bin][chi2_bin]['mass2s'][idx]
        spin1z = self.massbank[chi1_bin][chi2_bin]['spin1s'][idx]
        spin2z = self.massbank[chi1_bin][chi2_bin]['spin2s'][idx]
        return mass1, mass2, spin1z, spin2z

    def get_freq_map_and_normalizations(self, frequency_list,
                                        upper_freq_formula):
        """
        If using the --vary-fupper capability we need to store the mapping
        between index and frequencies in the list. We also precalculate the
        normalization factor at every frequency, which is used when estimating
        overlaps to account for abrupt changes in termination frequency.

        frequency_list : array of floats
            The frequencies for which the metric has been computed and lie
            within the parameter space being considered.
        upper_freq_formula : string
        """
        self.frequency_map = {}
        self.normalization_map = {}
        self.upper_freq_formula = upper_freq_formula
        # FIXME: Must this be sorted on input
        frequency_list.sort()

        for idx, frequency in enumerate(frequency_list):
            self.frequency_map[frequency] = idx
            self.normalization_map[frequency] = \
                             (self.metric_params.moments['I7'][frequency])**0.5

    def find_point_bin(self, chi_coords):
        """
        Given a set of coordinates in the chi parameter space, identify the
        indices of the chi1 and chi2 bins that the point occurs in. Returns
        these indices.

        chi_coords : numpy.array
            The position of the point in the chi coordinates.

        Returns
        --------
            Index of the chi_1 bin.
            Index of the chi_2 bin.
        """
        # Identify bin
        chi1_bin = int((chi_coords[0] - self.chi1_min) // self.bin_spacing)
        chi2_bin = int((chi_coords[1] - self.chi2_min) // self.bin_spacing)
        self.check_bin_existence(chi1_bin, chi2_bin)
        return chi1_bin, chi2_bin

    def check_bin_existence(self, chi1_bin, chi2_bin):
        """
        Given indices for bins in chi1 and chi2 space check that the bin
        exists in the object. If not add it. Also check for the existence of
        all bins within +/- self.bin_range_check and add if not present.

            The index of the chi1_bin to check
            The index of the chi2_bin to check
        """
        bin_range_check = self.bin_range_check
        # Check if this bin actually exists. If not add it
        if ( (chi1_bin < self.min_chi1_bin+bin_range_check) or
             (chi1_bin > self.max_chi1_bin-bin_range_check) or
             (chi2_bin < self.min_chi2_bin+bin_range_check) or
             (chi2_bin > self.max_chi2_bin-bin_range_check) ):
            for temp_chi1 in range(chi1_bin-bin_range_check,
                                                   chi1_bin+bin_range_check+1):
                if temp_chi1 not in self.massbank:
                    self.massbank[temp_chi1] = {}
                    self.bank[temp_chi1] = {}
                for temp_chi2 in range(chi2_bin-bin_range_check,
                                                   chi2_bin+bin_range_check+1):
                    if temp_chi2 not in self.massbank[temp_chi1]:
                        self.massbank[temp_chi1][temp_chi2] = {}
                        self.massbank[temp_chi1][temp_chi2]['mass1s'] =\
                                                                numpy.array([])
                        self.bank[temp_chi1][temp_chi2] = []

    def calc_point_distance(self, chi_coords):
        """
        Calculate distance between point and the bank. Return the closest
        distance.

        Returns
        --------
        min_dist : float
            The smallest **SQUARED** metric distance between the test point and
            the bank.
        indexes : The chi1_bin, chi2_bin and position within that bin at which
            the closest matching point lies.
        """
        chi1_bin, chi2_bin = self.find_point_bin(chi_coords)
        min_dist = 1000000000
        indexes = None
        for chi1_bin_offset, chi2_bin_offset in self.bin_loop_order:
            curr_chi1_bin = chi1_bin + chi1_bin_offset
            curr_chi2_bin = chi2_bin + chi2_bin_offset
            for idx, bank_chis in \
                            enumerate(self.bank[curr_chi1_bin][curr_chi2_bin]):
                dist = coord_utils.calc_point_dist(chi_coords, bank_chis)
                if dist < min_dist:
                    min_dist = dist
                    indexes = (curr_chi1_bin, curr_chi2_bin, idx)
        return min_dist, indexes

    def test_point_distance(self, chi_coords, distance_threshold):
        """
        Test if the distance between the supplied point and the bank is less
        than the supplied distance theshold.

        distance_threshold : float
            The **SQUARE ROOT** of the metric distance to test as threshold.
            E.g. if you want to test to a minimal match of 0.97 you would
            use 1 - 0.97 = 0.03 for this value.

        Returns
        --------
        Boolean
            True if point is within the distance threshold. False if not.

        """
            for bank_chis in self.bank[curr_chi1_bin][curr_chi2_bin]:
                if dist < distance_threshold:
                    return True
        else:

    def calc_point_distance_vary(self, chi_coords, point_fupper, mus):
        """
        Calculate distance between point and the bank allowing the metric to
        vary based on varying upper frequency cutoff. Slower than
        calc_point_distance, but more reliable when upper frequency cutoff can
        change a lot.

        point_fupper : float
            The upper frequency cutoff to use for this point. This value must
            be one of the ones already calculated in the metric.
        mus : numpy.array
            A 2D array where idx 0 holds the upper frequency cutoff and idx 1
            holds the coordinates in the [not covaried] mu parameter space for
            each value of the upper frequency cutoff.

        Returns
        --------
            the bank.
        """
            # No points = Next iteration
            curr_bank = self.massbank[curr_chi1_bin][curr_chi2_bin]
            if not curr_bank['mass1s'].size:
                continue

            # *NOT* the same of .min and .max
            f_upper = numpy.minimum(point_fupper, curr_bank['freqcuts'])
            f_other = numpy.maximum(point_fupper, curr_bank['freqcuts'])
            # NOTE: freq_idxes is a vector!
            freq_idxes = numpy.array([self.frequency_map[f] for f in f_upper])
            # vecs1 gives a 2x2 vector: idx0 = stored index, idx1 = mu index
            vecs1 = mus[freq_idxes, :]
            # vecs2 gives a 2x2 vector: idx0 = stored index, idx1 = mu index
            range_idxes = numpy.arange(len(freq_idxes))
            vecs2 = curr_bank['mus'][range_idxes, freq_idxes, :]

            # Now do the sums
            dists = (vecs1 - vecs2)*(vecs1 - vecs2)
            # This reduces to 1D: idx = stored index
            dists = numpy.sum(dists, axis=1)
            norm_upper = numpy.array([self.normalization_map[f] \
                                      for f in f_upper])
            norm_other = numpy.array([self.normalization_map[f] \
                                      for f in f_other])
            norm_fac = norm_upper / norm_other
            renormed_dists = 1 - (1 - dists)*norm_fac
            curr_min_dist = renormed_dists.min()
            if curr_min_dist < min_dist:
                min_dist = curr_min_dist
                indexes = curr_chi1_bin, curr_chi2_bin, renormed_dists.argmin()

    def test_point_distance_vary(self, chi_coords, point_fupper, mus,
                                 distance_threshold):
        """
        Test if distance between point and the bank is greater than distance
        threshold while allowing the metric to
        test_point_distance, but more reliable when upper frequency cutoff can

        Returns
        --------
        Boolean
        """
                continue

            vecs2 = curr_bank['mus'][range_idxes,freq_idxes,:]

            # I wonder if this line actually speeds things up?
            if (dists > distance_threshold).all():
                continue
            # This is only needed for close templates, should we prune?
            if (renormed_dists < distance_threshold).any():
        else:

    def add_point_by_chi_coords(self, chi_coords, mass1, mass2, spin1z, spin2z,
                          point_fupper=None, mus=None):
        """
        Add a point to the partitioned template bank. The point_fupper and mus
        kwargs must be provided for all templates if the vary fupper capability
        is desired. This requires that the chi_coords, as well as mus and
        point_fupper if needed, to be precalculated. If you just have the
        masses and don't want to worry about translations see
        add_point_by_masses, which will do translations and then call this.

            The heavier mass of the point to add.
            The lighter mass of the point to add.
        spin1z: float
            The [aligned] spin on the heavier body.
        spin2z: float
            The [aligned] spin on the lighter body.
        """
        self.bank[chi1_bin][chi2_bin].append(copy.deepcopy(chi_coords))
        curr_bank = self.massbank[chi1_bin][chi2_bin]

        if curr_bank['mass1s'].size:
            curr_bank['mass1s'] = numpy.append(curr_bank['mass1s'],
                                               numpy.array([mass1]))
            curr_bank['mass2s'] = numpy.append(curr_bank['mass2s'],
                                               numpy.array([mass2]))
            curr_bank['spin1s'] = numpy.append(curr_bank['spin1s'],
                                               numpy.array([spin1z]))
            curr_bank['spin2s'] = numpy.append(curr_bank['spin2s'],
                                               numpy.array([spin2z]))
            if point_fupper is not None:
                curr_bank['freqcuts'] = numpy.append(curr_bank['freqcuts'],
                                                 numpy.array([point_fupper]))
            # Mus needs to append onto axis 0. See below for contents of
            # the mus variable
            if mus is not None:
                curr_bank['mus'] = numpy.append(curr_bank['mus'],
                                            numpy.array([mus[:,:]]), axis=0)
        else:
            curr_bank['mass1s'] = numpy.array([mass1])
            curr_bank['mass2s'] = numpy.array([mass2])
            curr_bank['spin1s'] = numpy.array([spin1z])
            curr_bank['spin2s'] = numpy.array([spin2z])
                curr_bank['freqcuts'] = numpy.array([point_fupper])
            # curr_bank['mus'] is a 3D array
            # NOTE: mu relates to the non-covaried Cartesian coordinate system
            # Axis 0: Template index
            # Axis 1: Frequency cutoff index
            # Axis 2: Mu coordinate index
                curr_bank['mus'] = numpy.array([mus[:,:]])

    def add_point_by_masses(self, mass1, mass2, spin1z, spin2z,
                            vary_fupper=False):
        """
        Add a point to the template bank. This differs from add point to bank
        as it assumes that the chi coordinates and the products needed to use
        vary_fupper have not already been calculated. This function calculates
        these products and then calls add_point_by_chi_coords.
        This function also
        carries out a number of sanity checks (eg. is the point within the
        ranges given by mass_range_params) that add_point_by_chi_coords does
        not do for speed concerns.

            Mass of the heavier body
            Mass of the lighter body
            Spin of the heavier body
            Spin of the lighter body
        """
        # Test that masses are the expected way around (ie. mass1 > mass2)
        if mass2 > mass1:
            if not self.spin_warning_given:
                warn_msg = "Am adding a template where mass2 > mass1. The "
                warn_msg += "convention is that mass1 > mass2. Swapping mass1 "
                warn_msg += "and mass2 and adding point to bank. This message "
                warn_msg += "will not be repeated."
                logging.warn(warn_msg)
                self.spin_warning_given = True

        # These that masses obey the restrictions of mass_range_params
        if self.mass_range_params.is_outside_range(mass1, mass2, spin1z,
                                                                       spin2z):
            err_msg = "Point with masses given by "
            err_msg += "%f %f %f %f " %(mass1, mass2, spin1z, spin2z)
            err_msg += "(mass1, mass2, spin1z, spin2z) is not consistent "
            err_msg += "with the provided command-line restrictions on masses "
            err_msg += "and spins."

        # Get chi coordinates
        chi_coords = coord_utils.get_cov_params(mass1, mass2, spin1z, spin2z,
                                                self.metric_params,
                                                self.ref_freq)

        # Get mus and best fupper for this point, if needed
        if vary_fupper:
            mass_dict = {}
            mass_dict['m1'] = numpy.array([mass1])
            mass_dict['m2'] = numpy.array([mass2])
            mass_dict['s1z'] = numpy.array([spin1z])
            mass_dict['s2z'] = numpy.array([spin2z])
            freqs = numpy.array([self.frequency_map.keys()], dtype=float)
            freq_cutoff = coord_utils.return_nearest_cutoff(\
                                     self.upper_freq_formula, mass_dict, freqs)
            freq_cutoff = freq_cutoff[0]
            lambdas = coord_utils.get_chirp_params\
                (mass1, mass2, spin1z, spin2z, self.metric_params.f0,
                 self.metric_params.pnOrder)
            mus = []
            for freq in self.frequency_map:
                mus.append(coord_utils.get_mu_params(lambdas,
                                                    self.metric_params, freq) )
            mus = numpy.array(mus)
        else:
            freq_cutoff=None
            mus=None

        self.add_point_by_chi_coords(chi_coords, mass1, mass2, spin1z, spin2z,
                               point_fupper=freq_cutoff, mus=mus)

    def add_tmpltbank_from_xml_table(self, sngl_table, vary_fupper=False):
        """
        This function will take a sngl_inspiral_table of templates and add them
        into the partitioned template bank object.

        sngl_table : sngl_inspiral_table
            List of sngl_inspiral templates.
        vary_fupper : False
            If given also include the additional information needed to compute
            distances with a varying upper frequency cutoff.
        """
        for sngl in sngl_table:
            self.add_point_by_masses(sngl.mass1, sngl.mass2, sngl.spin1z,
                                     sngl.spin2z, vary_fupper=vary_fupper)

    def add_tmpltbank_from_hdf_file(self, hdf_fp, vary_fupper=False):
        """
        This function will take a pointer to an open HDF File object containing
        a list of templates and add them into the partitioned template bank
        object.

        hdf_fp : h5py.File object
            The template bank in HDF5 format.
        """
        mass1s = hdf_fp['mass1'][:]
        mass2s = hdf_fp['mass2'][:]
        spin1zs = hdf_fp['spin1z'][:]
        spin2zs = hdf_fp['spin2z'][:]
        for idx in range(len(mass1s)):
            self.add_point_by_masses(mass1s[idx], mass2s[idx], spin1zs[idx],
                                     spin2zs[idx], vary_fupper=vary_fupper)

    def output_all_points(self):
        """Return all points in the bank.

        Return all points in the bank as lists of m1, m2, spin1z, spin2z.

        Returns
        -------
        mass1 : list
            List of mass1 values.
        mass2 : list
            List of mass2 values.
        spin1z : list
            List of spin1z values.
        spin2z : list
            List of spin2z values.
        """
        mass1 = []
        mass2 = []
        spin1z = []
        spin2z = []
        for i in self.massbank.keys():
            for j in self.massbank[i].keys():
                for k in range(len(self.massbank[i][j]['mass1s'])):
                    curr_bank = self.massbank[i][j]
                    mass1.append(curr_bank['mass1s'][k])
                    mass2.append(curr_bank['mass2s'][k])
                    spin1z.append(curr_bank['spin1s'][k])
                    spin2z.append(curr_bank['spin2s'][k])

# Copyright (C) 2014  Josh Willis, Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
"""
This module provides a class derived from numpy.ndarray that also indicates
whether or not its memory is aligned.  It further provides functions for
creating zeros and empty (unitialized) arrays with this class.
"""
from pycbc import PYCBC_ALIGNMENT

def check_aligned(ndarr):
    return ((ndarr.ctypes.data % PYCBC_ALIGNMENT) == 0)

def zeros(n, dtype):
    d = _np.dtype(dtype)
    nbytes = (d.itemsize)*int(n)
    tmp = _np.zeros(nbytes+PYCBC_ALIGNMENT, dtype=_np.uint8)
    address = tmp.__array_interface__['data'][0]
    offset = (PYCBC_ALIGNMENT - address%PYCBC_ALIGNMENT)%PYCBC_ALIGNMENT
    ret_ary = tmp[offset:offset+nbytes].view(dtype=d)
    del tmp
    return ret_ary

def empty(n, dtype):
    tmp = _np.empty(nbytes+PYCBC_ALIGNMENT, dtype=_np.uint8)
    del tmp

# Copyright (C) 2012  Alex Nitz, Josh Willis, Andrew Miller, Tito Dal Canton
#
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides a device independent Array class based on PyCUDA and Numpy.
"""

BACKEND_PREFIX="pycbc.types.array_"

import os as _os

import lal as _lal
import numpy as _numpy
from numpy import float32, float64, complex64, complex128, ones
from numpy.linalg import norm

from pycbc.scheme import schemed, cpuonly

#! FIXME: the uint32 datatype has not been fully tested,
# we should restrict any functions that do not allow an
# array of uint32 integers
_ALLOWED_DTYPES = [_numpy.float32, _numpy.float64, _numpy.complex64,
                   _numpy.complex128, _numpy.uint32, _numpy.int32, _numpy.int]
try:
    _ALLOWED_SCALARS = [int, long, float, complex] + _ALLOWED_DTYPES
except NameError:
    _ALLOWED_SCALARS = [int, float, complex] + _ALLOWED_DTYPES

def _convert_to_scheme(ary):
    if not isinstance(ary._scheme, _scheme.mgr.state.__class__):
        converted_array = Array(ary, dtype=ary._data.dtype)
        ary._data = converted_array._data
        ary._scheme = _scheme.mgr.state
      
@decorator  
def _convert(fn, self, *args):
    # Convert this array to the current processing scheme
    _convert_to_scheme(self)
    return fn(self, *args)
    
def _nocomplex(fn, self, *args):
    if self.kind == 'real':
        return fn(self, *args)
    else:
        raise TypeError( fn.__name__ + " does not support complex types")
        
def _noreal(fn, self, *args):
    if self.kind == 'complex':
    else:
        raise TypeError( fn.__name__ + " does not support real types")

def force_precision_to_match(scalar, precision):
    if _numpy.iscomplexobj(scalar):
        if precision is 'single':
            return _numpy.complex64(scalar)
        else:
            return _numpy.complex128(scalar)
    else:
            return _numpy.float32(scalar)
        else:
            return _numpy.float64(scalar)

def common_kind(*dtypes):
    for dtype in dtypes:
        if dtype.kind is 'c':
            return dtype
    return dtypes[0]
   
@schemed(BACKEND_PREFIX) 
def _to_device(array):
    """ Move input to device """
    
@schemed(BACKEND_PREFIX)
def _copy_base_array(array):
    """ Copy a backend array"""

def _scheme_matches_base_array(array):
    """ Check that input matches array type for scheme """

class Array(object):
    """Array used to do numeric calculations on a various compute
    devices. It is a convience wrapper around numpy, and
    pycuda.
    """

    def __init__(self, initial_array, dtype=None, copy=True):
        """ initial_array: An array-like object as specified by NumPy, this
        also includes instances of an underlying data type as described in
        section 3 or an instance of the PYCBC Array class itself. This
        object is used to populate the data of the array.

        dtype: A NumPy style dtype that describes the type of
        encapsulated data (float32,compex64, etc)

        copy: This defines whether the initial_array is copied to instantiate
        the array or is simply referenced. If copy is false, new data is not
        created, and so all arguments that would force a copy are ignored.
        The default is to copy the given object.
        """
        self._scheme=_scheme.mgr.state
        self._saved = LimitedSizeDict(size_limit=2**5)
        
        #Unwrap initial_array
        if isinstance(initial_array, Array):
            initial_array = initial_array._data

        if not copy:
            if not _scheme_matches_base_array(initial_array):
                raise TypeError("Cannot avoid a copy of this array")
            else:
                self._data = initial_array

            # Check that the dtype is supported.
            if self._data.dtype not in _ALLOWED_DTYPES:
                raise TypeError(str(self._data.dtype) + ' is not supported')

            if dtype and dtype != self._data.dtype:
                raise TypeError("Can only set dtype when allowed to copy data")

        if copy:
            # First we will check the dtype that we are given
            if not hasattr(initial_array, 'dtype'):
                initial_array = _numpy.array(initial_array)

            # Determine the dtype to use
            if dtype is not None:  
                dtype = _numpy.dtype(dtype)
                if dtype not in _ALLOWED_DTYPES:
                    raise TypeError(str(dtype) + ' is not supported')
                if dtype.kind != 'c' and initial_array.dtype.kind == 'c':
                    raise TypeError(str(initial_array.dtype) + ' cannot be cast as ' + str(dtype))          
            elif initial_array.dtype in _ALLOWED_DTYPES:
                dtype = initial_array.dtype
            else:
                if initial_array.dtype.kind == 'c':
                    dtype = complex128
                else:
                    dtype = float64
                     
            # Cast to the final dtype if needed
            if initial_array.dtype != dtype:
                initial_array = initial_array.astype(dtype)
                                              
            #Create new instance with initial_array as initialization.
            if issubclass(type(self._scheme), _scheme.CPUScheme):
                if hasattr(initial_array, 'get'):
                    self._data = _numpy.array(initial_array.get())
                else:
                    self._data = _numpy.array(initial_array, dtype=dtype, ndmin=1)
            elif _scheme_matches_base_array(initial_array):
                self._data = _copy_base_array(initial_array) # pylint:disable=assignment-from-no-return
            else:
                initial_array = _numpy.array(initial_array, dtype=dtype, ndmin=1)
                self._data = _to_device(initial_array) # pylint:disable=assignment-from-no-return

    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
        inputs = [i.numpy() if isinstance(i, Array) else i for i in inputs]
        ret = getattr(ufunc, method)(*inputs, **kwargs)
        if hasattr(ret, 'shape') and ret.shape == self.shape:
            ret = self._return(ret)
        return ret

    def __array__(self, dtype=None):
        arr = self.numpy()
        if dtype is not None:
            arr = arr.astype(dtype)
        return arr

    @property
    def shape(self):
        return self._data.shape
     
    def _memoize_single(fn, self, arg):
        badh = str(arg)
        
        if badh in self._saved:
            return self._saved[badh]

        res = fn(self, arg) # pylint:disable=not-callable
        self._saved[badh] = res      
        return res
                   
    def _returnarray(fn, self, *args):
        return Array(fn(self, *args), copy=False) # pylint:disable=not-callable

    def _returntype(fn, self, *args):
        ary = fn(self,*args) # pylint:disable=not-callable
        if ary is NotImplemented:
            return NotImplemented
        return self._return(ary)
        
    def _return(self, ary):
        """Wrap the ary to return an Array type """
        if isinstance(ary, Array):
            return ary
        return Array(ary, copy=False)

    def _checkother(fn, self, *args):
        nargs = ()
        for other in args:
            self._typecheck(other)  
            if type(other) in _ALLOWED_SCALARS:
                other = force_precision_to_match(other, self.precision)
                nargs +=(other,)
            elif isinstance(other, type(self)) or type(other) is Array:
                if len(other) != len(self):
                    raise ValueError('lengths do not match')
                if other.precision == self.precision:
                    _convert_to_scheme(other)
                    nargs += (other._data,)
                else:
                    raise TypeError('precisions do not match')
            else:
                return NotImplemented

        return fn(self, *nargs) # pylint:disable=not-callable
    
    @decorator  
    def _vcheckother(fn, self, *args):
        nargs = ()
            if isinstance(other, type(self)) or type(other) is Array:
                else:
            else:
                raise TypeError('array argument required')                    

        return fn(self,*nargs) # pylint:disable=not-callable
        
    def _vrcheckother(fn, self,*args):
        nargs = ()
                else:
            else:

    def _icheckother(fn, self, other):
        """ Checks the input to in-place operations """
        self._typecheck(other) 
        if type(other) in _ALLOWED_SCALARS:
            if self.kind == 'real' and type(other) == complex:
                raise TypeError('dtypes are incompatible')
            other = force_precision_to_match(other, self.precision)
        elif isinstance(other, type(self)) or type(other) is Array:
            if len(other) != len(self):
                raise ValueError('lengths do not match')
            if self.kind == 'real' and other.kind == 'complex':
            if other.precision == self.precision:
                _convert_to_scheme(other)
                other = other._data
            else:
                raise TypeError('precisions do not match')
        else:

        return fn(self, other) # pylint:disable=not-callable

    def _typecheck(self, other):
        """ Additional typechecking for other. Placeholder for use by derived
        types. 
        """
        pass

    @_returntype
    @_convert
    @_checkother
    def __mul__(self,other):
        """ Multiply by an Array or a scalar and return an Array. """
        return self._data * other

    __rmul__ = __mul__

    @_convert
    @_icheckother
    def __imul__(self,other):
        self._data *= other

    @_convert
    def __add__(self,other):
        """ Add Array to Array or scalar and return an Array. """
        return self._data + other

    __radd__ = __add__
       
    def fill(self, value):
        self._data.fill(value)

    @_convert
    def __iadd__(self,other):
        self._data += other

    @_convert
    def __truediv__(self,other):
        """ Divide Array by Array or scalar and return an Array. """
        return self._data / other

    @_convert
    def __rtruediv__(self,other):
        return self._data.__rtruediv__(other)

    @_convert
    def __itruediv__(self,other):
        self._data /= other
        
    __div__ = __truediv__
    __idiv__ = __itruediv__
    __rdiv__ = __rtruediv__

    @_convert
    def __neg__(self):
        """ Return negation of self """
        return - self._data

    @_convert
    def __sub__(self,other):
        """ Subtract Array or scalar from Array and return an Array. """
        return self._data - other

    @_convert
    def __rsub__(self,other):
        return self._data.__rsub__(other)

    @_convert
    def __isub__(self,other):
        self._data -= other

    @_convert
    def __pow__(self,other):
        """ Exponentiate Array by scalar """
        return self._data ** other

    @_convert
        """ Return absolute value of Array """
        return abs(self._data)

        """ Return length of Array """
        return len(self._data)

    def __str__(self):
        return str(self._data)
        
    @property
    def ndim(self):
        return self._data.ndim

    def __eq__(self,other):
        """
        This is the Python special method invoked whenever the '=='
        comparison is used.  It will return true if the data of two
        PyCBC arrays are identical, and all of the numeric meta-data
        are identical, irrespective of whether or not the two
        instances live in the same memory (for that comparison, the
        Python statement 'a is b' should be used instead).

        Thus, this method returns 'True' if the types of both 'self'
        and 'other' are identical, as well as their lengths, dtypes
        and the data in the arrays, element by element.  It will always
        do the comparison on the CPU, but will *not* move either object
        to the CPU if it is not already there, nor change the scheme of
        either object. It is possible to compare a CPU object to a GPU
        object, and the comparison should be true if the data and
        meta-data of the two objects are the same.

        Note in particular that this function returns a single boolean,
        and not an array of booleans as Numpy does.  If the numpy
        behavior is instead desired it can be obtained using the numpy()
        method of the PyCBC type to get a numpy instance from each
        object, and invoking '==' on those two instances.

        other: another Python object, that should be tested for equality
            with 'self'.

        Returns
        -------
        boolean: 'True' if the types, dtypes, lengths, and data of the
            two objects are each identical.
        """

        # Writing the first test as below allows this method to be safely
        # called from subclasses.
        if type(self) != type(other):
        if self.dtype != other.dtype:
        if len(self) != len(other):

        # Now we've checked meta-data, so look at the actual data itself:
        # The numpy() method call will put a copy of GPU data onto a CPU
        # array, and could therefore be slow.  As noted in the help for
        # this function we don't worry about that.

        sary = self.numpy()
        oary = other.numpy()

        # Now we know that both sary and oary are numpy arrays. The
        # '==' statement returns an array of booleans, and the all()
        # method of that array returns 'True' only if every element
        # of that array of booleans is True.
        return (sary == oary).all()

    def almost_equal_elem(self,other,tol,relative=True):
        """
        Compare whether two array types are almost equal, element
        by element.

        If the 'relative' parameter is 'True' (the default) then the
        'tol' parameter (which must be positive) is interpreted as a
        relative tolerance, and the comparison returns 'True' only if
        abs(self[i]-other[i]) <= tol*abs(self[i])
        for all elements of the array.

        If 'relative' is 'False', then 'tol' is an absolute tolerance,
        and the comparison is true only if
        abs(self[i]-other[i]) <= tol

        Other meta-data (type, dtype, and length) must be exactly equal.
        If either object's memory lives on the GPU it will be copied to
        the CPU for the comparison, which may be slow.  But the original
        object itself will not have its memory relocated nor scheme
        changed.

        other
            Another Python object, that should be tested for
            almost-equality with 'self', element-by-element.
        tol
            A non-negative number, the tolerance, which is interpreted
            as either a relative tolerance (the default) or an absolute
            tolerance.
        relative
            A boolean, indicating whether 'tol' should be interpreted
            as a relative tolerance (if True, the default if this argument
            is omitted) or as an absolute tolerance (if tol is False).

        Returns
        -------
        boolean 
            'True' if the data agree within the tolerance, as
            interpreted by the 'relative' keyword, and if the types,
            lengths, and dtypes are exactly the same.
        """
        # Check that the tolerance is non-negative and raise an
        # exception otherwise.
        if (tol<0):
            raise ValueError("Tolerance cannot be negative")
        # Check that the meta-data agree; the type check is written in
        # this way so that this method may be safely called from
        # subclasses as well.
        if type(other) != type(self):

        # The numpy() method will move any GPU memory onto the CPU.
        # Slow, but the user was warned.

        diff = abs(self.numpy()-other.numpy())
        if relative:
            cmpary = tol*abs(self.numpy())
        else:
            cmpary = tol*ones(len(self),dtype=self.dtype)

        return (diff<=cmpary).all()

    def almost_equal_norm(self,other,tol,relative=True):
        """
        Compare whether two array types are almost equal, normwise.

        abs(norm(self-other)) <= tol*abs(norm(self)).

        abs(norm(self-other)) <= tol

        changed.

        other
            another Python object, that should be tested for
            almost-equality with 'self', based on their norms.
        tol 
            a non-negative number, the tolerance, which is interpreted
        relative

        Returns
        -------
        boolean
        """

        diff = self.numpy()-other.numpy()
        dnorm = norm(diff)
            return (dnorm <= tol*norm(self))
        else:
            return (dnorm <= tol)

    @_convert
    def real(self):
        """ Return real part of Array """
        return Array(self._data.real, copy=True)

    @_convert
    def imag(self):
        """ Return imaginary part of Array """
        return Array(self._data.imag, copy=True)

    @_convert
    def conj(self):
        """ Return complex conjugate of Array. """
        return self._data.conj()
        
    @_convert
    @schemed(BACKEND_PREFIX)
    def squared_norm(self):
        """ Return the elementwise squared norm of the array """
        err_msg = "This function is a stub that should be overridden using "
        err_msg += "the scheme. You shouldn't be seeing this error!"

    @_convert
    def multiply_and_add(self, other, mult_fac):
        """ Return other multiplied by mult_fac and with self added.
        Self is modified in place and returned as output.
        Precisions of inputs must match.
        """

    @_vrcheckother
    @_convert
    def inner(self, other):
        """ Return the inner product of the array with complex conjugation.
        """

    @_convert
    def vdot(self, other):
        """

    @_convert
    def clear(self): 
        """ Clear out the values of the array. """

    @_convert
    def weighted_inner(self, other, weight):
        """

    @_convert
    def sum(self):
        """ Return the sum of the the array. """

    @_convert
    def cumsum(self):
        """ Return the cumulative sum of the the array. """
     
    @_convert
    @_nocomplex
        """ Return the maximum value in the array. """
            
    @_convert
    def max_loc(self):
        """Return the maximum value in the array along with the index location """

    @_convert
    def abs_arg_max(self):
        """ Return location of the maximum argument max """

    @_convert
    def abs_max_loc(self):
        """Return the maximum elementwise norm in the array along with the index location"""

    @_convert
        """ Return the maximum value in the array. """ 
        
    @_returnarray
    @_convert
    def take(self, indices):

    @_convert
    @_vcheckother
    def dot(self, other):
        """ Return the dot product"""
    
    def _getvalue(self, index):
        """Helper function to return a single value from an array. May be very
           slow if the memory is on a gpu.
        """

    @_memoize_single
    def _getslice(self, index):
        return self._return(self._data[index])
    
    @_convert
    def __getitem__(self, index):
        """ Return items from the Array. This not guaranteed to be fast for
            returning single values. 
        """
        if isinstance(index, slice):
            return self._getslice(index)
        else:
            return self._getvalue(index)

    @_convert
    def resize(self, new_size):
        """Resize self to new_size
        """
        if new_size == len(self):
            return
        else:
            self._saved = LimitedSizeDict(size_limit=2**5)
            new_arr = zeros(new_size, dtype=self.dtype)
            if len(self) <= new_size:
                new_arr[0:len(self)] = self
            else:
                new_arr[:] = self[0:new_size]
                
            self._data = new_arr._data

    @_convert
    def roll(self, shift):
        """shift vector
        """
        new_arr = zeros(len(self), dtype=self.dtype)

        if shift < 0:
            shift = shift - len(self) * (shift // len(self))
        
        if shift == 0:
            return
        
        new_arr[0:shift] = self[len(self)-shift: len(self)]
        new_arr[shift:len(self)] = self[0:len(self)-shift]
        
        self._data = new_arr._data

    @_convert
    def astype(self, dtype):
        if _numpy.dtype(self.dtype) == _numpy.dtype(dtype):
        else:
            return self._data.astype(dtype)
    
    def _copy(self, self_ref, other_ref):
        """Helper function to copy between two arrays. The arrays references
           should be bare array types and not `Array` class instances. 
        """
                
    @_convert
    def __setitem__(self, index, other):
        if isinstance(other,Array):
            _convert_to_scheme(other)

            if self.kind is 'real' and other.kind is 'complex':
                raise ValueError('Cannot set real value with complex')

            if isinstance(index,slice):          
                self_ref = self._data[index]
                other_ref = other._data
            else:
                self_ref = self._data[index:index+1]

            self._copy(self_ref, other_ref)

        elif type(other) in _ALLOWED_SCALARS:
            if isinstance(index, slice):
                self[index].fill(other)
            else:
                self[index:index+1].fill(other)
        else:
            raise TypeError('Can only copy data from another Array')

    @property
    def precision(self):
        if self.dtype == float32 or self.dtype == complex64:
            return 'single'
        else:
            return 'double'        
                
    @property
    def kind(self):
        if self.dtype == float32 or self.dtype == float64:
            return 'real'
        elif self.dtype == complex64 or self.dtype == complex128:
            return 'complex'
        else:
            return 'unknown'

    @property
    @_convert
        """Returns the internal python array """

    def data(self,other):
        dtype = None
        if hasattr(other,'dtype'):
            dtype = other.dtype
        temp = Array(other, dtype=dtype)
        self._data = temp._data

    @property
    @_convert
    def ptr(self):
        """ Returns a pointer to the memory of this array """
        
    @property
    def itemsize(self):
        return self.dtype.itemsize
    
    @property
        return len(self.data) * self.itemsize

    @property
    @cpuonly
    @_convert
    def _swighelper(self):
        """ Used internally by SWIG typemaps to ensure @_convert 
            is called and scheme is correct  
        """
        return self;

    @_convert
    def numpy(self):
        """ Returns a Numpy Array that contains this data """     
    
    @_convert
        """ Returns a LAL Object that contains this data """

        lal_data = None
        if self._data.dtype == float32:
            lal_data = _lal.CreateREAL4Vector(len(self))
        elif self._data.dtype == float64:
            lal_data = _lal.CreateREAL8Vector(len(self))
        elif self._data.dtype == complex64:
            lal_data = _lal.CreateCOMPLEX8Vector(len(self))
        elif self._data.dtype == complex128:
            lal_data = _lal.CreateCOMPLEX16Vector(len(self))

        lal_data.data[:] = self.numpy()

        return lal_data

    @property
    def dtype(self):
        return self._data.dtype
    
    def save(self, path, group=None):
        """
        Save array to a Numpy .npy, hdf, or text file. When saving a complex array as
        text, the real and imaginary parts are saved as the first and second
        column respectively. When using hdf format, the data is stored
        as a single vector, along with relevant attributes.

            Destination file path. Must end with either .hdf, .npy or .txt.
            
        group: string 
            Additional name for internal storage use. Ex. hdf storage uses
            this as the key value.

        Raises
        ------
        ValueError
            If path does not end in .npy or .txt.
        """

        ext = _os.path.splitext(path)[1]
        if ext == '.npy':
            _numpy.save(path, self.numpy())
        elif ext == '.txt':
            if self.kind == 'real':
                _numpy.savetxt(path, self.numpy())
            elif self.kind == 'complex':
                output = _numpy.vstack((self.numpy().real,
                                        self.numpy().imag)).T
                _numpy.savetxt(path, output)
        elif ext == '.hdf':
            key = 'data' if group is None else group
            with h5py.File(path, 'a') as f:
                f.create_dataset(key, data=self.numpy(), compression='gzip',
                                 compression_opts=9, shuffle=True)
        else:
            raise ValueError('Path must end with .npy, .txt, or .hdf')
           
    @_convert 
    def trim_zeros(self):
        """Remove the leading and trailing zeros.
        """      
        tmp = self.numpy()
        f = len(self)-len(_numpy.trim_zeros(tmp, trim='f'))
        b = len(self)-len(_numpy.trim_zeros(tmp, trim='b'))
        return self[f:len(self)-b]

    @_convert
    def view(self, dtype):
        """
        Return a 'view' of the array with its bytes now interpreted according
        to 'dtype'. The location in memory is unchanged and changing elements
        in a view of an array will also change the original array.

        dtype : numpy dtype (one of float32, float64, complex64 or complex128)
            The new dtype that should be used to interpret the bytes of self
        """
        return self._data.view(dtype)

        """ Return copy of this array """
        return self._return(self.data.copy())
        
    def __lt__(self, other):
        return self.numpy().__lt__(other)
        
    def __le__(self, other):
        return self.numpy().__le__(other)
        
    def __ne__(self, other):
        return self.numpy().__ne__(other)
        
    def __gt__(self, other):
        return self.numpy().__gt__(other)
        
    def __ge__(self, other):
        return self.numpy().__ge__(other)
            
# Convenience functions for determining dtypes
def real_same_precision_as(data):
    if data.precision is 'single':
        return float32
    elif data.precision is 'double':
        return float64

def complex_same_precision_as(data):
        return complex64
        return complex128

def _return_array(fn, *args, **kwds):
    return Array(fn(*args, **kwds), copy=False)

@_return_array
def zeros(length, dtype=float64):
    """ Return an Array filled with zeros.
    """

def empty(length, dtype=float64):
    """ Return an empty Array (no initialization)
    """

def load_array(path, group=None):
    """
    Load an Array from a .hdf, .txt or .npy file. The
    default data types will be double precision floating point.

    path : string
        source file path. Must end with either .npy or .txt.

    group: string 
        Additional name for internal storage use. Ex. hdf storage uses
        this as the key value.

    Raises
    ------
        If path does not end in .npy or .txt.
    """
    ext = _os.path.splitext(path)[1]
    if ext == '.npy':
        data = _numpy.load(path)    
    elif ext == '.txt':
        data = _numpy.loadtxt(path)
    elif ext == '.hdf':
        key = 'data' if group is None else group
        return Array(h5py.File(path)[key]) 
    else:
        raise ValueError('Path must end with .npy, .hdf, or .txt')
        
    if data.ndim == 1:
        return Array(data)
    elif data.ndim == 2:
        return Array(data[:,0] + 1j*data[:,1])
    else:
        raise ValueError('File has %s dimensions, cannot convert to Array, \
                          must be 1 (real) or 2 (complex)' % data.ndim)
    
# Copyright (C) 2013,2017,2021 Ian Harry, Duncan Brown, Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
"""
This module provides a wrapper to the ConfigParser utilities for pycbc.
This module is described in the page here:
"""
import re
from six.moves import StringIO

class DeepCopyableConfigParser(ConfigParser.SafeConfigParser):
    """
    The standard SafeConfigParser no longer supports deepcopy() as of python
    2.7 (see http://bugs.python.org/issue16058). This subclass restores that
    functionality.
    """

    def __deepcopy__(self, memo):
        # http://stackoverflow.com/questions/23416370
        # /manually-building-a-deep-copy-of-a-configparser-in-python-2-7
        config_string = StringIO()
        self.write(config_string)
        config_string.seek(0)
        new_config = self.__class__()
        new_config.readfp(config_string)
        return new_config

class InterpolatingConfigParser(DeepCopyableConfigParser):
    """
    This is a sub-class of DeepCopyableConfigParser, which lets
    us add a few additional helper features that are useful in workflows.
    """

        self,
        configFiles=None,
        overrideTuples=None,
        parsedFilePath=None,
        deleteTuples=None,
        skip_extended=False,
    ):
        """
         Initialize an InterpolatingConfigParser. This reads the input configuration
         files, overrides values if necessary and performs the interpolation.

         Parameters
         -----------
         configFiles : Path to .ini file, or list of paths
             The file(s) to be read in and parsed.
         overrideTuples : List of (section, option, value) tuples
             Add the (section, option, value) triplets provided
             in this list to the provided .ini file(s). If the section, option
             pair is already present, it will be overwritten.
         parsedFilePath : Path, optional (default=None)
             If given, write the parsed .ini file back to disk at this location.
         deleteTuples : List of (section, option) tuples
             Delete the (section, option) pairs provided
             in this list from provided .ini file(s). If the section only
             is provided, the entire section will be deleted.

        Returns
         --------
         InterpolatingConfigParser
             Initialized InterpolatingConfigParser instance.
        """
        if configFiles is None:
            configFiles = []
        if overrideTuples is None:
            overrideTuples = []
        if deleteTuples is None:
            deleteTuples = []
        DeepCopyableConfigParser.__init__(self)

        # Enable case sensitive options
        self.optionxform = str

        self.read_ini_file(configFiles)

        # Split sections like [inspiral&tmplt] into [inspiral] and [tmplt]
        self.split_multi_sections()

        # Populate shared options from the [sharedoptions] section
        self.populate_shared_sections()

        # Do deletes from command line
        for delete in deleteTuples:
            if len(delete) == 1:
                if self.remove_section(delete[0]) is False:
                    raise ValueError(
                        "Cannot delete section %s, "
                        "no such section in configuration." % delete
                    )

                    "Deleting section %s from configuration", delete[0]
                )
            elif len(delete) == 2:
                if self.remove_option(delete[0], delete[1]) is False:
                        "Cannot delete option %s from section %s,"
                        " no such option in configuration." % delete
                    )

                    "Deleting option %s from section %s in " "configuration",
                    delete[1],
                    delete[0],
                )
            else:
                    "Deletes must be tuples of length 1 or 2. "
                    "Got %s." % str(delete)
                )

        # Do overrides from command line
        for override in overrideTuples:
            if len(override) not in [2, 3]:
                errmsg = "Overrides must be tuples of length 2 or 3."
                errmsg = "Got %s." % (str(override))
                raise ValueError(errmsg)
            section = override[0]
            option = override[1]
            value = ""
            if len(override) == 3:
                value = override[2]
            # Check for section existence, create if needed
            if not self.has_section(section):
                self.add_section(section)
            self.set(section, option, value)
                "Overriding section %s option %s with value %s "
                "in configuration.",
                section,
                option,
                value,
            )

        # Check for any substitutions that can be made
        if not skip_extended:
            self.perform_extended_interpolation()

        # Check for duplicate options in sub-sections
        self.sanity_check_subsections()

        # Dump parsed .ini file if needed
        if parsedFilePath:
            fp = open(parsedFilePath, "w")
            self.write(fp)

    def from_cli(cls, opts):
        """Initialize the config parser using options parsed from the command
        line.

        The parsed options ``opts`` must include options provided by
        :py:func:`add_workflow_command_line_group`.

        opts : argparse.ArgumentParser
            The command line arguments parsed by argparse
        """
        # read configuration file
        logging.info("Reading configuration file")
        if opts.config_overrides is not None:
            overrides = [
                tuple(override.split(":", 2))
                for override in opts.config_overrides
            ]
        else:
            overrides = None
        if opts.config_delete is not None:
            deletes = [
                tuple(delete.split(":")) for delete in opts.config_delete
            ]
        else:
            deletes = None
        return cls(opts.config_files, overrides, deleteTuples=deletes)

    def read_ini_file(self, fpath):
        """
        Read a .ini file and return it as a ConfigParser class.
        This function does none of the parsing/combining of sections. It simply
        reads the file and returns it unedited

        Stub awaiting more functionality - see configparser_test.py

        fpath : Path to .ini file, or list of paths
            The path(s) to a .ini file to be read in

        Returns
        -------
            The ConfigParser class containing the read in .ini file
        """
        # Read the file
        self.read(fpath)

    def get_subsections(self, section_name):
        """Return a list of subsections for the given section name"""
        # Keep only subsection names
        subsections = [
            sec[len(section_name) + 1:]
            for sec in self.sections()
            if sec.startswith(section_name + "-")
        ]

        for sec in subsections:
            sp = sec.split("-")
            # This is unusual, but a format [section-subsection-tag] is okay. Just
            # check that [section-subsection] section exists. If not it is possible
            # the user is trying to use an subsection name with '-' in it
            if (len(sp) > 1) and not self.has_section(
                "%s-%s" % (section_name, sp[0])
            ):
                    "Workflow uses the '-' as a delimiter so "
                    "this is interpreted as section-subsection-tag. "
                    "While checking section %s, no section with "
                    "name %s-%s was found. "
                    "If you did not intend to use tags in an "
                    "'advanced user' manner, or do not understand what "
                    "this means, don't use dashes in section "
                    "names. So [injection-nsbhinj] is good. "
                    "[injection-nsbh-inj] is not." % (sec, sp[0], sp[1])
                )

        if len(subsections) > 0:
            return [sec.split("-")[0] for sec in subsections]
        elif self.has_section(section_name):
            return [""]
        else:
            return []

    def perform_extended_interpolation(self):
        """
        Filter through an ini file and replace all examples of
        ExtendedInterpolation formatting with the exact value. For values like
        ${example} this is replaced with the value that corresponds to the
        option called example ***in the same section***

        For values like ${common|example} this is replaced with the value that
        corresponds to the option example in the section [common]. Note that
        in the python3 config parser this is ${common:example} but python2.7
        interprets the : the same as a = and this breaks things

        Nested interpolation is not supported here.
        """

        # Do not allow any interpolation of the section names
        for section in self.sections():
            for option, value in self.items(section):
                # Check the option name
                new_str = self.interpolate_string(option, section)
                if new_str != option:
                    self.set(section, new_str, value)
                    self.remove_option(section, option)
                # Check the value
                new_str = self.interpolate_string(value, section)
                if new_str != value:
                    self.set(section, option, new_str)

    def interpolate_string(self, test_string, section):
        """
        Take a string and replace all example of ExtendedInterpolation
        formatting within the string with the exact value.

        For values like ${example} this is replaced with the value that
        corresponds to the option called example ***in the same section***

        test_string : String
            The string to parse and interpolate
        section : String
            The current section of the ConfigParser object

        Returns
            Interpolated string
        """

        # First check if any interpolation is needed and abort if not
        re_obj = re.search(r"\$\{.*?\}", test_string)
        while re_obj:
            # Not really sure how this works, but this will obtain the first
            # instance of a string contained within ${....}
            rep_string = (re_obj).group(0)[2:-1]
            # Need to test which of the two formats we have
            split_string = rep_string.split("|")
            if len(split_string) == 1:
                try:
                    test_string = test_string.replace(
                        "${" + rep_string + "}",
                        self.get(section, split_string[0]),
                    )
                except ConfigParser.NoOptionError:
                    print("Substitution failed")
                    raise
            if len(split_string) == 2:
                try:
                        self.get(split_string[0], split_string[1]),
                    )
                    raise
            re_obj = re.search(r"\$\{.*?\}", test_string)

        return test_string

    def split_multi_sections(self):
        """
        Parse through the WorkflowConfigParser instance and splits any sections
        labelled with an "&" sign (for e.g. [inspiral&tmpltbank]) into
        [inspiral] and [tmpltbank] sections. If these individual sections
        already exist they  will be appended to. If an option exists in both the
        [inspiral] and [inspiral&tmpltbank] sections an error will be thrown
        """
        # Begin by looping over all sections
            # Only continue if section needs splitting
            if "&" not in section:
                continue
            # Get list of section names to add these options to
            split_sections = section.split("&")
            for new_sec in split_sections:
                # Add sections if they don't already exist
                if not self.has_section(new_sec):
                    self.add_section(new_sec)
                self.add_options_to_section(new_sec, self.items(section))
            self.remove_section(section)

    def populate_shared_sections(self):
        """Parse the [sharedoptions] section of the ini file.

        That section should contain entries according to:

          * massparams = inspiral, tmpltbank
          * dataparams = tmpltbank

        This will result in all options in [sharedoptions-massparams] being
        copied into the [inspiral] and [tmpltbank] sections and the options
        in [sharedoptions-dataparams] being copited into [tmpltbank].
        In the case of duplicates an error will be raised.
        """
        if not self.has_section("sharedoptions"):
            # No sharedoptions, exit
            return
        for key, value in self.items("sharedoptions"):
            assert self.has_section("sharedoptions-%s" % (key))
            # Comma separated
            values = value.split(",")
            common_options = self.items("sharedoptions-%s" % (key))
            for section in values:
                if not self.has_section(section):
                    self.add_section(section)
                for arg, val in common_options:
                    if arg in self.options(section):
                        raise ValueError(
                            "Option exists in both original "
                            + "ConfigParser section [%s] and " % (section,)
                            + "sharedoptions section: %s %s"
                            % (arg, "sharedoptions-%s" % (key))
                        )
                    self.set(section, arg, val)
            self.remove_section("sharedoptions-%s" % (key))
        self.remove_section("sharedoptions")

    def add_options_to_section(self, section, items, overwrite_options=False):
        """
        Add a set of options and values to a section of a ConfigParser object.
        Will throw an error if any of the options being added already exist,
        this behaviour can be overridden if desired

            The name of the section to add options+values to
        items : list of tuples
            Each tuple contains (at [0]) the option and (at [1]) the value to
            add to the section of the ini file
        overwrite_options : Boolean, optional
            By default this function will throw a ValueError if an option exists
            in both the original section in the ConfigParser *and* in the
            provided items.
            This will override so that the options+values given in items
            will replace the original values if the value is set to True.
            Default = True
        """
        # Sanity checking
        if not self.has_section(section):
                "Section %s not present in ConfigParser." % (section,)
            )

        # Check for duplicate options first
        for option, value in items:
            if not overwrite_options:
                if option in self.options(section):
                        "Option exists in both original "
                        + "ConfigParser section [%s] and " % (section,)
                        + "input list: %s" % (option,)
                    )

    def sanity_check_subsections(self):
        """
        This function goes through the ConfigParset and checks that any options
        given in the [SECTION_NAME] section are not also given in any
        [SECTION_NAME-SUBSECTION] sections.

        """
        # Loop over the sections in the ini file
            # [pegasus_profile] specially is allowed to be overriden by
            # sub-sections
            if section == "pegasus_profile":
                continue

            # Loop over the sections again
            for section2 in self.sections():
                # Check if any are subsections of section
                if section2.startswith(section + "-"):
                    # Check for duplicate options whenever this exists
                    self.check_duplicate_options(
                        section, section2, raise_error=True
                    )

    def check_duplicate_options(self, section1, section2, raise_error=False):
        """
        Check for duplicate options in two sections, section1 and section2.
        Will return a list of the duplicate options.

        section1 : string
            The name of the first section to compare
        section2 : string
            The name of the second section to compare
        raise_error : Boolean, optional (default=False)
            If True, raise an error if duplicates are present.

        Returns
        duplicates : List
            List of duplicate options
        """
        if not self.has_section(section1):
                "Section %s not present in ConfigParser." % (section1,)
            )
        if not self.has_section(section2):
                "Section %s not present in ConfigParser." % (section2,)
            )

        items1 = self.options(section1)
        items2 = self.options(section2)

        # The list comprehension here creates a list of all duplicate items
        duplicates = [x for x in items1 if x in items2]

        if duplicates and raise_error:
                "The following options appear in both section "
                + "%s and %s: %s" % (section1, section2, " ".join(duplicates))
            )

        return duplicates

    def get_opt_tag(self, section, option, tag):
        """
        Convenience function accessing get_opt_tags() for a single tag: see
        documentation for that function.
        NB calling get_opt_tags() directly is preferred for simplicity.

        self : ConfigParser object
            The ConfigParser object (automatically passed when this is appended
            to the ConfigParser class)
            The section of the ConfigParser object to read
        option : string
            The ConfigParser option to look for
        tag : string
            The name of the subsection to look in, if not found in [section]

        Returns
        --------
        string
            The value of the options being searched for
        """
        return self.get_opt_tags(section, option, [tag])

    def get_opt_tags(self, section, option, tags):
        """
        Supplement to ConfigParser.ConfigParser.get(). This will search for an
        option in [section] and if it doesn't find it will also try in
        [section-tag] for every value of tag in tags.
        Will raise a ConfigParser.Error if it cannot find a value.

        tags : list of strings
            The name of subsections to look in, if not found in [section]

        Returns
        --------
        string
        """
        # Need lower case tag name; also exclude cases with tag=None
        if tags:
            tags = [tag.lower() for tag in tags if tag is not None]

        try:
            return self.get(section, option)
            err_string = "No option '%s' in section [%s] " % (option, section)
            if not tags:
                raise ConfigParser.Error(err_string + ".")
            return_vals = []
            sub_section_list = []
            for sec_len in range(1, len(tags) + 1):
                for tag_permutation in itertools.permutations(tags, sec_len):
                    joined_name = "-".join(tag_permutation)
                    sub_section_list.append(joined_name)
            section_list = ["%s-%s" % (section, sb) for sb in sub_section_list]
            err_section_list = []
            for sub in sub_section_list:
                if self.has_section("%s-%s" % (section, sub)):
                    if self.has_option("%s-%s" % (section, sub), option):
                        err_section_list.append("%s-%s" % (section, sub))
                        return_vals.append(
                            self.get("%s-%s" % (section, sub), option)
                        )

            # We also want to recursively go into sections

            if not return_vals:
                err_string += "or in sections [%s]." % (
                    "] [".join(section_list)
                )
                raise ConfigParser.Error(err_string)
            if len(return_vals) > 1:
                err_string += (
                    "and multiple entries found in sections [%s]."
                    % ("] [".join(err_section_list))
                )
            return return_vals[0]

    def has_option_tag(self, section, option, tag):
        """
        Convenience function accessing has_option_tags() for a single tag: see
        NB calling has_option_tags() directly is preferred for simplicity.

        Returns
        --------
        Boolean
            Is the option in the section or [section-tag]
        """
        return self.has_option_tags(section, option, [tag])

    def has_option_tags(self, section, option, tags):
        """
        Supplement to ConfigParser.ConfigParser.has_option().
        This will search for an option in [section] and if it doesn't find it
        will also try in [section-tag] for each value in tags.
        Returns True if the option is found and false if not.

            The names of the subsection to look in, if not found in [section]

        Returns
        --------
        Boolean
            Is the option in the section or [section-tag] (for tag in tags)
        """
        try:
            self.get_opt_tags(section, option, tags)

# Copyright (C) 2012  Tito Dal Canton, Josh Willis
#
#
#

"""
Provides a class representing a frequency series.
"""
import os as _os, h5py
from pycbc.types.array import Array, _convert, zeros, _noreal

class FrequencySeries(Array):
    """Models a frequency series consisting of uniformly sampled scalar values.

    initial_array : array-like
        Array containing sampled data.
        Frequency between consecutive samples in Hertz.
    epoch : {None, lal.LIGOTimeGPS}, optional
        Start time of the associated time domain data in seconds.
    dtype : {None, data-type}, optional
        Sample data type.
    copy : boolean, optional
        If True, samples are copied to a new array.

        Frequency spacing
    epoch : lal.LIGOTimeGPS
        Time at 0 index.
    sample_frequencies : Array
        Frequencies that each index corresponds to.
    """

    def __init__(self, initial_array, delta_f=None, epoch="", dtype=None, copy=True):
        if len(initial_array) < 1:
            raise ValueError('initial_array must contain at least one sample.')
            try:
                delta_f = initial_array.delta_f
                raise TypeError('must provide either an initial_array with a delta_f attribute, or a value for delta_f')
        if not delta_f > 0:
            raise ValueError('delta_f must be a positive number')
        # We gave a nonsensical default value to epoch so we can test if it's been set.
        # If the user passes in an initial_array that has an 'epoch' attribute and doesn't
        # pass in a value of epoch, then our new object's epoch comes from initial_array.
        # But if the user passed in a value---even 'None'---that will take precedence over
        # anything set in initial_array.  Finally, if the user passes in something without
        # an epoch attribute *and* doesn't pass in a value of epoch, it becomes 'None'
        if not isinstance(epoch,_lal.LIGOTimeGPS):
            if epoch == "":
                if isinstance(initial_array,FrequencySeries):
                    epoch = initial_array._epoch
                else:
                    epoch = _lal.LIGOTimeGPS(0)
            elif epoch is not None:
                try:
                    if isinstance(epoch, _numpy.generic):
                        # In python3 lal LIGOTimeGPS will not work on numpy
                        # types as input. A quick google on how to generically
                        # convert numpy floats/ints to the python equivalent
                        # https://stackoverflow.com/questions/9452775/
                        epoch = _lal.LIGOTimeGPS(epoch.item())
                    else:
                        epoch = _lal.LIGOTimeGPS(epoch)
                except:
                    raise TypeError('epoch must be either None or a lal.LIGOTimeGPS')
        Array.__init__(self, initial_array, dtype=dtype, copy=copy)
        self._delta_f = delta_f
        self._epoch = epoch

        return FrequencySeries(ary, self._delta_f, epoch=self._epoch, copy=False)

        if isinstance(other, FrequencySeries):
            try:
                _numpy.testing.assert_almost_equal(other._delta_f,
                                                   self._delta_f)
            except:
                raise ValueError('different delta_f')
            # consistency of _epoch is not required because we may want
            # to combine frequency series estimated at different times
            # (e.g. PSD estimation)

    def get_delta_f(self):
        """Return frequency between consecutive samples in Hertz.
        """
        return self._delta_f
    delta_f = property(get_delta_f,
                       doc="Frequency between consecutive samples in Hertz.")

    def get_epoch(self):
        """Return frequency series epoch as a LIGOTimeGPS.
        """
        return self._epoch
    epoch = property(get_epoch,
                     doc="Frequency series epoch as a LIGOTimeGPS.")

    def get_sample_frequencies(self):
        """Return an Array containing the sample frequencies.
        """
        return Array(range(len(self))) * self._delta_f
    sample_frequencies = property(get_sample_frequencies,
                                  doc="Array of the sample frequencies.")

        if index.step is not None:
            new_delta_f = self._delta_f * index.step
        else:
            new_delta_f = self._delta_f
        return FrequencySeries(Array._getslice(self, index),
                               delta_f=new_delta_f,
                               epoch=self._epoch,
                               copy=False)

    def at_frequency(self, freq):
        """ Return the value at the specified frequency
        """
        return self[int(freq / self.delta_f)]

    @property
        """Return the start time of this vector
        """
        return self.epoch

    @start_time.setter
    def start_time(self, time):
        """ Set the start time
        """
        self._epoch = _lal.LIGOTimeGPS(time)

    @property
        """Return the end time of this vector
        """
        return self.start_time + self.duration

    @property
    def duration(self):
        """Return the time duration of this vector
        """
        return 1.0 / self.delta_f

    @property
    def delta_t(self):
        """Return the time between samples if this were a time series.
        This assume the time series is even in length!
        """
        return 1.0 / self.sample_rate

    @property
    def sample_rate(self):
        """Return the sample rate this would have in the time domain. This
        assumes even length time series!
        """
        return (len(self) - 1) * self.delta_f * 2.0

        """
        frequency series are identical, and all of the numeric meta-data

        and 'other' are identical, as well as their lengths, dtypes,
        epochs, delta_fs and the data in the arrays, element by element.
        It will always do the comparison on the CPU, but will *not* move
        either object to the CPU if it is not already there, nor change
        the scheme of either object. It is possible to compare a CPU
        object to a GPU object, and the comparison should be true if the
        data and meta-data of the two objects are the same.

        Returns
        -------
        boolean: 'True' if the types, dtypes, lengths, epochs, delta_fs
            and data of the two objects are each identical.
        """
        if super(FrequencySeries,self).__eq__(other):
            return (self._epoch == other._epoch and self._delta_f == other._delta_f)
        else:

    def almost_equal_elem(self,other,tol,relative=True,dtol=0.0):
        """
        Compare whether two frequency series are almost equal, element

        for all elements of the series.

        The method also checks that self.delta_f is within 'dtol' of
        other.delta_f; if 'dtol' has its default value of 0 then exact
        equality between the two is required.

        Other meta-data (type, dtype, length, and epoch) must be exactly
        equal.  If either object's memory lives on the GPU it will be
        copied to the CPU for the comparison, which may be slow. But the
        original object itself will not have its memory relocated nor
        scheme changed.

        other: another Python object, that should be tested for
        tol: a non-negative number, the tolerance, which is interpreted
        relative: A boolean, indicating whether 'tol' should be interpreted
        dtol: a non-negative number, the tolerance for delta_f. Like 'tol',
            it is interpreted as relative or absolute based on the value of
            'relative'.  This parameter defaults to zero, enforcing exact
            equality between the delta_f values of the two FrequencySeries.

        Returns
        -------
        boolean: 'True' if the data and delta_fs agree within the tolerance,
            as interpreted by the 'relative' keyword, and if the types,
            lengths, dtypes, and epochs are exactly the same.
        """
        # Check that the delta_f tolerance is non-negative; raise an exception
        # if needed.
        if (dtol < 0.0):
            raise ValueError("Tolerance in delta_f cannot be negative")
        if super(FrequencySeries,self).almost_equal_elem(other,tol=tol,relative=relative):
            if relative:
                return (self._epoch == other._epoch and
                        abs(self._delta_f-other._delta_f) <= dtol*self._delta_f)
            else:
                        abs(self._delta_f-other._delta_f) <= dtol)
        else:

    def almost_equal_norm(self,other,tol,relative=True,dtol=0.0):
        """
        Compare whether two frequency series are almost equal, normwise.

        Returns
        -------
        """
        if super(FrequencySeries,self).almost_equal_norm(other,tol=tol,relative=relative):
            else:
        else:

    @_convert
        """Produces a LAL frequency series object equivalent to self.

        Returns
        -------
        lal_data : {lal.*FrequencySeries}
            LAL frequency series object containing the same data as self.
            The actual type depends on the sample's dtype. If the epoch of
            self was 'None', the epoch of the returned LAL object will be
            LIGOTimeGPS(0,0); otherwise, the same as that of self.

        Raises
        ------
        TypeError
            If frequency series is stored in GPU memory.
        """

        if self._epoch is None:
            ep = _lal.LIGOTimeGPS(0,0)
        else:
            ep = self._epoch

        if self._data.dtype == _numpy.float32:
            lal_data = _lal.CreateREAL4FrequencySeries("",ep,0,self.delta_f,_lal.SecondUnit,len(self))
        elif self._data.dtype == _numpy.float64:
            lal_data = _lal.CreateREAL8FrequencySeries("",ep,0,self.delta_f,_lal.SecondUnit,len(self))
        elif self._data.dtype == _numpy.complex64:
            lal_data = _lal.CreateCOMPLEX8FrequencySeries("",ep,0,self.delta_f,_lal.SecondUnit,len(self))
        elif self._data.dtype == _numpy.complex128:
            lal_data = _lal.CreateCOMPLEX16FrequencySeries("",ep,0,self.delta_f,_lal.SecondUnit,len(self))

        lal_data.data.data[:] = self.numpy()

    def save(self, path, group=None, ifo='P1'):
        """
        Save frequency series to a Numpy .npy, hdf, or text file. The first column
        contains the sample frequencies, the second contains the values.
        In the case of a complex frequency series saved as text, the imaginary
        part is written as a third column.  When using hdf format, the data is stored

        group: string

        Raises
        ------
        """

            output = _numpy.vstack((self.sample_frequencies.numpy(),
                                    self.numpy())).T
            _numpy.save(path, output)
                output = _numpy.vstack((self.sample_frequencies.numpy(),
                                        self.numpy())).T
                                        self.numpy().real,
            _numpy.savetxt(path, output)
        elif ext == '.xml' or path.endswith('.xml.gz'):
            from pycbc.io.live import make_psd_xmldoc
            from glue.ligolw import utils

            if self.kind != 'real':
                raise ValueError('XML only supports real frequency series')
            output = self.lal()
            # When writing in this format we must *not* have the 0 values at
            # frequencies less than flow. To resolve this we set the first
            # non-zero value < flow.
            data_lal = output.data.data
            first_idx = _numpy.argmax(data_lal>0)
            if not first_idx == 0:
                data_lal[:first_idx] = data_lal[first_idx]
            psddict = {ifo: output}
            utils.write_filename(make_psd_xmldoc(psddict), path,
                                 gz=path.endswith(".gz"))
                ds = f.create_dataset(key, data=self.numpy(),
                                      compression_opts=9, shuffle=True)
                ds.attrs['epoch'] = float(self.epoch)
                ds.attrs['delta_f'] = float(self.delta_f)
        else:
            raise ValueError('Path must end with .npy, .txt, .xml, .xml.gz '
                             'or .hdf')

    def to_frequencyseries(self):
        """ Return frequency series """

    @_noreal
    def to_timeseries(self, delta_t=None):
        """ Return the Fourier transform of this time series.

        Note that this assumes even length time series!

        delta_t : {None, float}, optional
            The time resolution of the returned series. By default the
        resolution is determined by length and delta_f of this frequency
        series.

        Returns
        -------
        TimeSeries:
            The inverse fourier transform of this frequency series.
        """
        from pycbc.fft import ifft
        from pycbc.types import TimeSeries, real_same_precision_as
        nat_delta_t =  1.0 / ((len(self)-1)*2) / self.delta_f
        if not delta_t:
            delta_t = nat_delta_t

        # add 0.5 to round integer
        tlen  = int(1.0 / self.delta_f / delta_t + 0.5)
        flen = int(tlen / 2 + 1)

        if flen < len(self):
            raise ValueError("The value of delta_t (%s) would be "
                             "undersampled. Maximum delta_t "
                             "is %s." % (delta_t, nat_delta_t))
            tmp = self
        else:
            tmp = FrequencySeries(zeros(flen, dtype=self.dtype),
                             delta_f=self.delta_f, epoch=self.epoch)
            tmp[:len(self)] = self[:]

        f = TimeSeries(zeros(tlen,
                           dtype=real_same_precision_as(self)),
                           delta_t=delta_t)
        ifft(tmp, f)
        return f

    @_noreal
    def cyclic_time_shift(self, dt):
        """Shift the data and timestamps by a given number of seconds

        Shift the data and timestamps in the time domain a given number of
        seconds. To just change the time stamps, do ts.start_time += dt.
        The time shift may be smaller than the intrinsic sample rate of the data.
        Note that data will be cycliclly rotated, so if you shift by 2
        seconds, the final 2 seconds of your data will now be at the
        beginning of the data set.

        dt : float
            Amount of time to shift the vector.

        Returns
        -------
        data : pycbc.types.FrequencySeries
            The time shifted frequency series.
        """
        from pycbc.waveform import apply_fseries_time_shift
        data = apply_fseries_time_shift(self, dt)
        data.start_time = self.start_time - dt

    def match(self, other, psd=None,
              low_frequency_cutoff=None, high_frequency_cutoff=None):
        """ Return the match between the two TimeSeries or FrequencySeries.

        Return the match between two waveforms. This is equivalent to the overlap
        maximized over time and phase. By default, the other vector will be
        resized to match self. Beware, this may remove high frequency content or the
        end of the vector.

        other : TimeSeries or FrequencySeries
            The input vector containing a waveform.
        psd : Frequency Series
            A power spectral density to weight the overlap.
            The frequency to begin the match.
            The frequency to stop the match.
        index: int
            The number of samples to shift to get the match.

        Returns
        -------
        match: float
        index: int
        """
        from pycbc.types import TimeSeries
        from pycbc.filter import match

        if isinstance(other, TimeSeries):
            if other.duration != self.duration:
                other = other.copy()
                other.resize(int(other.sample_rate * self.duration))

            other = other.to_frequencyseries()

        if len(other) != len(self):
            other = other.copy()
            other.resize(len(self))

        if psd is not None and len(psd) > len(self):
            psd = psd.copy()
            psd.resize(len(self))

        return match(self, other, psd=psd,
                     low_frequency_cutoff=low_frequency_cutoff,

    def plot(self, **kwds):
        """ Basic plot of this frequency series
        """
        from matplotlib import pyplot

        if self.kind == 'real':
            plot = pyplot.plot(self.sample_frequencies, self, **kwds)
            return plot
        elif self.kind == 'complex':
            plot1 = pyplot.plot(self.sample_frequencies, self.real(), **kwds)
            plot2 = pyplot.plot(self.sample_frequencies, self.imag(), **kwds)
            return plot1, plot2

def load_frequencyseries(path, group=None):
    """
    Load a FrequencySeries from a .hdf, .txt or .npy file. The

    path: string

    group: string

    Raises
    ------
    """
        data = _numpy.load(path)
        f = h5py.File(path, 'r')
        data = f[key][:]
        series = FrequencySeries(data, delta_f=f[key].attrs['delta_f'],
                                       epoch=f[key].attrs['epoch'])
        f.close()
    else:

    if data.ndim == 2:
        delta_f = (data[-1][0] - data[0][0]) / (len(data)-1)
        epoch = _lal.LIGOTimeGPS(data[0][0])
        return FrequencySeries(data[:,1], delta_f=delta_f, epoch=epoch)
    elif data.ndim == 3:
        return FrequencySeries(data[:,1] + 1j*data[:,2], delta_f=delta_f,
                               epoch=epoch)
    else:
                          must be 2 (real) or 3 (complex)' % data.ndim)

#
#
#
"""
This modules contains extensions for use with argparse
"""
from collections import defaultdict

class DictWithDefaultReturn(defaultdict):
    default_set = False
    ifo_set = False
    def __bool__(self):
        if self.items() and not all(entry is None for entry in self.values()):
            # True if any values are explictly set.
        elif self['RANDOM_STRING_314324'] is not None:
            # Or true if the default value was set
            # NOTE: This stores the string RANDOM_STRING_314324 in the dict
            # so subsequent calls will be caught in the first test here.
        else:
            # Else false
    # Python 2 and 3 have different conventions for boolean method
    __nonzero__ = __bool__

class MultiDetOptionAction(argparse.Action):
    # Initialise the same as the standard 'append' action
    def __init__(self,
                 option_strings,
                 dest,
                 nargs='+',
                 const=None,
                 default=None,
                 type=None,
                 choices=None,
                 required=False,
                 help=None,
                 metavar=None):
        if type is not None:
            self.internal_type = type
        else:
            self.internal_type = str
        new_default = DictWithDefaultReturn(lambda: default)
        #new_default.default_value=default
            raise ValueError('nargs for append actions must be > 0; if arg '
                             'strings are not supplying the value to append, '
                             'the append const action may be more appropriate')
        if const is not None and nargs != argparse.OPTIONAL:
            raise ValueError('nargs must be %r to supply const'
                             % argparse.OPTIONAL)
        super(MultiDetOptionAction, self).__init__(
            option_strings=option_strings,
            dest=dest,
            nargs=nargs,
            const=const,
            default=new_default,
            type=str,
            choices=choices,
            required=required,
            help=help,
            metavar=metavar)

        # Again this is modified from the standard argparse 'append' action
        err_msg = "Issue with option: %s \n" %(self.dest,)
        err_msg += "Received value: %s \n" %(' '.join(values),)
        if getattr(namespace, self.dest, None) is None:
            setattr(namespace, self.dest, DictWithDefaultReturn())
        items = getattr(namespace, self.dest)
        items = copy.copy(items)
        for value in values:
            value = value.split(':')
            if len(value) == 2:
                # "Normal" case, all ifos supplied independently as "H1:VALUE"
                if items.default_set:
                    err_msg += "If you are supplying a value for all ifos, you "
                    err_msg += "cannot also supply values for specific ifos."
                items[value[0]] = self.internal_type(value[1])
                items.ifo_set = True
            elif len(value) == 1:
                # OR supply only one value and use this for all ifos
                    err_msg += "must only supply one value."
                # Can't use a global and ifo specific options
                if items.ifo_set:
                #items.default_value = self.internal_type(value[0])
                new_default = self.internal_type(value[0])
                items.default_factory = lambda: new_default
                items.default_set = True
            else:
                err_msg += "The character ':' is used to deliminate the "
                err_msg += "ifo and the value. Please do not use it more than "
                err_msg += "once."
        setattr(namespace, self.dest, items)

class MultiDetOptionActionSpecial(MultiDetOptionAction):
    """
    This class in an extension of the MultiDetOptionAction class to handle
    cases where the : is already a special character. For example the channel
    name is something like H1:CHANNEL_NAME. Here the channel name *must*
    be provided uniquely for each ifo. The dictionary key is set to H1 and the
    value to H1:CHANNEL_NAME for this example.
    """
            setattr(namespace, self.dest, {})
            value_split = value.split(':')
            if len(value_split) == 2:
                if value_split[0] in items:
                    err_msg += "Multiple values supplied for ifo %s.\n" \
                               %(value_split[0],)
                    err_msg += "Already have %s." %(items[value_split[0]])
                else:
                    items[value_split[0]] = value
            elif len(value_split) == 3:
                # This is an unadvertised feature. It is used for cases where I
                # want to pretend H1 data is actually L1 (or similar). So if I
                # supply --channel-name H1:L1:LDAS-STRAIN I can use L1 data and
                # pretend it is H1 internally.
                else:
                    items[value_split[0]] = ':'.join(value_split[1:3])
            else:
                err_msg += "ifo and the value. It must appear exactly "

class MultiDetMultiColonOptionAction(MultiDetOptionAction):
    """A special case of `MultiDetOptionAction` which allows one to use
    arguments containing colons, such as `V1:FOOBAR:1`. The first colon is
    assumed to be the separator between the detector and the argument.
    All subsequent colons are kept as part of the argument. Unlike
    `MultiDetOptionAction`, all arguments must be prefixed by the
    corresponding detector.
    """
        err_msg = ('Issue with option: {}\n'
                   'Received value: {}\n').format(self.dest, ' '.join(values))
        items = copy.copy(getattr(namespace, self.dest))
            if ':' not in value:
                err_msg += ("Each argument must contain at least one ':' "
                            "character")
            detector, argument = value.split(':', 1)
            if detector in items:
                err_msg += ('Multiple values supplied for detector {},\n'
                            'already have {}.')
                err_msg = err_msg.format(detector, items[detector])
            items[detector] = self.internal_type(argument)

class MultiDetOptionAppendAction(MultiDetOptionAction):
                # "Normal" case, all ifos supplied independetly as "H1:VALUE"
                if value[0] in items:
                    items[value[0]].append(self.internal_type(value[1]))
                else:
                    items[value[0]] = [self.internal_type(value[1])]
            else:
                err_msg = "Issue with option: %s \n" %(self.dest,)
                err_msg += "Received value: %s \n" %(' '.join(values),)
                err_msg += "The character ':' is used to distinguish the "
                err_msg += "ifo and the value. It must be given exactly once "
                err_msg += "for all entries"

def required_opts(opt, parser, opt_list, required_by=None):
    """Check that all the opts are defined

        Result of option parsing
    opt_list : list of strings
    required_by : string, optional
        the option that requires these options (if applicable)
    """
    for name in opt_list:
        attr = name[2:].replace('-', '_')
        if not hasattr(opt, attr) or (getattr(opt, attr) is None):
            err_str = "%s is missing " % name
            if required_by is not None:
                err_str += ", required by %s" % required_by
            parser.error(err_str)

def required_opts_multi_ifo(opt, parser, ifo, opt_list, required_by=None):

    """
        try:
            if getattr(opt, attr)[ifo] is None:
                raise KeyError

def ensure_one_opt(opt, parser, opt_list):
    """  Check that one and only one in the opt_list is defined in opt

    """

    the_one = None
        if hasattr(opt, attr) and (getattr(opt, attr) is not None):
            if the_one is None:
                the_one = name
            else:
                parser.error("%s and %s are mutually exculsive" \
                              % (the_one, name))

    if the_one is None:
        parser.error("you must supply one of the following %s" \
                      % (', '.join(opt_list)))

def ensure_one_opt_multi_ifo(opt, parser, ifo, opt_list):

    """

        try:
            pass
        else:
            else:

def copy_opts_for_single_ifo(opt, ifo):
    """
    Takes the namespace object (opt) from the multi-detector interface and
    returns a namespace object for a single ifo that can be used with
    functions expecting output from the single-detector interface.
    """
    opt = copy.deepcopy(opt)
    for arg, val in vars(opt).items():
        if isinstance(val, DictWithDefaultReturn):
            setattr(opt, arg, getattr(opt, arg)[ifo])
    return opt

def convert_to_process_params_dict(opt):
    """
    returns a dictionary of command line options that will be handled correctly
    by the register_to_process_params ligolw function.
    """
            new_val = []
            for key in val.keys():
                if isinstance(val[key], list):
                    for item in val[key]:
                        if item is not None:
                            new_val.append(':'.join([key, str(item)]))
                else:
                    if val[key] is not None:
                        new_val.append(':'.join([key, str(val[key])]))
            setattr(opt, arg, new_val)
    return vars(opt)

def positive_float(s):
    """
    Ensure argument is a positive real number and return it as float.

    To be used as type in argparse arguments.
    """
    err_msg = "must be a positive number, not %r" % s
    try:
        value = float(s)
        raise argparse.ArgumentTypeError(err_msg)
    if value <= 0:

def nonnegative_float(s):
    """
    Ensure argument is a positive real number or zero and return it as float.

    """
    err_msg = "must be either positive or zero, not %r" % s
    try:
    if value < 0:

# Copyright (C) 2014  Tito Dal Canton, Josh Willis, Alex Nitz
#
#
#

"""
Provides a class representing a time series.
"""
from pycbc.types.array import Array, _convert, complex_same_precision_as, zeros
from pycbc.types.array import _nocomplex
from pycbc.types.frequencyseries import FrequencySeries
from scipy.io.wavfile import write as write_wav

class TimeSeries(Array):
    """Models a time series consisting of uniformly sampled scalar values.

        Time between consecutive samples in seconds.
        Time of the first sample in seconds.

    delta_t
    duration
    start_time
    end_time
    sample_times
    sample_rate
    """

    def __init__(self, initial_array, delta_t=None,
                 epoch=None, dtype=None, copy=True):
        if delta_t is None:
            try:
                delta_t = initial_array.delta_t
                raise TypeError('must provide either an initial_array with a delta_t attribute, or a value for delta_t')
        if not delta_t > 0:
            raise ValueError('delta_t must be a positive number')

        # Get epoch from initial_array if epoch not given (or is None)
        # If initialy array has no epoch, set epoch to 0.
        # If epoch is provided, use that.
        if not isinstance(epoch, _lal.LIGOTimeGPS):
            if epoch is None:
                if isinstance(initial_array, TimeSeries):
                else:
                try:
                    epoch = _lal.LIGOTimeGPS(epoch)
                except:
        self._delta_t = delta_t

    def epoch_close(self, other):
        """ Check if the epoch is close enough to allow operations """
        dt = abs(float(self.start_time - other.start_time))
        return dt <= 1e-7

    def sample_rate_close(self, other):
        """ Check if the sample rate is close enough to allow operations """

        # compare our delta_t either to a another time series' or
        # to a given sample rate (float)
            odelta_t = other.delta_t
        else:
            odelta_t = 1.0/other

        if (odelta_t - self.delta_t) / self.delta_t > 1e-4:

        if abs(1 - odelta_t / self.delta_t) * len(self) > 0.5:

        return TimeSeries(ary, self._delta_t, epoch=self._epoch, copy=False)

            if not self.sample_rate_close(other):
                raise ValueError('different delta_t, {} vs {}'.format(
                    self.delta_t, other.delta_t))
            if not self.epoch_close(other):
                raise ValueError('different epoch, {} vs {}'.format(
                    self.start_time, other.start_time))

        # Set the new epoch---note that index.start may also be None
        if index.start is None:
            new_epoch = self._epoch
        else:
            if index.start < 0:
                raise ValueError(('Negative start index ({})'
                                  ' not supported').format(index.start))
            new_epoch = self._epoch + index.start * self._delta_t

            new_delta_t = self._delta_t * index.step
        else:
            new_delta_t = self._delta_t

        return TimeSeries(Array._getslice(self, index), new_delta_t,
                          new_epoch, copy=False)

    def prepend_zeros(self, num):
        """Prepend num zeros onto the beginning of this TimeSeries. Update also
        epoch to include this prepending.
        """
        self.resize(len(self) + num)
        self.roll(num)
        self._epoch = self._epoch - num * self._delta_t

    def append_zeros(self, num):
        """Append num zeros onto the end of this TimeSeries.
        """

    def get_delta_t(self):
        """Return time between consecutive samples in seconds.
        """
        return self._delta_t
    delta_t = property(get_delta_t,
                       doc="Time between consecutive samples in seconds.")

    def get_duration(self):
        """Return duration of time series in seconds.
        """
        return len(self) * self._delta_t
    duration = property(get_duration,
                        doc="Duration of time series in seconds.")

    def get_sample_rate(self):
        """Return the sample rate of the time series.
        """
        return 1.0/self.delta_t
    sample_rate = property(get_sample_rate,
                           doc="The sample rate of the time series.")

    def time_slice(self, start, end, mode='floor'):
        """Return the slice of the time series that contains the time range
        in GPS seconds.
        """
        if start < self.start_time:
            raise ValueError('Time series does not contain a time as early as %s' % start)

        if end > self.end_time:
            raise ValueError('Time series does not contain a time as late as %s' % end)

        start_idx = float(start - self.start_time) * self.sample_rate
        end_idx = float(end - self.start_time) * self.sample_rate

        if _numpy.isclose(start_idx, round(start_idx)):
            start_idx = round(start_idx)

        if _numpy.isclose(end_idx, round(end_idx)):
            end_idx = round(end_idx)

        if mode == 'floor':
            start_idx = int(start_idx)
            end_idx = int(end_idx)
        elif mode == 'nearest':
            start_idx = int(round(start_idx))
            end_idx = int(round(end_idx))
        else:
            raise ValueError("Invalid mode: {}".format(mode))

        return self[start_idx:end_idx]

    @property
    def delta_f(self):
        """Return the delta_f this ts would have in the frequency domain
        """
        return 1.0 / self.duration

    @property
        """Return time series start time as a LIGOTimeGPS.
        """

        """

        """Return time series end time as a LIGOTimeGPS.
        """
        return self._epoch + self.get_duration()
    end_time = property(get_end_time,
                        doc="Time series end time as a LIGOTimeGPS.")

    def get_sample_times(self):
        """Return an Array containing the sample times.
        """
            return Array(range(len(self))) * self._delta_t
        else:
            return Array(range(len(self))) * self._delta_t + float(self._epoch)
    sample_times = property(get_sample_times,
                            doc="Array containing the sample times.")

    def at_time(self, time, nearest_sample=False):
        """ Return the value at the specified gps time
        """
        if nearest_sample:
            time += self.delta_t / 2.0
        return self[int((time-self.start_time)*self.sample_rate)]

        """
        time series are identical, and all of the numeric meta-data

        epochs, delta_ts and the data in the arrays, element by element.

        Returns
        -------
        boolean: 'True' if the types, dtypes, lengths, epochs, delta_ts
        """
        if super(TimeSeries,self).__eq__(other):
            return (self._epoch == other._epoch and self._delta_t == other._delta_t)
        else:

        """
        Compare whether two time series are almost equal, element

        The method also checks that self.delta_t is within 'dtol' of
        other.delta_t; if 'dtol' has its default value of 0 then exact

        dtol: a non-negative number, the tolerance for delta_t. Like 'tol',
            equality between the delta_t values of the two TimeSeries.

        Returns
        -------
        boolean: 'True' if the data and delta_ts agree within the tolerance,
        """
        # Check that the delta_t tolerance is non-negative; raise an exception
            raise ValueError("Tolerance in delta_t cannot be negative")
        if super(TimeSeries,self).almost_equal_elem(other,tol=tol,relative=relative):
                        abs(self._delta_t-other._delta_t) <= dtol*self._delta_t)
            else:
                        abs(self._delta_t-other._delta_t) <= dtol)
        else:

        """
        Compare whether two time series are almost equal, normwise.

        Returns
        -------
        """
        if super(TimeSeries,self).almost_equal_norm(other,tol=tol,relative=relative):
            else:
        else:

    @_convert
        """Produces a LAL time series object equivalent to self.

        Returns
        -------
        lal_data : {lal.*TimeSeries}
            LAL time series object containing the same data as self.
            The actual type depends on the sample's dtype.  If the epoch of
            self is 'None', the epoch of the returned LAL object will be

        Raises
        ------
        TypeError
            If time series is stored in GPU memory.
        """
        ep = self._epoch

            lal_data = _lal.CreateREAL4TimeSeries("",ep,0,self.delta_t,_lal.SecondUnit,len(self))
            lal_data = _lal.CreateREAL8TimeSeries("",ep,0,self.delta_t,_lal.SecondUnit,len(self))
            lal_data = _lal.CreateCOMPLEX8TimeSeries("",ep,0,self.delta_t,_lal.SecondUnit,len(self))
            lal_data = _lal.CreateCOMPLEX16TimeSeries("",ep,0,self.delta_t,_lal.SecondUnit,len(self))

    def crop(self, left, right):
        """ Remove given seconds from either end of time series

        left : float
            Number of seconds of data to remove from the left of the time series.
        right : float
            Number of seconds of data to remove from the right of the time series.

        Returns
        -------
        cropped : pycbc.types.TimeSeries
            The reduced time series
        """
        if left + right > self.duration:
            raise ValueError('Cannot crop more data than we have')

        s = int(left * self.sample_rate)
        e = len(self) - int(right * self.sample_rate)
        return self[s:e]

    def save_to_wav(self, file_name):
        """ Save this time series to a wav format audio file.

        file_name : string
             The output file name
        """
        scaled = _numpy.int16(self.numpy()/max(abs(self)) * 32767)
        write_wav(file_name, int(self.sample_rate), scaled)

    def psd(self, segment_duration, **kwds):
        """ Calculate the power spectral density of this time series.

        Use the `pycbc.psd.welch` method to estimate the psd of this time segment.
        For more complete options, please see that function.

        segment_duration: float
            Duration in seconds to use for each sample of the spectrum.
        kwds : keywords
            Additional keyword arguments are passed on to the `pycbc.psd.welch` method.

        Returns
        -------
        psd : FrequencySeries
            Frequency series containing the estimated PSD.
        """
        from pycbc.psd import welch
        seg_len = int(segment_duration * self.sample_rate)
        seg_stride = int(seg_len / 2)
        return welch(self, seg_len=seg_len,
                           seg_stride=seg_stride,
                           **kwds)

    def gate(self, time, window=0.25, method='taper', copy=True,
             taper_width=0.25, invpsd=None):
        """ Gate out portion of time series

            Central time of the gate in seconds
        window: float
            Half-length in seconds to remove data around gate time.
        method: str
            Method to apply gate, options are 'hard', 'taper', and 'paint'.
        copy: bool
            If False, do operations inplace to this time series, else return
            new time series.
        taper_width: float
            Length of tapering region on either side of excized data. Only
            applies to the taper gating method.
        invpsd: pycbc.types.FrequencySeries
            The inverse PSD to use for painting method. If not given,
            a PSD is generated using default settings.

        Returns
        -------
        data: pycbc.types.TimeSeris
            Gated time series
        """
        data = self.copy() if copy else self
        if method == 'taper':
            from pycbc.strain import gate_data
            return gate_data(data, [(time, window, taper_width)])
        elif method == 'paint':
            # Uses the hole-filling method of
            # https://arxiv.org/pdf/1908.05644.pdf
            from pycbc.strain.gate import gate_and_paint
            if invpsd is None:
                # These are some bare minimum settings, normally you
                # should probably provide a psd
                invpsd = 1. / self.filter_psd(self.duration/32, self.delta_f, 0)
            lindex = int((time - window - self.start_time) / self.delta_t)
            rindex = lindex + int(2 * window / self.delta_t)
            lindex = lindex if lindex >= 0 else 0
            rindex = rindex if rindex <= len(self) else len(self)
            return gate_and_paint(data, lindex, rindex, invpsd, copy=False)
        elif method == 'hard':
            tslice = data.time_slice(time - window, time + window)
            tslice[:] = 0
            return data
        else:
            raise ValueError('Invalid method name: {}'.format(method))

    def filter_psd(self, segment_duration, delta_f, flow):

        The psd is then truncated in the time domain to the segment duration
        and interpolated to the requested sample frequency.

        delta_f : float
            Frequency spacing to return psd at.
        flow : float
            The low frequency cutoff to apply when truncating the inverse
            spectrum.

        Returns
        -------
        """
        from pycbc.psd import interpolate, inverse_spectrum_truncation
        p = self.psd(segment_duration)
        samples = int(p.sample_rate * segment_duration)
        p = interpolate(p, delta_f)
        return inverse_spectrum_truncation(p, samples,
                                           low_frequency_cutoff=flow,
                                           trunc_method='hann')

    def whiten(self, segment_duration, max_filter_duration, trunc_method='hann',
                     remove_corrupted=True, low_frequency_cutoff=None,
                     return_psd=False, **kwds):
        """ Return a whitened time series

        max_filter_duration : int
            Maximum length of the time-domain filter in seconds.
        trunc_method : {None, 'hann'}
            Function used for truncating the time-domain filter.
            None produces a hard truncation at `max_filter_len`.
        remove_corrupted : {True, boolean}
            If True, the region of the time series corrupted by the whitening
            is excised before returning. If false, the corrupted regions
            are not excised and the full time series is returned.
        low_frequency_cutoff : {None, float}
            Low frequency cutoff to pass to the inverse spectrum truncation.
            This should be matched to a known low frequency cutoff of the
            data if there is one.
        return_psd : {False, Boolean}
            Return the estimated and conditioned PSD that was used to whiten
            the data.

        Returns
        -------
        whitened_data : TimeSeries
            The whitened time series
        """
        from pycbc.psd import inverse_spectrum_truncation, interpolate
        # Estimate the noise spectrum
        psd = self.psd(segment_duration, **kwds)
        psd = interpolate(psd, self.delta_f)
        max_filter_len = int(max_filter_duration * self.sample_rate)

        # Interpolate and smooth to the desired corruption length
                   max_filter_len=max_filter_len,
                   low_frequency_cutoff=low_frequency_cutoff,
                   trunc_method=trunc_method)

        # Whiten the data by the asd
        white = (self.to_frequencyseries() / psd**0.5).to_timeseries()

        if remove_corrupted:
            white = white[int(max_filter_len/2):int(len(self)-max_filter_len/2)]

        if return_psd:
            return white, psd

        return white

    def qtransform(self, delta_t=None, delta_f=None, logfsteps=None,
                  frange=None, qrange=(4,64), mismatch=0.2, return_complex=False):
        """ Return the interpolated 2d qtransform of this data

        delta_t : {self.delta_t, float}
            The time resolution to interpolate to
        delta_f : float, Optional
            The frequency resolution to interpolate to
        logfsteps : int
            Do a log interpolation (incompatible with delta_f option) and set
            the number of steps to take.
        frange : {(30, nyquist*0.8), tuple of ints}
            frequency range
        qrange : {(4, 64), tuple}
            q range
        mismatch : float
            Mismatch between frequency tiles
        return_complex: {False, bool}
            return the raw complex series instead of the normalized power.

        Returns
        -------
        times : numpy.ndarray
            The time that the qtransform is sampled.
        freqs : numpy.ndarray
            The frequencies that the qtransform is sampled.
        qplane : numpy.ndarray (2d)
            The two dimensional interpolated qtransform of this time series.
        """
        from pycbc.filter.qtransform import qtiling, qplane
        from scipy.interpolate import interp2d

        if frange is None:
            frange = (30, int(self.sample_rate / 2 * 8))

        q_base = qtiling(self, qrange, frange, mismatch)
        _, times, freqs, q_plane = qplane(q_base, self.to_frequencyseries(),
                                          return_complex=return_complex)
        if logfsteps and delta_f:
            raise ValueError("Provide only one (or none) of delta_f and logfsteps")

        # Interpolate if requested
        if delta_f or delta_t or logfsteps:
            if return_complex:
                interp_amp = interp2d(times, freqs, abs(q_plane))
                interp_phase = interp2d(times, freqs, _numpy.angle(q_plane))
            else:
                interp = interp2d(times, freqs, q_plane)

        if delta_t:
            times = _numpy.arange(float(self.start_time),
                                    float(self.end_time), delta_t)
        if delta_f:
            freqs = _numpy.arange(int(frange[0]), int(frange[1]), delta_f)
        if logfsteps:
            freqs = _numpy.logspace(_numpy.log10(frange[0]),
                                    _numpy.log10(frange[1]),
                                     logfsteps)

                q_plane = _numpy.exp(1.0j * interp_phase(times, freqs))
                q_plane *= interp_amp(times, freqs)
            else:
                q_plane = interp(times, freqs)

        return times, freqs, q_plane

    def notch_fir(self, f1, f2, order, beta=5.0, remove_corrupted=True):
        """ notch filter the time series using an FIR filtered generated from
        the ideal response passed through a time-domain kaiser
        window (beta = 5.0)

        The suppression of the notch filter is related to the bandwidth and
        the number of samples in the filter length. For a few Hz bandwidth,
        a length corresponding to a few seconds is typically
        required to create significant suppression in the notched band.

        Time Series: TimeSeries
            The time series to be notched.
        f1: float
            The start of the frequency suppression.
        f2: float
            The end of the frequency suppression.
        order: int
            Number of corrupted samples on each side of the time series
        beta: float
            Beta parameter of the kaiser window that sets the side lobe attenuation.
        """
        from pycbc.filter import notch_fir
        ts = notch_fir(self, f1, f2, order, beta=beta)
            ts = ts[order:len(ts)-order]
        return ts

    def lowpass_fir(self, frequency, order, beta=5.0, remove_corrupted=True):
        """ Lowpass filter the time series using an FIR filtered generated from
        the ideal response passed through a kaiser window (beta = 5.0)

            The time series to be low-passed.
        frequency: float
            The frequency below which is suppressed.
        order: int
            If True, the region of the time series corrupted by the filtering
        """
        from pycbc.filter import lowpass_fir
        ts = lowpass_fir(self, frequency, order, beta=beta)
        return ts

    def highpass_fir(self, frequency, order, beta=5.0, remove_corrupted=True):
        """ Highpass filter the time series using an FIR filtered generated from

            The time series to be high-passed.
        order: int
        """
        from pycbc.filter import highpass_fir
        ts = highpass_fir(self, frequency, order, beta=beta)
        return ts

    def fir_zero_filter(self, coeff):
        """Filter the timeseries with a set of FIR coefficients

        coeff: numpy.ndarray
            FIR coefficients. Should be and odd length and symmetric.

        Returns
        -------
        filtered_series: pycbc.types.TimeSeries
            Return the filtered timeseries, which has been properly shifted to account
        for the FIR filter delay and the corrupted regions zeroed out.
        """
        from pycbc.filter import fir_zero_filter
        return self._return(fir_zero_filter(coeff, self))

    def save(self, path, group = None):
        """
        Save time series to a Numpy .npy, hdf, or text file. The first column
        contains the sample times, the second contains the values.
        In the case of a complex time series saved as text, the imaginary
        part is written as a third column. When using hdf format, the data is stored

        Raises
        ------
        """

            output = _numpy.vstack((self.sample_times.numpy(), self.numpy())).T
                output = _numpy.vstack((self.sample_times.numpy(),
        elif ext =='.hdf':
                ds.attrs['start_time'] = float(self.start_time)
                ds.attrs['delta_t'] = float(self.delta_t)
        else:
            raise ValueError('Path must end with .npy, .txt or .hdf')

    def to_timeseries(self):
        """ Return time series"""

    def to_frequencyseries(self, delta_f=None):
        """ Return the Fourier transform of this time series

        delta_f : {None, float}, optional
            The frequency resolution of the returned frequency series. By
        default the resolution is determined by the duration of the timeseries.

        Returns
        -------
        FrequencySeries:
            The fourier transform of this time series.
        """
        from pycbc.fft import fft
        if not delta_f:
            delta_f = 1.0 / self.duration

        tlen  = int(1.0 / delta_f / self.delta_t + 0.5)

        if tlen < len(self):
            raise ValueError("The value of delta_f (%s) would be "
                             "undersampled. Maximum delta_f "
                             "is %s." % (delta_f, 1.0 / self.duration))
            tmp = self
        else:
            tmp = TimeSeries(zeros(tlen, dtype=self.dtype),
                             delta_t=self.delta_t, epoch=self.start_time)

        f = FrequencySeries(zeros(flen,
                           dtype=complex_same_precision_as(self)),
                           delta_f=delta_f)
        fft(tmp, f)
        return f

    def inject(self, other, copy=True):
        """Return copy of self with other injected into it.

        The other vector will be resized and time shifted with sub-sample
        precision before adding. This assumes that one can assume zeros
        outside of the original vector range.
        """
        # only handle equal sample rate for now.
        if not self.sample_rate_close(other):
            raise ValueError('Sample rate must be the same')
        # determine if we want to inject in place or not
        if copy:
            ts = self.copy()
        else:
            ts = self
        # Other is disjoint
        if ((other.start_time >= ts.end_time) or
           (ts.start_time > other.end_time)):
            return ts

        other = other.copy()
        dt = float((other.start_time - ts.start_time) * ts.sample_rate)

        # This coaligns other to the time stepping of self
        if not dt.is_integer():
            diff = (dt - _numpy.floor(dt)) * ts.delta_t

            # insert zeros at end
            other.resize(len(other) + (len(other) + 1) % 2 + 1)

            # fd shift to the right
            other = other.cyclic_time_shift(diff)

        # get indices of other with respect to self
        # this is already an integer to floating point precission
        left = float(other.start_time - ts.start_time) * ts.sample_rate
        left = int(round(left))
        right = left + len(other)

        oleft = 0
        oright = len(other)

        # other overhangs on left so truncate
        if left < 0:
            oleft = -left
            left = 0

        # other overhangs on right so truncate
        if right > len(ts):
            oright = len(other) - (right - len(ts))
            right = len(ts)

        ts[left:right] += other[oleft:oright]
        return ts

    add_into = inject  # maintain backwards compatibility for now

        Note that data will be cyclically rotated, so if you shift by 2

        dt : float

        Returns
        -------
        data : pycbc.types.TimeSeries
            The time shifted time series.
        """
        # We do this in the frequency domain to allow us to do sub-sample
        # time shifts. This also results in the shift being circular. It
        # is left to a future update to do a faster impelementation in the case
        # where the time shift can be done with an exact number of samples.
        return self.to_frequencyseries().cyclic_time_shift(dt).to_timeseries()

        resized to match self. This may remove high frequency content or the

        Returns
        -------
        index: int
        """
        return self.to_frequencyseries().match(other, psd=psd,

    def detrend(self, type='linear'):
        """ Remove linear trend from the data

        Remove a linear trend from the data to improve the approximation that
        the data is circularly convolved, this helps reduce the size of filter
        transients from a circular convolution / filter.

        type: str
            The choice of detrending. The default ('linear') removes a linear
        least squares fit. 'constant' removes only the mean of the data.
        """
        from scipy.signal import detrend
        return self._return(detrend(self.numpy(), type=type))

        """ Basic plot of this time series
        """

            plot = pyplot.plot(self.sample_times, self, **kwds)
            plot1 = pyplot.plot(self.sample_times, self.real(), **kwds)
            plot2 = pyplot.plot(self.sample_times, self.imag(), **kwds)

def load_timeseries(path, group=None):
    """
    Load a TimeSeries from a .hdf, .txt or .npy file. The

    Raises
    ------
    """
        f = h5py.File(path)
        series = TimeSeries(data, delta_t=f[key].attrs['delta_t'],
                                  epoch=f[key].attrs['start_time'])
        f.close()
    else:

        delta_t = (data[-1][0] - data[0][0]) / (len(data)-1)
        return TimeSeries(data[:,1], delta_t=delta_t, epoch=epoch)
        return TimeSeries(data[:,1] + 1j*data[:,2],
                          delta_t=delta_t, epoch=epoch)
    else:

# Copyright (C) 2013  Stas Babak
#
#

from pycbc.filter import make_frequency_series
from pycbc.filter import  matched_filter_core

BACKEND_PREFIX="pycbc.vetoes.autochisq_"

def autochisq_from_precomputed(sn, corr_sn, hautocorr, indices,
                       stride=1, num_points=None, oneside=None,
                       twophase=True, maxvalued=False):
    """
    Compute correlation (two sided) between template and data
    and compares with autocorrelation of the template: C(t) = IFFT(A*A/S(f))

    sn: Array[complex]
        normalized (!) array of complex snr for the template that produced the
        trigger(s) being tested
    corr_sn : Array[complex]
        normalized (!) array of complex snr for the template that you want to
        produce a correlation chisq test for. In the [common] case that sn and
        corr_sn are the same, you are computing auto-correlation chisq.
    hautocorr: Array[complex]
        time domain autocorrelation for the template
    indices: Array[int]
        compute correlation chisquare at the points specified in this array,
    num_points: [int, optional; default=None]
        Number of points used for autochisq on each side, if None all points
        are used.
    stride: [int, optional; default = 1]
        stride for points selection for autochisq
        total length <= 2*num_points*stride
    oneside: [str, optional; default=None]
        whether to use one or two sided autochisquare. If None (or not
        provided) twosided chi-squared will be used. If given, options are
        'left' or 'right', to do one-sided chi-squared on the left or right.
    twophase: Boolean, optional; default=True
        If True calculate the auto-chisq using both phases of the filter.
        If False only use the phase of the obtained trigger(s).
    maxvalued: Boolean, optional; default=False
        Return the largest auto-chisq at any of the points tested if True.
        If False, return the sum of auto-chisq at all points tested.

    Returns
    -------
    autochisq: [tuple]
        returns autochisq values and snr corresponding to the instances
        of time defined by indices
    """
    Nsnr = len(sn)

    achisq = np.zeros(len(indices))
    num_points_all = int(Nsnr/stride)
    if num_points is None:
        num_points = num_points_all
    if (num_points > num_points_all):

    snrabs = np.abs(sn[indices])
    cphi_array = (sn[indices]).real / snrabs
    sphi_array = (sn[indices]).imag / snrabs

    start_point = - stride*num_points
    end_point = stride*num_points+1
    if oneside == 'left':
        achisq_idx_list = np.arange(start_point, 0, stride)
    elif oneside == 'right':
        achisq_idx_list = np.arange(stride, end_point, stride)
    else:
        achisq_idx_list_pt1 = np.arange(start_point, 0, stride)
        achisq_idx_list_pt2 = np.arange(stride, end_point, stride)
        achisq_idx_list = np.append(achisq_idx_list_pt1,
                                    achisq_idx_list_pt2)

    hauto_corr_vec = hautocorr[achisq_idx_list]
    hauto_norm = hauto_corr_vec.real*hauto_corr_vec.real
    # REMOVE THIS LINE TO REPRODUCE OLD RESULTS
    hauto_norm += hauto_corr_vec.imag*hauto_corr_vec.imag
    chisq_norm = 1.0 - hauto_norm

    for ip,ind in enumerate(indices):
        curr_achisq_idx_list = achisq_idx_list + ind

        cphi = cphi_array[ip]
        sphi = sphi_array[ip]
        # By construction, the other "phase" of the SNR is 0
        snr_ind =  sn[ind].real*cphi + sn[ind].imag*sphi

        # Wrap index if needed (maybe should fail in this case?)
        if curr_achisq_idx_list[0] < 0:
            curr_achisq_idx_list[curr_achisq_idx_list < 0] += Nsnr
        if curr_achisq_idx_list[-1] > (Nsnr - 1):
            curr_achisq_idx_list[curr_achisq_idx_list > (Nsnr-1)] -= Nsnr

        z = corr_sn[curr_achisq_idx_list].real*cphi + \
             corr_sn[curr_achisq_idx_list].imag*sphi
        dz = z - hauto_corr_vec.real*snr_ind
        curr_achisq_list = dz*dz/chisq_norm

        if twophase:
            chisq_norm = 1.0 - hauto_norm
            z = -corr_sn[curr_achisq_idx_list].real*sphi + \
                 corr_sn[curr_achisq_idx_list].imag*cphi
            dz = z - hauto_corr_vec.imag*snr_ind
            curr_achisq_list += dz*dz/chisq_norm

        if maxvalued:
            achisq[ip] = curr_achisq_list.max()
        else:
            achisq[ip] = curr_achisq_list.sum()

    dof = num_points
    if oneside is None:
        dof = dof * 2
    if twophase:
        dof = dof * 2

    return dof, achisq, indices

class SingleDetAutoChisq(object):
    """Class that handles precomputation and memory management for efficiently
    running the auto chisq in a single detector inspiral analysis.
    """
    def __init__(self, stride, num_points, onesided=None, twophase=False,
                 reverse_template=False, take_maximum_value=False,
                 maximal_value_dof=None):
        """
        Initialize autochisq calculation instance

        stride : int
            Number of sample points between points at which auto-chisq is
            calculated.
        num_points : int
            Number of sample points at which to calculate auto-chisq in each
            direction from the trigger
        onesided : optional, default=None, choices=['left','right']
            If None (default), calculate auto-chisq in both directions from the
            trigger. If left (backwards in time) or right (forwards in time)
            calculate auto-chisq only in that direction.
        twophase : optional, default=False
            If False calculate auto-chisq using only the phase of the trigger.
            If True, compare also against the orthogonal phase.
        reverse_template : optional, default=False
            If true, time-reverse the template before calculating auto-chisq.
            In this case this is more of a cross-correlation chisq than auto.
        take_maximum_value : optional, default=False
            If provided, instead of adding the auto-chisq value at each sample
            point tested, return only the maximum value.
        maximal_value_dof : int, required if using take_maximum_value
            If using take_maximum_value the expected value is not known. This
            value specifies what to store in the cont_chisq_dof output.
        """
        if stride > 0:
            self.do = True
            self.column_name = "cont_chisq"
            self.table_dof_name = "cont_chisq_dof"
            self.dof = num_points
            self.num_points = num_points
            self.stride = stride
            self.one_sided = onesided
            if (onesided is not None):
                self.dof = self.dof * 2
            self.two_phase = twophase
            if self.two_phase:
            self.reverse_template = reverse_template
            self.take_maximum_value=take_maximum_value
            if self.take_maximum_value:
                if maximal_value_dof is None:
                    err_msg = "Must provide the maximal_value_dof keyword "
                    err_msg += "argument if using the take_maximum_value "
                    err_msg += "option."
                self.dof = maximal_value_dof

            self._autocor = None
            self._autocor_id = None
        else:
            self.do = False

    def values(self, sn, indices, template, psd, norm, stilde=None,
               low_frequency_cutoff=None, high_frequency_cutoff=None):
        """
        Calculate the auto-chisq at the specified indices.

        sn : Array[complex]
            SNR time series of the template for which auto-chisq is being
            computed. Provided unnormalized.
        indices : Array[int]
            List of points at which to calculate auto-chisq
        template : Pycbc template object
            The template for which we are calculating auto-chisq
        psd : Pycbc psd object
            The PSD of the data being analysed
            The normalization factor to apply to sn
        stilde : Pycbc data object, needed if using reverse-template
            The data being analysed. Only needed if using reverse-template,
            otherwise ignored
        low_frequency_cutoff : float
            The lower frequency to consider in matched-filters
        high_frequency_cutoff : float
            The upper frequency to consider in matched-filters
        """
        if self.do and (len(indices) > 0):
            htilde = make_frequency_series(template)

            # Check if we need to recompute the autocorrelation
            key = (id(template), id(psd))
            if key != self._autocor_id:
                logging.info("Calculating autocorrelation")

                if not self.reverse_template:
                    Pt, _, P_norm = matched_filter_core(htilde,
                              htilde, psd=psd,
                              low_frequency_cutoff=low_frequency_cutoff,
                              high_frequency_cutoff=high_frequency_cutoff)
                    Pt = Pt * (1./ Pt[0])
                    self._autocor = Array(Pt, copy=True)
                else:
                    Pt, _, P_norm = matched_filter_core(htilde.conj(),

                    # T-reversed template has same norm as forward template
                    # so we can normalize using that
                    # FIXME: Here sigmasq has to be cast to a float or the
                    #        code is really slow ... why??
                    norm_fac = P_norm / float(((template.sigmasq(psd))**0.5))
                    Pt *= norm_fac
                self._autocor_id = key

            logging.info("...Calculating autochisquare")
            sn = sn*norm
            if self.reverse_template:
                assert(stilde is not None)
                asn, _, ahnrm = matched_filter_core(htilde.conj(), stilde,
                                 low_frequency_cutoff=low_frequency_cutoff,
                                 high_frequency_cutoff=high_frequency_cutoff,
                                 h_norm=template.sigmasq(psd))
                correlation_snr = asn * ahnrm
            else:
                correlation_snr = sn

            achi_list = np.array([])
            index_list = np.array(indices)
            dof, achi_list, _ = autochisq_from_precomputed(sn, correlation_snr,
                               self._autocor, index_list, stride=self.stride,
                               num_points=self.num_points,
                               oneside=self.one_sided, twophase=self.two_phase,
                               maxvalued=self.take_maximum_value)
            self.dof = dof
            return achi_list

class SingleDetSkyMaxAutoChisq(SingleDetAutoChisq):
    """Stub for precessing auto chisq if anyone ever wants to code it up.
    """
        super(SingleDetSkyMaxAutoChisq, self).__init__(*args, **kwds)

    def values(self, *args, **kwargs):
        if self.do:
            err_msg = "Precessing single detector sky-max auto chisq has not "
            err_msg += "been written. If you want to use it, why not help "
            err_msg += "write it?"
        else:

# Copyright (C) 2013  Ian Harry, Alex Nitz
#
#

#
#
#                                   Preamble
#
#
from pycbc.types import Array, zeros, real_same_precision_as, TimeSeries
from pycbc.filter import overlap_cplx, matched_filter_core
from pycbc.waveform import FilterBank

def segment_snrs(filters, stilde, psd, low_frequency_cutoff):
    """ This functions calculates the snr of each bank veto template against
    the segment

    filters: list of FrequencySeries
        The list of bank veto templates filters.
    stilde: FrequencySeries
        The current segment of data.
    psd: FrequencySeries

    Returns
    -------
    snr (list): List of snr time series.
    norm (list): List of normalizations factors for the snr time series.
    """
    snrs = []
    norms = []

    for bank_template in filters:
        # For every template compute the snr against the stilde segment
        snr, _, norm = matched_filter_core(
                bank_template, stilde, h_norm=bank_template.sigmasq(psd),
                psd=None, low_frequency_cutoff=low_frequency_cutoff)
        # SNR time series stored here
        snrs.append(snr)
        # Template normalization factor stored here
        norms.append(norm)

    return snrs, norms

def template_overlaps(bank_filters, template, psd, low_frequency_cutoff):
    """ This functions calculates the overlaps between the template and the
    bank veto templates.

    bank_filters: List of FrequencySeries
    template: FrequencySeries

    Returns
    -------
    overlaps: List of complex overlap values.
    """
    overlaps = []
    template_ow = template / psd
    for bank_template in bank_filters:
        overlap = overlap_cplx(template_ow, bank_template,
                low_frequency_cutoff=low_frequency_cutoff, normalized=False)
        norm = sqrt(1 / template.sigmasq(psd) / bank_template.sigmasq(psd))
        overlaps.append(overlap * norm)
        if (abs(overlaps[-1]) > 0.99):
            errMsg = "Overlap > 0.99 between bank template and filter. "
            errMsg += "This bank template will not be used to calculate "
            errMsg += "bank chisq for this filter template. The expected "
            errMsg += "value will be added to the chisq to account for "
            errMsg += "the removal of this template.\n"
            errMsg += "Masses of filter template: %e %e\n" \
                      %(template.params.mass1, template.params.mass2)
            errMsg += "Masses of bank filter template: %e %e\n" \
                      %(bank_template.params.mass1, bank_template.params.mass2)
            errMsg += "Overlap: %e" %(abs(overlaps[-1]))
            logging.info(errMsg)
    return overlaps

def bank_chisq_from_filters(tmplt_snr, tmplt_norm, bank_snrs, bank_norms,
        tmplt_bank_matches, indices=None):
    """ This function calculates and returns a TimeSeries object containing the
    bank veto calculated over a segment.

    tmplt_snr: TimeSeries
        The SNR time series from filtering the segment against the current
        search template
    tmplt_norm: float
        The normalization factor for the search template
    bank_snrs: list of TimeSeries
        The precomputed list of SNR time series between each of the bank veto
        templates and the segment
    bank_norms: list of floats
        The normalization factors for the list of bank veto templates
        (usually this will be the same for all bank veto templates)
    tmplt_bank_matches: list of floats
        The complex overlap between the search template and each
        of the bank templates
    indices: {None, Array}, optional
        Array of indices into the snr time series. If given, the bank chisq
        will only be calculated at these values.

    Returns
    -------
    bank_chisq: TimeSeries of the bank vetos
    """
        tmplt_snr = Array(tmplt_snr, copy=False)
        bank_snrs_tmp = []
        for bank_snr in bank_snrs:
            bank_snrs_tmp.append(bank_snr.take(indices))
        bank_snrs=bank_snrs_tmp

    # Initialise bank_chisq as 0s everywhere
    bank_chisq = zeros(len(tmplt_snr), dtype=real_same_precision_as(tmplt_snr))

    # Loop over all the bank templates
    for i in range(len(bank_snrs)):
        bank_match = tmplt_bank_matches[i]
        if (abs(bank_match) > 0.99):
            # Not much point calculating bank_chisquared if the bank template
            # is very close to the filter template. Can also hit numerical
            # error due to approximations made in this calculation.
            # The value of 2 is the expected addition to the chisq for this
            # template
            bank_chisq += 2.
            continue
        bank_norm = sqrt((1 - bank_match*bank_match.conj()).real)

        bank_SNR = bank_snrs[i] * (bank_norms[i] / bank_norm)
        tmplt_SNR = tmplt_snr * (bank_match.conj() * tmplt_norm / bank_norm)

        bank_SNR = Array(bank_SNR, copy=False)
        tmplt_SNR = Array(tmplt_SNR, copy=False)

        bank_chisq += (bank_SNR - tmplt_SNR).squared_norm()

        return bank_chisq
    else:
        return TimeSeries(bank_chisq, delta_t=tmplt_snr.delta_t,
                          epoch=tmplt_snr.start_time, copy=False)

class SingleDetBankVeto(object):
    """This class reads in a template bank file for a bank veto, handles the
       memory management of its filters internally, and calculates the bank
       veto TimeSeries.
    """
    def __init__(self, bank_file, flen, delta_f, f_low, cdtype, approximant=None, **kwds):
        if bank_file is not None:

            self.column_name = "bank_chisq"
            self.table_dof_name = "bank_chisq_dof"

            self.cdtype = cdtype
            self.delta_f = delta_f
            self.f_low = f_low
            self.seg_len_freq = flen
            self.seg_len_time = (self.seg_len_freq-1)*2

            logging.info("Read in bank veto template bank")
            bank_veto_bank = FilterBank(bank_file,
                    self.seg_len_freq,
                    self.delta_f, self.cdtype,
                    low_frequency_cutoff=f_low,
                    approximant=approximant, **kwds)

            self.filters = list(bank_veto_bank)
            self.dof = len(bank_veto_bank) * 2

            self._overlaps_cache = {}
            self._segment_snrs_cache = {}
        else:

    def cache_segment_snrs(self, stilde, psd):
        key = (id(stilde), id(psd))
        if key not in self._segment_snrs_cache:
            logging.info("Precalculate the bank veto template snrs")
            data = segment_snrs(self.filters, stilde, psd, self.f_low)
            self._segment_snrs_cache[key] = data
        return self._segment_snrs_cache[key]

    def cache_overlaps(self, template, psd):
        key = (id(template.params), id(psd))
        if key not in self._overlaps_cache:
            logging.info("...Calculate bank veto overlaps")
            o = template_overlaps(self.filters, template, psd, self.f_low)
            self._overlaps_cache[key] = o
        return self._overlaps_cache[key]

    def values(self, template, psd, stilde, snrv, norm, indices):
        """
        Returns
        -------
        bank_chisq_from_filters: TimeSeries of bank veto values - if indices
        is None then evaluated at all time samples, if not then only at
        requested sample indices

        bank_chisq_dof: int, approx number of statistical degrees of freedom
        """
            logging.info("...Doing bank veto")
            overlaps = self.cache_overlaps(template, psd)
            bank_veto_snrs, bank_veto_norms = self.cache_segment_snrs(stilde, psd)
            chisq = bank_chisq_from_filters(snrv, norm, bank_veto_snrs,
                                            bank_veto_norms, overlaps, indices)
            dof = numpy.repeat(self.dof, len(chisq))
            return chisq, dof
        else:

class SingleDetSkyMaxBankVeto(SingleDetBankVeto):
    """Stub for precessing bank veto if anyone ever wants to code it up.
    """
        super(SingleDetSkyMaxBankVeto, self).__init__(*args, **kwds)

            err_msg = "Precessing single detector sky-max bank veto has not "
        else:

#
#

#
#
#                                   Preamble
#
#
import numpy, logging, math, pycbc.fft

from pycbc.types import zeros, real_same_precision_as, TimeSeries, complex_same_precision_as
from pycbc.filter import sigmasq_series, make_frequency_series, matched_filter_core, get_cutoff_indices
import pycbc.pnutils

BACKEND_PREFIX="pycbc.vetoes.chisq_"

def power_chisq_bins_from_sigmasq_series(sigmasq_series, num_bins, kmin, kmax):
    """Returns bins of equal power for use with the chisq functions

    sigmasq_series: FrequencySeries
        A frequency series containing the cumulative power of a filter template
        preweighted by a psd.
    num_bins: int
        The number of chisq bins to calculate.
    kmin: int
        DOCUMENTME
    kmax: int

    Returns
    -------

    bins: List of ints
        A list of the edges of the chisq bins is returned.

    """
    sigmasq = sigmasq_series[kmax - 1]
    edge_vec = numpy.arange(0, num_bins) * sigmasq / num_bins
    bins = numpy.searchsorted(sigmasq_series[kmin:kmax], edge_vec, side='right')
    bins += kmin
    return numpy.append(bins, kmax)

def power_chisq_bins(htilde, num_bins, psd, low_frequency_cutoff=None,
                     high_frequency_cutoff=None):

    htilde: FrequencySeries
        A frequency series containing the template waveform
        A frequency series containing the psd. Its length must be commensurate
        with the template waveform.
    low_frequency_cutoff: {None, float}, optional
        The low frequency cutoff to apply
    high_frequency_cutoff: {None, float}, optional
        The high frequency cutoff to apply

    Returns
    -------

    """
    sigma_vec = sigmasq_series(htilde, psd, low_frequency_cutoff,
                               high_frequency_cutoff).numpy()
                                    high_frequency_cutoff,
                                    htilde.delta_f,
                                    (len(htilde)-1)*2)
    return power_chisq_bins_from_sigmasq_series(sigma_vec, num_bins, kmin, kmax)

def chisq_accum_bin(chisq, q):

def shift_sum(v1, shifts, bins):
    """ Calculate the time shifted sum of the FrequencySeries
    """

def power_chisq_at_points_from_precomputed(corr, snr, snr_norm, bins, indices):
    """Calculate the chisq timeseries from precomputed values for only select points.

    This function calculates the chisq at each point by explicitly time shifting
    and summing each bin. No FFT is involved.

    corr: FrequencySeries
        The product of the template and data in the frequency domain.
    snr: numpy.ndarray
        The unnormalized array of snr values at only the selected points in `indices`.
    snr_norm: float
        The normalization of the snr (EXPLAINME : refer to Findchirp paper?)
    bins: List of integers
        The edges of the equal power bins
        The indices where we will calculate the chisq. These must be relative
        to the given `corr` series.

    Returns
    -------
    chisq: Array
        An array containing only the chisq at the selected points.
    """
    num_bins = len(bins) - 1
    chisq = shift_sum(corr, indices, bins) # pylint:disable=assignment-from-no-return
    return (chisq * num_bins - (snr.conj() * snr).real) * (snr_norm ** 2.0)

_q_l = None
_qtilde_l = None
_chisq_l = None
def power_chisq_from_precomputed(corr, snr, snr_norm, bins, indices=None, return_bins=False):
    """Calculate the chisq timeseries from precomputed values.

    This function calculates the chisq at all times by performing an
    inverse FFT of each bin.

        The produce of the template and data in the frequency domain.
    snr: TimeSeries
        The unnormalized snr time series.
    snr_norm:
        The snr normalization factor (true snr = snr * snr_norm) EXPLAINME - define 'true snr'?
        The edges of the chisq bins.
    indices: {Array, None}, optional
        Index values into snr that indicate where to calculate
        chisq values. If none, calculate chisq for all possible indices.
    return_bins: {boolean, False}, optional
        Return a list of the SNRs for each chisq bin.

    Returns
    -------
    chisq: TimeSeries
    """
    # Get workspace memory
    global _q_l, _qtilde_l, _chisq_l

    bin_snrs = []

    if _q_l is None or len(_q_l) != len(snr):
        q = zeros(len(snr), dtype=complex_same_precision_as(snr))
        qtilde = zeros(len(snr), dtype=complex_same_precision_as(snr))
        _q_l = q
        _qtilde_l = qtilde
    else:
        q = _q_l
        qtilde = _qtilde_l

        snr = snr.take(indices)

    if _chisq_l is None or len(_chisq_l) < len(snr):
        chisq = zeros(len(snr), dtype=real_same_precision_as(snr))
        _chisq_l = chisq
    else:
        chisq = _chisq_l[0:len(snr)]
        chisq.clear()

    for j in range(num_bins):
        k_min = int(bins[j])
        k_max = int(bins[j+1])

        qtilde[k_min:k_max] = corr[k_min:k_max]
        pycbc.fft.ifft(qtilde, q)
        qtilde[k_min:k_max].clear()

        if return_bins:
            bin_snrs.append(TimeSeries(q  * snr_norm *  num_bins ** 0.5,
                                      delta_t=snr.delta_t,
                                      epoch=snr.start_time))

        if indices is not None:
            chisq_accum_bin(chisq, q.take(indices))
        else:
            chisq_accum_bin(chisq, q)

    chisq = (chisq * num_bins - snr.squared_norm()) * (snr_norm ** 2.0)

    if indices is None:
        chisq = TimeSeries(chisq, delta_t=snr.delta_t, epoch=snr.start_time, copy=False)

    if return_bins:
        return chisq, bin_snrs
    else:
        return chisq

def fastest_power_chisq_at_points(corr, snr, snrv, snr_norm, bins, indices):
    """Calculate the chisq values for only selected points.

    This function looks at the number of points to be evaluated and selects
    the fastest method (FFT, or direct time shift and sum). In either case,
    only the selected points are returned.

    snr: Array
        The unnormalized snr
        The snr normalization factor  --- EXPLAINME
        to the given `snr` series.

    Returns
    -------
    """
    import pycbc.scheme
    if isinstance(pycbc.scheme.mgr.state, pycbc.scheme.CPUScheme):
        # We don't have that many points so do the direct time shift.
        return power_chisq_at_points_from_precomputed(corr, snrv,
                                                      snr_norm, bins, indices)
    else:
        # We have a lot of points so it is faster to use the fourier transform
        return power_chisq_from_precomputed(corr, snr, snr_norm, bins,
                                            indices=indices)

def power_chisq(template, data, num_bins, psd,
                low_frequency_cutoff=None,
                high_frequency_cutoff=None,
                return_bins=False):
    """Calculate the chisq timeseries

    template: FrequencySeries or TimeSeries
        A time or frequency series that contains the filter template.
    data: FrequencySeries or TimeSeries
        A time or frequency series that contains the data to filter. The length
        must be commensurate with the template.
        (EXPLAINME - does this mean 'the same as' or something else?)
        The number of bins in the chisq. Note that the dof goes as 2*num_bins-2.
        The psd of the data.
        The low frequency cutoff for the filter
        The high frequency cutoff for the filter
        Return a list of the individual chisq bins

    Returns
    -------
        TimeSeries containing the chisq values for all times.
    """

    bins = power_chisq_bins(htilde, num_bins, psd, low_frequency_cutoff,
                            high_frequency_cutoff)
    corra = zeros((len(htilde)-1)*2, dtype=htilde.dtype)
    total_snr, corr, tnorm = matched_filter_core(htilde, stilde, psd,
                           low_frequency_cutoff, high_frequency_cutoff,
                           corr_out=corra)

    return power_chisq_from_precomputed(corr, total_snr, tnorm, bins, return_bins=return_bins)

class SingleDetPowerChisq(object):
    running the power chisq in a single detector inspiral analysis.
    """
    def __init__(self, num_bins=0, snr_threshold=None):
        if not (num_bins == "0" or num_bins == 0):
            self.column_name = "chisq"
            self.table_dof_name = "chisq_dof"
            self.num_bins = num_bins
        else:

    def parse_option(row, arg):
        safe_dict = {}
        safe_dict.update(row.__dict__)
        safe_dict.update(math.__dict__)
        safe_dict.update(pycbc.pnutils.__dict__)
        return eval(arg, {"__builtins__":None}, safe_dict)

    def cached_chisq_bins(self, template, psd):
        from pycbc.opt import LimitedSizeDict

        key = id(psd)
        if not hasattr(psd, '_chisq_cached_key'):
            psd._chisq_cached_key = {}

        if not hasattr(template, '_bin_cache'):
            template._bin_cache = LimitedSizeDict(size_limite=2**2)

        if key not in template._bin_cache or id(template.params) not in psd._chisq_cached_key:
            psd._chisq_cached_key[id(template.params)] = True
            num_bins = int(self.parse_option(template, self.num_bins))

            if hasattr(psd, 'sigmasq_vec') and \
                    template.approximant in psd.sigmasq_vec:
                kmin = int(template.f_lower / psd.delta_f)
                kmax = template.end_idx
                bins = power_chisq_bins_from_sigmasq_series(
                    psd.sigmasq_vec[template.approximant],
                    num_bins,
                    kmin,
                    kmax
                )
            else:
                bins = power_chisq_bins(template, num_bins, psd, template.f_lower)
            template._bin_cache[key] = bins

        return template._bin_cache[key]

    def values(self, corr, snrv, snr_norm, psd, indices, template):
        """ Calculate the chisq at points given by indices.

        Returns
        -------
        chisq: Array
            Chisq values, one for each sample index

        chisq_dof: Array
            Number of statistical degrees of freedom for the chisq test
            in the given template
        """

            num_above = len(indices)
            if self.snr_threshold:
                above = abs(snrv * snr_norm) > self.snr_threshold
                num_above = above.sum()
                logging.info('%s above chisq activation threshold' % num_above)
                above_indices = indices[above]
                above_snrv = snrv[above]
                rchisq = numpy.zeros(len(indices), dtype=numpy.float32)
                dof = -100
            else:
                above_indices = indices
                above_snrv = snrv

            if num_above > 0:
                bins = self.cached_chisq_bins(template, psd)
                dof = (len(bins) - 1) * 2 - 2
                chisq = power_chisq_at_points_from_precomputed(corr,
                                     above_snrv, snr_norm, bins, above_indices)

                if num_above > 0:
                    rchisq[above] = chisq
            else:
                rchisq = chisq

            return rchisq, numpy.repeat(dof, len(indices))# dof * numpy.ones_like(indices)
        else:

class SingleDetSkyMaxPowerChisq(SingleDetPowerChisq):
    running the power chisq in a single detector inspiral analysis when
    maximizing analytically over sky location.
    """
    def __init__(self, **kwds):
        super(SingleDetSkyMaxPowerChisq, self).__init__(**kwds)
        self.template_mem = None
        self.corr_mem = None

    def calculate_chisq_bins(self, template, psd):
        """ Obtain the chisq bins for this template and PSD.
        """
        num_bins = int(self.parse_option(template, self.num_bins))
        if hasattr(psd, 'sigmasq_vec') and \
                                       template.approximant in psd.sigmasq_vec:
            kmin = int(template.f_lower / psd.delta_f)
            kmax = template.end_idx
            bins = power_chisq_bins_from_sigmasq_series(
                   psd.sigmasq_vec[template.approximant], num_bins, kmin, kmax)
        else:
            bins = power_chisq_bins(template, num_bins, psd, template.f_lower)
        return bins

    def values(self, corr_plus, corr_cross, snrv, psd,
               indices, template_plus, template_cross, u_vals,
               hplus_cross_corr, hpnorm, hcnorm):

        Returns
        -------

        """
                above = abs(snrv) > self.snr_threshold
                u_vals = u_vals[above]
                dof = -100
            else:

                chisq = []
                curr_tmplt_mult_fac = 0.
                curr_corr_mult_fac = 0.
                if self.template_mem is None or \
                        (not len(self.template_mem) == len(template_plus)):
                    self.template_mem = zeros(len(template_plus),
                                dtype=complex_same_precision_as(corr_plus))
                if self.corr_mem is None or \
                                (not len(self.corr_mem) == len(corr_plus)):
                    self.corr_mem = zeros(len(corr_plus),

                tmplt_data = template_cross.data
                corr_data = corr_cross.data
                numpy.copyto(self.template_mem.data, template_cross.data)
                numpy.copyto(self.corr_mem.data, corr_cross.data)
                template_cross._data = self.template_mem.data
                corr_cross._data = self.corr_mem.data

                for lidx, index in enumerate(above_indices):
                    above_local_indices = numpy.array([index])
                    above_local_snr = numpy.array([above_snrv[lidx]])
                    local_u_val = u_vals[lidx]
                    # Construct template from _plus and _cross
                    # Note that this modifies in place, so we store that and
                    # revert on the next pass.
                    template = template_cross.multiply_and_add(template_plus,
                                               local_u_val-curr_tmplt_mult_fac)
                    curr_tmplt_mult_fac = local_u_val

                    template.f_lower = template_plus.f_lower
                    template.params = template_plus.params
                    # Construct the corr vector
                    norm_fac = local_u_val*local_u_val + 1
                    norm_fac += 2 * local_u_val * hplus_cross_corr
                    norm_fac = hcnorm / (norm_fac**0.5)
                    hp_fac = local_u_val * hpnorm / hcnorm
                    corr = corr_cross.multiply_and_add(corr_plus,
                                                   hp_fac - curr_corr_mult_fac)
                    curr_corr_mult_fac = hp_fac

                    bins = self.calculate_chisq_bins(template, psd)
                    dof = (len(bins) - 1) * 2 - 2
                    curr_chisq = power_chisq_at_points_from_precomputed(corr,
                                          above_local_snr/ norm_fac, norm_fac,
                                          bins, above_local_indices)
                    chisq.append(curr_chisq[0])
                chisq = numpy.array(chisq)
                # Must reset corr and template to original values!
                template_cross._data = tmplt_data
                corr_cross._data = corr_data

            else:

        else:

"""Chisq based on sine-gaussian tiles.
See https://arxiv.org/abs/1709.08974 for a discussion.
"""

from pycbc.waveform.utils import apply_fseries_time_shift
from pycbc.filter import sigma
from pycbc.waveform import sinegauss
from pycbc.vetoes.chisq import SingleDetPowerChisq

class SingleDetSGChisq(SingleDetPowerChisq):
    running the sine-Gaussian chisq
    """
    returns = {'sg_chisq': numpy.float32}

    def __init__(self, bank, num_bins=0,
                       snr_threshold=None,
                       chisq_locations=None):
        """ Create sine-Gaussian Chisq Calculator

        bank: pycbc.waveform.TemplateBank
            The template bank that will be processed.
        num_bins: str
            The string determining the number of power chisq bins
            The threshold to calculate the sine-Gaussian chisq
        chisq_locations: list of strs
            List of strings which detail where to place a sine-Gaussian.
            The format is 'region-boolean:q1-offset1,q2-offset2'.
            The offset is relative to the end frequency of the approximant.
            The region is a boolean expression such as 'mtotal>40' indicating
            which templates to apply this set of sine-Gaussians to.
        """
        if snr_threshold is not None:
            self.snr_threshold = snr_threshold
            self.params = {}
            for descr in chisq_locations:
                region, values = descr.split(":")
                mask = bank.table.parse_boolargs([(1, region), (0, 'else')])[0]
                hashes = bank.table['template_hash'][mask.astype(bool)]
                for h in hashes:
                    self.params[h] = values
        else:

    def insert_option_group(parser):
        group = parser.add_argument_group("Sine-Gaussian Chisq")
        group.add_argument("--sgchisq-snr-threshold", type=float,
            help="Minimum SNR threshold to use SG chisq")
        group.add_argument("--sgchisq-locations", type=str, nargs='+',
            help="Frequency offsets and quality factors of the sine-Gaussians"
                 " to use, format 'region-boolean:q1-offset1,q2-offset2'. "
                 "Offset is relative to the end frequency of the approximant."
                 " Region is a boolean expression selecting templates to "
                 "apply the sine-Gaussians to, ex. 'mtotal>40'")

    def from_cli(cls, args, bank, chisq_bins):
        return cls(bank, chisq_bins,
                   args.sgchisq_snr_threshold,
                   args.sgchisq_locations)

    def values(self, stilde, template, psd, snrv, snr_norm,
                     bchisq, bchisq_dof, indices):
        """ Calculate sine-Gaussian chisq

        stilde: pycbc.types.Frequencyseries
            The overwhitened strain
        template: pycbc.types.Frequencyseries
            The waveform template being analyzed
        psd: pycbc.types.Frequencyseries
            The power spectral density of the data
        snrv: numpy.ndarray
            The peak unnormalized complex SNR values
        snr_norm: float
            The normalization factor for the snr
        bchisq: numpy.ndarray
            The Bruce Allen power chisq values for these triggers
        bchisq_dof: numpy.ndarray
            The degrees of freedom of the Bruce chisq
        indics: numpy.ndarray
            The indices of the snr peaks.

        Returns
        -------
        """
        if not self.do:

        if template.params.template_hash not in self.params:
            return numpy.ones(len(snrv))
        values = self.params[template.params.template_hash].split(',')

        # Get the chisq bins to use as the frequency reference point
        bins = self.cached_chisq_bins(template, psd)

        # This is implemented slowly, so let's not call it often, OK?
        chisq = numpy.ones(len(snrv))
        for i, snrvi in enumerate(snrv):
            #Skip if newsnr too low
            snr = abs(snrvi * snr_norm)
            nsnr = ranking.newsnr(snr, bchisq[i] / bchisq_dof[i])
            if nsnr < self.snr_threshold:
                continue

            N = (len(template) - 1) * 2
            dt = 1.0 / (N * template.delta_f)
            time = float(template.epoch) + dt * indices[i]
            # Shift the time of interest to be centered on 0
            stilde_shift = apply_fseries_time_shift(stilde, -time)

            # Only apply the sine-Gaussian in a +-50 Hz range around the
            # central frequency
            qwindow = 50
            chisq[i] = 0

            # Estimate the maximum frequency up to which the waveform has
            # power by approximating power per frequency
            # as constant over the last 2 chisq bins. We cannot use the final
            # chisq bin edge as it does not have to be where the waveform
            # terminates.
            fstep = (bins[-2] - bins[-3])
            fpeak = (bins[-2] + fstep) * template.delta_f

            # This is 90% of the Nyquist frequency of the data
            # This allows us to avoid issues near Nyquist due to resample
            # Filtering
            fstop = len(stilde) * stilde.delta_f * 0.9

            dof = 0
            # Calculate the sum of SNR^2 for the sine-Gaussians specified
            for descr in values:
                # Get the q and frequency offset from the descriptor
                q, offset = descr.split('-')
                q, offset = float(q), float(offset)
                fcen = fpeak + offset
                flow = max(kmin * template.delta_f, fcen - qwindow)
                fhigh = fcen + qwindow

                # If any sine-gaussian tile has an upper frequency near
                # nyquist return 1 instead.
                if fhigh > fstop:
                    return numpy.ones(len(snrv))

                kmin = int(flow / template.delta_f)
                kmax = int(fhigh / template.delta_f)

                #Calculate sine-gaussian tile
                gtem = sinegauss.fd_sine_gaussian(1.0, q, fcen, flow,
                                      len(template) * template.delta_f,
                                      template.delta_f).astype(numpy.complex64)
                gsigma = sigma(gtem, psd=psd,
                                     low_frequency_cutoff=flow,
                                     high_frequency_cutoff=fhigh)
                #Calculate the SNR of the tile
                gsnr = (gtem[kmin:kmax] * stilde_shift[kmin:kmax]).sum()
                gsnr *= 4.0 * gtem.delta_f / gsigma
                chisq[i] += abs(gsnr)**2.0
                dof += 2
            if dof == 0:
                chisq[i] = 1
            else:
                chisq[i] /= dof

# Copyright (C) 2012  Alex Nitz, Josh Willis, Andrew Miller
#
#
#

#
#
#                                   Preamble
#
#
"""
This module provides classes that describe banks of waveforms
"""
from copy import copy
from glue.ligolw import ligolw, table, lsctables, utils as ligolw_utils
import pycbc.waveform
import pycbc.waveform.compress
import six
import hashlib

def sigma_cached(self, psd):
    """ Cache sigma calculate for use in tandem with the FilterBank class
    """
    if not hasattr(self, '_sigmasq'):
        self._sigmasq = LimitedSizeDict(size_limit=2**5)

    key = id(psd)
    if not hasattr(psd, '_sigma_cached_key'):
        psd._sigma_cached_key = {}

    if key not in self._sigmasq or id(self) not in psd._sigma_cached_key:
        psd._sigma_cached_key[id(self)] = True
        # If possible, we precalculate the sigmasq vector for all possible waveforms
        if pycbc.waveform.waveform_norm_exists(self.approximant):
            if not hasattr(psd, 'sigmasq_vec'):
                psd.sigmasq_vec = {}

            if self.approximant not in psd.sigmasq_vec:
                psd.sigmasq_vec[self.approximant] = \
                    pycbc.waveform.get_waveform_filter_norm(
                        self.approximant,
                        psd,
                        len(psd),
                        psd.delta_f,
                        self.min_f_lower
                    )

            if not hasattr(self, 'sigma_scale'):
                # Get an amplitude normalization (mass dependant constant norm)
                amp_norm = pycbc.waveform.get_template_amplitude_norm(
                                     self.params, approximant=self.approximant)
                amp_norm = 1 if amp_norm is None else amp_norm
                self.sigma_scale = (DYN_RANGE_FAC * amp_norm) ** 2.0

            curr_sigmasq = psd.sigmasq_vec[self.approximant]

            kmin = int(self.f_lower / psd.delta_f)
            self._sigmasq[key] = self.sigma_scale * \
                (curr_sigmasq[self.end_idx-1] - curr_sigmasq[kmin])

        else:
            if not hasattr(self, 'sigma_view'):
                from pycbc.filter.matchedfilter import get_cutoff_indices
                N = (len(self) -1) * 2
                kmin, kmax = get_cutoff_indices(
                        self.min_f_lower or self.f_lower, self.end_frequency,
                        self.delta_f, N)
                self.sslice = slice(kmin, kmax)
                self.sigma_view = self[self.sslice].squared_norm() * 4.0 * self.delta_f

            if not hasattr(psd, 'invsqrt'):
                psd.invsqrt = 1.0 / psd

            self._sigmasq[key] = self.sigma_view.inner(psd.invsqrt[self.sslice])
    return self._sigmasq[key]

    pass

# helper function for parsing approximant strings
def boolargs_from_apprxstr(approximant_strs):
    """Parses a list of strings specifying an approximant and where that
    approximant should be used into a list that can be understood by
    FieldArray.parse_boolargs.

    apprxstr : (list of) string(s)
        The strings to parse. Each string should be formatted `APPRX:COND`,
        where `APPRX` is the approximant and `COND` is a string specifying
        where it should be applied (see `FieldArgs.parse_boolargs` for examples
        of conditional strings). The last string in the list may exclude a
        conditional argument, which is the same as specifying ':else'.

    Returns
    -------
    boolargs : list
        A list of tuples giving the approximant and where to apply them. This
        can be passed directly to `FieldArray.parse_boolargs`.
    """
    if not isinstance(approximant_strs, list):
        approximant_strs = [approximant_strs]
    return [tuple(arg.split(':')) for arg in approximant_strs]

def add_approximant_arg(parser, default=None, help=None):
    """Adds an approximant argument to the given parser.

    parser : ArgumentParser
        The argument parser to add the argument to.
    default : {None, str}
        Specify a default for the approximant argument. Defaults to None.
    help : {None, str}
        Provide a custom help message. If None, will use a descriptive message
        on how to specify the approximant.
    """
    if help is None:
        help=str("The approximant(s) to use. Multiple approximants to use "
             "in different regions may be provided. If multiple "
             "approximants are provided, every one but the last must be "
             "be followed by a conditional statement defining where that "
             "approximant should be used. Conditionals can be any boolean "
             "test understood by numpy. For example, 'Apprx:(mtotal > 4) & "
             "(mchirp <= 5)' would use approximant 'Apprx' where total mass "
             "is > 4 and chirp mass is <= 5. "
             "Conditionals are applied in order, with each successive one "
             "only applied to regions not covered by previous arguments. "
             "For example, `'TaylorF2:mtotal < 4' 'IMRPhenomD:mchirp < 3'` "
             "would result in IMRPhenomD being used where chirp mass is < 3 "
             "and total mass is >= 4. The last approximant given may use "
             "'else' as the conditional or include no conditional. In either "
             "case, this will cause the last approximant to be used in any "
             "remaning regions after all the previous conditionals have been "
             "applied. For the full list of possible parameters to apply "
             "conditionals to, see WaveformArray.default_fields(). Math "
             "operations may also be used on parameters; syntax is python, "
             "with any operation recognized by numpy.")
    parser.add_argument("--approximant", nargs='+', type=str, default=default,
                        metavar='APPRX[:COND]',
                        help=help)

def parse_approximant_arg(approximant_arg, warray):
    """Given an approximant arg (see add_approximant_arg) and a field
    array, figures out what approximant to use for each template in the array.

    approximant_arg : list
        The approximant argument to parse. Should be the thing returned by
        ArgumentParser when parsing the argument added by add_approximant_arg.
    warray : FieldArray
        The array to parse. Must be an instance of a FieldArray, or a class
        that inherits from FieldArray.

    Returns
    -------
    array
        A numpy array listing the approximants to use for each element in
        the warray.
    """
    return warray.parse_boolargs(boolargs_from_apprxstr(approximant_arg))[0]

def tuple_to_hash(tuple_to_be_hashed):
    """
    Return a hash for a numpy array, avoids native (unsafe) python3 hash function

    tuple_to_be_hashed: tuple
        The tuple which is being hashed
        Must be convertible to a numpy array

    Returns
    -------
    int
        an integer representation of the hashed array
    """
    if six.PY2:
        return hash(tuple_to_be_hashed)
    h = hashlib.blake2b(np.array(tuple_to_be_hashed).tobytes('C'),
                        digest_size=8)
    return np.fromstring(h.digest(), dtype=int)[0]

class TemplateBank(object):
    """Class to provide some basic helper functions and information
    about elements of a template bank.

        The name of the file to load. Must end in '.xml[.gz]' or '.hdf'. If an
        hdf file, it should have a 'parameters' in its `attrs` which gives a
        list of the names of fields to load from the file. If no 'parameters'
        are found, all of the top-level groups in the file will assumed to be
        parameters (a warning will be printed to stdout in this case). If an
        xml file, it must have a `SnglInspiral` table.
    approximant : {None, (list of) string(s)}
        Specify the approximant(s) for each template in the bank. If None
        provided, will try to load the approximant from the file. The
        approximant may either be a single string (in which case the same
        approximant will be used for all templates) or a list of strings and
        conditionals specifying where to use the approximant. See
        `boolargs_from_apprxstr` for syntax.
    parameters : {None, (list of) sting(s)}
        Specify what parameters to load from the file. If None, all of the
        parameters in the file (if an xml file, this is all of the columns in
        the SnglInspiral table, if an hdf file, this is given by the
        parameters attribute in the file). The list may include parameters that
        are derived from the file's parameters, or functions thereof. For a
        full list of possible parameters, see `WaveformArray.default_fields`.
        If a derived parameter is specified, only the parameters needed to
        compute that parameter will be loaded from the file. For example, if
        `parameters='mchirp'`, then only `mass1, mass2` will be loaded from
        the file. Note that derived parameters can only be used if the
        needed parameters are in the file; e.g., you cannot use `chi_eff` if
        `spin1z`, `spin2z`, `mass1`, and `mass2` are in the input file.
    \**kwds :
        Any additional keyword arguments are stored to the `extra_args`

    table : WaveformArray
        An instance of a WaveformArray containing all of the information about
        the parameters of the bank.
    has_compressed_waveforms : {False, bool}
        True if compressed waveforms are present in the the (hdf) file; False
    parameters : tuple
        The parameters loaded from the input file. Same as `table.fieldnames`.
    indoc : {None, xmldoc}
        If an xml file was provided, an in-memory representation of the xml.
        Otherwise, None.
    filehandler : {None, h5py.File}
        If an hdf file was provided, the file handler pointing to the hdf file
        (left open after initialization). Otherwise, None.
    extra_args : {None, dict}
        Any extra keyword arguments that were provided on initialization.
    """
    def __init__(self, filename, approximant=None, parameters=None,
                 **kwds):
        self.has_compressed_waveforms = False
            self.filehandler = None
            self.indoc = ligolw_utils.load_filename(
                filename, False, contenthandler=LIGOLWContentHandler)
            self.table = table.get_table(
                self.indoc, lsctables.SnglInspiralTable.tableName)
            self.table = pycbc.io.WaveformArray.from_ligolw_table(self.table,
                columns=parameters)

            # inclination stored in xml alpha3 column
            names = list(self.table.dtype.names)
            names = tuple([n if n != 'alpha3' else 'inclination' for n in names])

            # low frequency cutoff in xml alpha6 column
            names = tuple([n if n!= 'alpha6' else 'f_lower' for n in names])
            self.table.dtype.names = names

        elif ext.endswith(('hdf', '.h5')):
            self.indoc = None
            self.filehandler = f
            try:
                fileparams = list(f.attrs['parameters'])
                # just assume all of the top-level groups are the parameters
                fileparams = list(f.keys())
                logging.info("WARNING: no parameters attribute found. "
                    "Assuming that %s " %(', '.join(fileparams)) +
                    "are the parameters.")
            tmp_params = []
            # At this point fileparams might be bytes. Fix if it is
            for param in fileparams:
                try:
                    param = param.decode()
                    tmp_params.append(param)
            fileparams = tmp_params

            # use WaveformArray's syntax parser to figure out what fields
            # need to be loaded
            if parameters is None:
                parameters = fileparams
            common_fields = list(pycbc.io.WaveformArray(1,
                names=parameters).fieldnames)
            add_fields = list(set(parameters) &
                (set(fileparams) - set(common_fields)))
            # load
            dtype = []
            data = {}
            for key in common_fields+add_fields:
                data[key] = f[key][:]
                dtype.append((key, data[key].dtype))
            num = f[fileparams[0]].size
            self.table = pycbc.io.WaveformArray(num, dtype=dtype)
            for key in data:
                self.table[key] = data[key]
            # add the compressed waveforms, if they exist
            self.has_compressed_waveforms = 'compressed_waveforms' in f
        else:
            raise ValueError("Unsupported template bank file extension %s" %(
                ext))

        # if approximant is specified, override whatever was in the file
        # (if anything was in the file)
        if approximant is not None:
            # get the approximant for each template
            apprxs = self.parse_approximant(approximant)
            if 'approximant' not in self.table.fieldnames:
                self.table = self.table.add_fields(apprxs, 'approximant')
            else:
                self.table['approximant'] = apprxs
        self.ensure_hash()

    @property
    def parameters(self):
        return self.table.fieldnames

    def ensure_hash(self):
        """Ensure that there is a correctly populated template_hash.

        Check for a correctly populated template_hash and create if it doesn't
        already exist.
        """
        fields = self.table.fieldnames
        if 'template_hash' in fields:
            return

        # The fields to use in making a template hash
        hash_fields = ['mass1', 'mass2', 'inclination',
                       'spin1x', 'spin1y', 'spin1z',
                       'spin2x', 'spin2y', 'spin2z',]

        fields = [f for f in hash_fields if f in fields]
        template_hash = np.array([tuple_to_hash(v) for v in zip(*[self.table[p]
                                  for p in fields])])
        if not np.unique(template_hash).size == template_hash.size:
            raise RuntimeError("Some template hashes clash. This should not "
                               "happen.")
        self.table = self.table.add_fields(template_hash, 'template_hash')

    def write_to_hdf(self, filename, start_index=None, stop_index=None,
                     force=False, skip_fields=None,
                     write_compressed_waveforms=True):
        """Writes self to the given hdf file.

            The name of the file to write to. Must end in '.hdf'.
        start_index : If a specific slice of the template bank is to be
            written to the hdf file, this would specify the index of the
            first template in the slice
        stop_index : If a specific slice of the template bank is to be
            last template in the slice
        force : {False, bool}
            If the file already exists, it will be overwritten if True.
            Otherwise, an OSError is raised if the file exists.
        skip_fields : {None, (list of) strings}
            Do not write the given fields to the hdf file. Default is None,
            in which case all fields in self.table.fieldnames are written.
        write_compressed_waveforms : {True, bool}
            Write compressed waveforms to the output (hdf) file if this is
            True, which is the default setting. If False, do not write the
            compressed waveforms group, but only the template parameters to
            the output file.

        Returns
        -------
        h5py.File
            The file handler to the output hdf file (left open).
        """
        if not filename.endswith('.hdf'):
            raise ValueError("Unrecoginized file extension")
        if os.path.exists(filename) and not force:
            raise IOError("File %s already exists" %(filename))
        f = h5py.File(filename, 'w')
        parameters = self.parameters
        if skip_fields is not None:
            if not isinstance(skip_fields, list):
                skip_fields = [skip_fields]
            parameters = [p for p in parameters if p not in skip_fields]
        # save the parameters
        f.attrs['parameters'] = parameters
        write_tbl = self.table[start_index:stop_index]
        for p in parameters:
            f[p] = write_tbl[p]
        if write_compressed_waveforms and self.has_compressed_waveforms:
            for tmplt_hash in write_tbl.template_hash:
                compressed_waveform = pycbc.waveform.compress.CompressedWaveform.from_hdf(
                                        self.filehandler, tmplt_hash,
                                        load_now=True)
                compressed_waveform.write_to_hdf(f, tmplt_hash)
        return f

    def end_frequency(self, index):
        """ Return the end frequency of the waveform at the given index value
        """
        from pycbc.waveform.waveform import props
        if hasattr(self.table[index], 'f_final'):
            return self.table[index].f_final

        return pycbc.waveform.get_waveform_end_frequency(
                                self.table[index],
                                approximant=self.approximant(index),
                                **self.extra_args)

    def parse_approximant(self, approximant):
        """Parses the given approximant argument, returning the approximant to
        use for each template in self. This is done by calling
        `parse_approximant_arg` using self's table as the array; see that
        function for more details."""
        return parse_approximant_arg(approximant, self.table)

    def approximant(self, index):
        """ Return the name of the approximant ot use at the given index
        """
        if 'approximant' not in self.table.fieldnames:
            raise ValueError("approximant not found in input file and no "
                "approximant was specified on initialization")
        apx = self.table["approximant"][index]
        if hasattr(apx, 'decode'):
            apx = apx.decode()
        return apx

        return len(self.table)

    def template_thinning(self, inj_filter_rejector):
        """Remove templates from bank that are far from all injections."""
        if not inj_filter_rejector.enabled or \
                inj_filter_rejector.chirp_time_window is None:
            return

        injection_parameters = inj_filter_rejector.injection_params.table
        fref = inj_filter_rejector.f_lower
        threshold = inj_filter_rejector.chirp_time_window
        m1= self.table['mass1']
        m2= self.table['mass2']
        tau0_temp, _ = pycbc.pnutils.mass1_mass2_to_tau0_tau3(m1, m2, fref)
        indices = []

        for inj in injection_parameters:
            tau0_inj, _ = \
                pycbc.pnutils.mass1_mass2_to_tau0_tau3(inj.mass1, inj.mass2,
                                                       fref)
            inj_indices = np.where(abs(tau0_temp - tau0_inj) <= threshold)[0]
            indices.append(inj_indices)
            indices_combined = np.concatenate(indices)

        indices_unique= np.unique(indices_combined)
        self.table = self.table[indices_unique]

    def ensure_standard_filter_columns(self, low_frequency_cutoff=None):
        """ Initialize FilterBank common fields

        low_frequency_cutoff: {float, None}, Optional
            A low frequency cutoff which overrides any given within the
            template bank file.
        """

        # Make sure we have a template duration field
        if not hasattr(self.table, 'template_duration'):
            self.table = self.table.add_fields(np.zeros(len(self.table),
                                     dtype=np.float32), 'template_duration')

        # Make sure we have a f_lower field
        if low_frequency_cutoff is not None:
            if not hasattr(self.table, 'f_lower'):
                vec = np.zeros(len(self.table), dtype=np.float32)
                self.table = self.table.add_fields(vec, 'f_lower')
            self.table['f_lower'][:] = low_frequency_cutoff

        self.min_f_lower = min(self.table['f_lower'])
        if self.f_lower is None and self.min_f_lower == 0.:
            raise ValueError('Invalid low-frequency cutoff settings')

class LiveFilterBank(TemplateBank):
    def __init__(self, filename, sample_rate, minimum_buffer,
                       approximant=None, increment=8, parameters=None,
                       low_frequency_cutoff=None,
                       **kwds):

        self.increment = increment
        self.minimum_buffer = minimum_buffer
        self.f_lower = low_frequency_cutoff

        super(LiveFilterBank, self).__init__(filename, approximant=approximant,
                parameters=parameters, **kwds)
        self.ensure_standard_filter_columns(low_frequency_cutoff=low_frequency_cutoff)
        self.param_lookup = {}
        for i, p in enumerate(self.table):
            key =  (p.mass1, p.mass2, p.spin1z, p.spin2z)
            assert(key not in self.param_lookup) # Uh, oh, template confusion!
            self.param_lookup[key] = i

    def round_up(self, num):
        """Determine the length to use for this waveform by rounding.

        num : int
            Proposed size of waveform in seconds

        Returns
        -------
        size: int
            The rounded size to use for the waveform buffer in seconds. This
        is calculaed using an internal `increment` attribute, which determines
        the discreteness of the rounding.
        """
        inc = self.increment
        size = np.ceil(num / self.sample_rate / inc) * self.sample_rate * inc
        return size

    def getslice(self, sindex):
        instance = copy(self)
        instance.table = self.table[sindex]
        return instance

    def id_from_param(self, param_tuple):
        """Get the index of this template based on its param tuple

        param_tuple : tuple
            Tuple of the parameters which uniquely identify this template

        Returns
        --------
        index : int
            The ordered index that this template has in the template bank.
        """
        return self.param_lookup[param_tuple]

            return self.getslice(index)

        return self.get_template(index)

    def get_template(self, index, min_buffer=None):

        approximant = self.approximant(index)
        f_end = self.end_frequency(index)
        flow = self.table[index].f_lower

        # Determine the length of time of the filter, rounded up to
        # nearest power of two
        if min_buffer is None:
            min_buffer =  self.minimum_buffer
        min_buffer += 0.5

        p = props(self.table[index])
        p.pop('approximant')
        buff_size = pycbc.waveform.get_waveform_filter_length_in_time(approximant, **p)

        tlen = self.round_up((buff_size + min_buffer) * self.sample_rate)

        delta_f = self.sample_rate / float(tlen)

        if f_end is None or f_end >= (flen * delta_f):
            f_end = (flen-1) * delta_f

        logging.info("Generating %s, %ss, %i, starting from %s Hz",
                     approximant, 1.0/delta_f, index, flow)

        # Get the waveform filter
        distance = 1.0 / DYN_RANGE_FAC
        htilde = pycbc.waveform.get_waveform_filter(
            zeros(flen, dtype=np.complex64), self.table[index],
            approximant=approximant, f_lower=flow, f_final=f_end,
            delta_f=delta_f, delta_t=1.0/self.sample_rate, distance=distance,

        # If available, record the total duration (which may
        # include ringdown) and the duration up to merger since they will be
        # erased by the type conversion below.
        ttotal = template_duration = -1
        time_offset = None
        if hasattr(htilde, 'length_in_time'):
            ttotal = htilde.length_in_time
        if hasattr(htilde, 'chirp_length'):
            template_duration = htilde.chirp_length
        if hasattr(htilde, 'time_offset'):
            time_offset = htilde.time_offset

        self.table[index].template_duration = template_duration

        htilde = htilde.astype(np.complex64)
        htilde.f_lower = flow
        htilde.min_f_lower = self.min_f_lower
        htilde.end_idx = int(f_end / htilde.delta_f)
        htilde.params = self.table[index]
        htilde.chirp_length = template_duration
        htilde.length_in_time = ttotal
        htilde.approximant = approximant
        htilde.end_frequency = f_end

        if time_offset:
            htilde.time_offset = time_offset

        # Add sigmasq as a method of this instance
        htilde.sigmasq = types.MethodType(sigma_cached, htilde)

        htilde.id = self.id_from_param((htilde.params.mass1,
                                        htilde.params.mass2,
                                        htilde.params.spin1z,
                                        htilde.params.spin2z))
        return htilde

class FilterBank(TemplateBank):
    def __init__(self, filename, filter_length, delta_f, dtype,
                 out=None, max_template_length=None,
                 approximant=None, parameters=None,
                 enable_compressed_waveforms=True,
                 low_frequency_cutoff=None,
                 waveform_decompression_method=None,
                 **kwds):
        self.out = out
        self.N = (filter_length - 1 ) * 2
        self.delta_t = 1.0 / (self.N * self.delta_f)
        self.filter_length = filter_length
        self.max_template_length = max_template_length
        self.enable_compressed_waveforms = enable_compressed_waveforms
        self.waveform_decompression_method = waveform_decompression_method

        super(FilterBank, self).__init__(filename, approximant=approximant,
            parameters=parameters, **kwds)

    def get_decompressed_waveform(self, tempout, index, f_lower=None,
                                  approximant=None, df=None):
        """Returns a frequency domain decompressed waveform for the template
        in the bank corresponding to the index taken in as an argument. The
        decompressed waveform is obtained by interpolating in frequency space,
        the amplitude and phase points for the compressed template that are
        read in from the bank."""

        from pycbc.waveform import get_waveform_filter_length_in_time

        # Get the template hash corresponding to the template index taken in as argument
        tmplt_hash = self.table.template_hash[index]

        # Read the compressed waveform from the bank file
        compressed_waveform = pycbc.waveform.compress.CompressedWaveform.from_hdf(
                                self.filehandler, tmplt_hash,
                                load_now=True)

        # Get the interpolation method to be used to decompress the waveform
        if self.waveform_decompression_method is not None :
            decompression_method = self.waveform_decompression_method
        else :
            decompression_method = compressed_waveform.interpolation
        logging.info("Decompressing waveform using %s", decompression_method)

        if df is not None :
            delta_f = df
        else :
            delta_f = self.delta_f

        # Create memory space for writing the decompressed waveform
        decomp_scratch = FrequencySeries(tempout[0:self.filter_length], delta_f=delta_f, copy=False)

        # Get the decompressed waveform
        hdecomp = compressed_waveform.decompress(out=decomp_scratch, f_lower=f_lower, interpolation=decompression_method)
        try:
            tmpltdur = self.table[index].template_duration
            tmpltdur = None
        if tmpltdur is None or tmpltdur==0.0 :
            tmpltdur = get_waveform_filter_length_in_time(approximant, **p)
        hdecomp.chirp_length = tmpltdur
        hdecomp.length_in_time = hdecomp.chirp_length
        return hdecomp

    def generate_with_delta_f_and_max_freq(self, t_num, max_freq, delta_f,
                                           low_frequency_cutoff=None,
                                           cached_mem=None):
        """Generate the template with index t_num using custom length."""
        approximant = self.approximant(t_num)
        # Don't want to use INTERP waveforms in here
        if approximant.endswith('_INTERP'):
            approximant = approximant.replace('_INTERP', '')
        # Using SPAtmplt here is bad as the stored cbrt and logv get
        # recalculated as we change delta_f values. Fall back to TaylorF2
        # in lalsimulation.
        if approximant == 'SPAtmplt':
            approximant = 'TaylorF2'
        if cached_mem is None:
            wav_len = int(max_freq / delta_f) + 1
            cached_mem = zeros(wav_len, dtype=np.complex64)
        if self.has_compressed_waveforms and self.enable_compressed_waveforms:
            htilde = self.get_decompressed_waveform(cached_mem, t_num,
                                                    f_lower=low_frequency_cutoff,
                                                    approximant=approximant,
                                                    df=delta_f)
        else :
            htilde = pycbc.waveform.get_waveform_filter(
                cached_mem, self.table[t_num], approximant=approximant,
                f_lower=low_frequency_cutoff, f_final=max_freq, delta_f=delta_f,
                distance=1./DYN_RANGE_FAC, delta_t=1./(2.*max_freq))

        # Make new memory for templates if we aren't given output memory
        if self.out is None:
            tempout = zeros(self.filter_length, dtype=self.dtype)
        else:
            tempout = self.out

        if f_end is None or f_end >= (self.filter_length * self.delta_f):
            f_end = (self.filter_length-1) * self.delta_f

        # Find the start frequency, if variable
        f_low = find_variable_start_frequency(approximant,
                                              self.table[index],
                                              self.f_lower,
                                              self.max_template_length)
        logging.info('%s: generating %s from %s Hz' % (index, approximant, f_low))

        # Clear the storage memory
        poke  = tempout.data # pylint:disable=unused-variable
        tempout.clear()

            htilde = self.get_decompressed_waveform(tempout, index, f_lower=f_low,
                                                    approximant=approximant, df=None)
        else :
                tempout[0:self.filter_length], self.table[index],
                approximant=approximant, f_lower=f_low, f_final=f_end,
                delta_f=self.delta_f, delta_t=self.delta_t, distance=distance,
                **self.extra_args)

        ttotal = template_duration = None

        htilde = htilde.astype(self.dtype)
        htilde.f_lower = f_low

        htilde._sigmasq = {}

def find_variable_start_frequency(approximant, parameters, f_start, max_length,
                                  delta_f = 1):
    """ Find a frequency value above the starting frequency that results in a
    waveform shorter than max_length.
    """
    if (f_start is None):
        f = parameters.f_lower
    elif (max_length is not None):
        l = max_length + 1
        f = f_start - delta_f
        while l > max_length:
            f += delta_f
            l = pycbc.waveform.get_waveform_filter_length_in_time(approximant,
                                                          parameters, f_lower=f)
    else :
        f = f_start
    return f

class FilterBankSkyMax(TemplateBank):
    def __init__(self, filename, filter_length, delta_f,
                 dtype, out_plus=None, out_cross=None,
                 max_template_length=None, parameters=None,
                 low_frequency_cutoff=None, **kwds):
        self.out_plus = out_plus
        self.out_cross = out_cross

        super(FilterBankSkyMax, self).__init__(filename, parameters=parameters,
              **kwds)

        if self.out_plus is None:
            tempoutplus = zeros(self.filter_length, dtype=self.dtype)
        else:
            tempoutplus = self.out_plus
        if self.out_cross is None:
            tempoutcross = zeros(self.filter_length, dtype=self.dtype)
        else:
            tempoutcross = self.out_cross

        # Get the end of the waveform if applicable (only for SPAtmplt atm)

        logging.info('%s: generating %s from %s Hz', index, approximant, f_low)

        # What does this do???
        poke1 = tempoutplus.data # pylint:disable=unused-variable
        poke2 = tempoutcross.data # pylint:disable=unused-variable

        tempoutplus.clear()
        tempoutcross.clear()

        hplus, hcross = pycbc.waveform.get_two_pol_waveform_filter(
            tempoutplus[0:self.filter_length],
            tempoutcross[0:self.filter_length], self.table[index],
            approximant=approximant, f_lower=f_low,
            f_final=f_end, delta_f=self.delta_f, delta_t=self.delta_t,
            distance=distance, **self.extra_args)

        if hasattr(hplus, 'chirp_length') and hplus.chirp_length is not None:
            self.table[index].template_duration = hplus.chirp_length

        hplus = hplus.astype(self.dtype)
        hcross = hcross.astype(self.dtype)
        hplus.f_lower = f_low
        hcross.f_lower = f_low
        hplus.min_f_lower = self.min_f_lower
        hcross.min_f_lower = self.min_f_lower
        hplus.end_frequency = f_end
        hcross.end_frequency = f_end
        hplus.end_idx = int(hplus.end_frequency / hplus.delta_f)
        hcross.end_idx = int(hplus.end_frequency / hplus.delta_f)
        hplus.params = self.table[index]
        hcross.params = self.table[index]
        hplus.approximant = approximant
        hcross.approximant = approximant

        hplus.sigmasq = types.MethodType(sigma_cached, hplus)
        hplus._sigmasq = {}
        hcross.sigmasq = types.MethodType(sigma_cached, hcross)
        hcross._sigmasq = {}

        return hplus, hcross

# Copyright (C) 2016  Alex Nitz, Collin Capano
#
#
#
#
#
#                                   Preamble
#
#
""" Utilities for handling frequency compressed an unequally spaced frequency
domain waveforms.
"""
import lalsimulation, lal, numpy, logging, h5py
from pycbc import pnutils, filter
from pycbc.types import FrequencySeries, zeros, complex_same_precision_as, real_same_precision_as
from pycbc.waveform import utils

def rough_time_estimate(m1, m2, flow, fudge_length=1.1, fudge_min=0.02):
    """ A very rough estimate of the duration of the waveform.

    An estimate of the waveform duration starting from flow. This is intended
    to be fast but not necessarily accurate. It should be an overestimate of
    the length. It is derived from a simplification of the 0PN post-newtonian
    terms and includes a fudge factor for possible ringdown, etc.

    m1: float
        mass of first component object in solar masses
    m2: float
        mass of second component object in solar masses
        starting frequency of the waveform
    fudge_length: optional, {1.1, float}
        Factor to multiply length estimate by to ensure it is a convservative
        value
    fudge_min: optional, {0.2, float}
        Minimum signal duration that can be returned. This should be long
        enough to encompass the ringdown and errors in the precise end time.

    Returns
    -------
    time: float
        Time from flow untill the end of the waveform
    """
    m = m1 + m2
    msun = m * lal.MTSUN_SI
    t =  5.0 / 256.0 * m * m * msun / (m1 * m2) / \
        (numpy.pi * msun * flow) **  (8.0 / 3.0)

    # fudge factoriness
    return .022 if t < 0 else (t + fudge_min) * fudge_length

def mchirp_compression(m1, m2, fmin, fmax, min_seglen=0.02, df_multiple=None):
    """Return the frequencies needed to compress a waveform with the given
    chirp mass. This is based on the estimate in rough_time_estimate.

    m1: float
    m2: float
        The starting frequency of the compressed waveform.
        The ending frequency of the compressed waveform.
    min_seglen : float
        The inverse of this gives the maximum frequency step that is used.
    df_multiple : {None, float}
        Make the compressed sampling frequencies a multiple of the given value.
        If None provided, the returned sample points can have any floating
        point value.

    Returns
    -------
    array
        The frequencies at which to evaluate the compressed waveform.
    """
    sample_points = []
    f = fmin
    while f < fmax:
        if df_multiple is not None:
            f = int(f/df_multiple)*df_multiple
        sample_points.append(f)
        f += 1.0 / rough_time_estimate(m1, m2, f, fudge_min=min_seglen)
    # add the last point
    if sample_points[-1] < fmax:
        sample_points.append(fmax)
    return numpy.array(sample_points)

def spa_compression(htilde, fmin, fmax, min_seglen=0.02,
        sample_frequencies=None):
    """Returns the frequencies needed to compress the given frequency domain
    waveform. This is done by estimating t(f) of the waveform using the
    stationary phase approximation.

        The waveform to compress.
    sample_frequencies : {None, array}
        The frequencies that the waveform is evaluated at. If None, will
        retrieve the frequencies from the waveform's sample_frequencies

    Returns
    -------
    array
    """
    if sample_frequencies is None:
        sample_frequencies = htilde.sample_frequencies.numpy()
    kmin = int(fmin/htilde.delta_f)
    kmax = int(fmax/htilde.delta_f)
    tf = abs(utils.time_from_frequencyseries(htilde,
            sample_frequencies=sample_frequencies).data[kmin:kmax])
    sample_frequencies = sample_frequencies[kmin:kmax]
    f = fmin
        f = int(f/htilde.delta_f)*htilde.delta_f
        jj = numpy.searchsorted(sample_frequencies, f)
        f += 1./(tf[jj:].max()+min_seglen)

compression_algorithms = {
        'mchirp': mchirp_compression,
        'spa': spa_compression
        }

def _vecdiff(htilde, hinterp, fmin, fmax, psd=None):
    return abs(filter.overlap_cplx(htilde, htilde,
                          low_frequency_cutoff=fmin,
                          high_frequency_cutoff=fmax,
                          normalized=False, psd=psd)
                - filter.overlap_cplx(htilde, hinterp,
                          normalized=False, psd=psd))

def vecdiff(htilde, hinterp, sample_points, psd=None):
    """Computes a statistic indicating between which sample points a waveform
    and the interpolated waveform differ the most.
    """
    vecdiffs = numpy.zeros(sample_points.size-1, dtype=float)
    for kk,thisf in enumerate(sample_points[:-1]):
        nextf = sample_points[kk+1]
        vecdiffs[kk] = abs(_vecdiff(htilde, hinterp, thisf, nextf, psd=psd))
    return vecdiffs

def compress_waveform(htilde, sample_points, tolerance, interpolation,
                      precision, decomp_scratch=None, psd=None):
    """Retrieves the amplitude and phase at the desired sample points, and adds
    frequency points in order to ensure that the interpolated waveform
    has a mismatch with the full waveform that is <= the desired tolerance. The
    mismatch is computed by finding 1-overlap between `htilde` and the
    decompressed waveform; no maximimization over phase/time is done, a
    PSD may be used.

    .. note::
        The decompressed waveform is only garaunteed to have a true mismatch
        <= the tolerance for the given `interpolation` and for no PSD.
        However, since no maximization over time/phase is performed when
        adding points, the actual mismatch between the decompressed waveform
        and `htilde` is better than the tolerance, using no PSD. Using a PSD
        does increase the mismatch, and can lead to mismatches > than the
        desired tolerance, but typically by only a factor of a few worse.

    sample_points : array
        The frequencies at which to store the amplitude and phase. More points
        may be added to this, depending on the desired tolerance.
    tolerance : float
        The maximum mismatch to allow between a decompressed waveform and
        `htilde`.
    interpolation : str
        The interpolation to use for decompressing the waveform when computing
        overlaps.
    precision : str
        The precision being used to generate and store the compressed waveform
        points.
    decomp_scratch : {None, FrequencySeries}
        Optionally provide scratch space for decompressing the waveform. The
        provided frequency series must have the same `delta_f` and length
        as `htilde`.
    psd : {None, FrequencySeries}
        The psd to use for calculating the overlap between the decompressed
        waveform and the original full waveform.

    Returns
    -------
    CompressedWaveform
        The compressed waveform data; see `CompressedWaveform` for details.
    """
    fmin = sample_points.min()
    df = htilde.delta_f
    sample_index = (sample_points / df).astype(int)
    amp = utils.amplitude_from_frequencyseries(htilde)
    phase = utils.phase_from_frequencyseries(htilde)

    comp_amp = amp.take(sample_index)
    comp_phase = phase.take(sample_index)
    if decomp_scratch is None:
        outdf = df
    else:
        outdf = None
    hdecomp = fd_decompress(comp_amp, comp_phase, sample_points,
                            out=decomp_scratch, df=outdf, f_lower=fmin,
                            interpolation=interpolation)
    kmax = min(len(htilde), len(hdecomp))
    htilde = htilde[:kmax]
    hdecomp = hdecomp[:kmax]
    mismatch = 1. - filter.overlap(hdecomp, htilde, psd=psd,
                                   low_frequency_cutoff=fmin)
    if mismatch > tolerance:
        # we'll need the difference in the waveforms as a function of frequency
        vecdiffs = vecdiff(htilde, hdecomp, sample_points, psd=psd)

    # We will find where in the frequency series the interpolated waveform
    # has the smallest overlap with the full waveform, add a sample point
    # there, and re-interpolate. We repeat this until the overall mismatch
    # is > than the desired tolerance
    added_points = []
    while mismatch > tolerance:
        minpt = vecdiffs.argmax()
        # add a point at the frequency halfway between minpt and minpt+1
        add_freq = sample_points[[minpt, minpt+1]].mean()
        addidx = int(round(add_freq/df))
        # ensure that only new points are added
        if addidx in sample_index:
            diffidx = vecdiffs.argsort()
            addpt = -1
            while addidx in sample_index:
                addpt -= 1
                try:
                    minpt = diffidx[addpt]
                except IndexError:
                    raise ValueError("unable to compress to desired tolerance")
                add_freq = sample_points[[minpt, minpt+1]].mean()
                addidx = int(round(add_freq/df))
        new_index = numpy.zeros(sample_index.size+1, dtype=int)
        new_index[:minpt+1] = sample_index[:minpt+1]
        new_index[minpt+1] = addidx
        new_index[minpt+2:] = sample_index[minpt+1:]
        sample_index = new_index
        sample_points = (sample_index * df).astype(
            real_same_precision_as(htilde))
        # get the new compressed points
        comp_amp = amp.take(sample_index)
        comp_phase = phase.take(sample_index)
        # update the vecdiffs and mismatch
        hdecomp = fd_decompress(comp_amp, comp_phase, sample_points,
                                out=decomp_scratch, df=outdf,
                                f_lower=fmin, interpolation=interpolation)
        hdecomp = hdecomp[:kmax]
        new_vecdiffs = numpy.zeros(vecdiffs.size+1)
        new_vecdiffs[:minpt] = vecdiffs[:minpt]
        new_vecdiffs[minpt+2:] = vecdiffs[minpt+1:]
        new_vecdiffs[minpt:minpt+2] = vecdiff(htilde, hdecomp,
                                              sample_points[minpt:minpt+2],
                                              psd=psd)
        vecdiffs = new_vecdiffs
        mismatch = 1. - filter.overlap(hdecomp, htilde, psd=psd,
                                       low_frequency_cutoff=fmin)
        added_points.append(addidx)
    logging.info("mismatch: %f, N points: %i (%i added)" %(mismatch,
                 len(comp_amp), len(added_points)))

    return CompressedWaveform(sample_points, comp_amp, comp_phase,
                              interpolation=interpolation,
                              tolerance=tolerance, mismatch=mismatch,
                              precision=precision)

_precision_map = {
    'float32': 'single',
    'float64': 'double',
    'complex64': 'single',
    'complex128': 'double'
}

_complex_dtypes = {
    'single': numpy.complex64,
    'double': numpy.complex128
}

_real_dtypes = {
    'single': numpy.float32,
    'double': numpy.float64
}

@schemed("pycbc.waveform.decompress_")
def inline_linear_interp(amp, phase, sample_frequencies, output,
                         df, f_lower, imin, start_index):
    """Generate a frequency-domain waveform via linear interpolation
    from sampled amplitude and phase. The sample frequency locations
    for the amplitude and phase must be the same. This function may
    be less accurate than scipy's linear interpolation, but should be
    much faster.  Additionally, it is 'schemed' and so may run under
    either CPU or GPU schemes.

    This function is not ordinarily called directly, but rather by
    giving the argument 'interpolation' the value 'inline_linear'
    when calling the function 'fd_decompress' below.

    amp : array
        The amplitude of the waveform at the sample frequencies.
    phase : array
        The phase of the waveform at the sample frequencies.
    sample_frequencies : array
        The frequency (in Hz) of the waveform at the sample frequencies.
    output : {None, FrequencySeries}
        The output array to save the decompressed waveform to. If this contains
        slots for frequencies > the maximum frequency in sample_frequencies,
        the rest of the values are zeroed. If not provided, must provide a df.
    df : {None, float}
        The frequency step to use for the decompressed waveform. Must be
        provided if out is None.
    f_lower : float
        The frequency to start the decompression at. All values at
        frequencies less than this will be 0 in the decompressed waveform.
    imin : int
        The index at which to start in the sampled frequency series. Must
        therefore be 0 <= imin < len(sample_frequencies)
    start_index : int
        The index at which to start in the output frequency;
        i.e., ceil(f_lower/df).

    Returns
    -------
    output : FrequencySeries
        If out was provided, writes to that array. Otherwise, a new
        FrequencySeries with the decompressed waveform.

    """
    return

def fd_decompress(amp, phase, sample_frequencies, out=None, df=None,
                  f_lower=None, interpolation='inline_linear'):
    """Decompresses an FD waveform using the given amplitude, phase, and the
    frequencies at which they are sampled at.

    amp : array
    out : {None, FrequencySeries}
    f_lower : {None, float}
        The frequency to start the decompression at. If None, will use whatever
        the lowest frequency is in sample_frequencies. All values at
    interpolation : {'inline_linear', str}
        The interpolation to use for the amplitude and phase. Default is
        'inline_linear'. If 'inline_linear' a custom interpolater is used.
        Otherwise, ``scipy.interpolate.interp1d`` is used; for other options,
        see possible values for that function's ``kind`` argument.

    Returns
    -------
    out : FrequencySeries
    """
    precision = _precision_map[sample_frequencies.dtype.name]
    if _precision_map[amp.dtype.name] != precision or \
            _precision_map[phase.dtype.name] != precision:
        raise ValueError("amp, phase, and sample_points must all have the "
            "same precision")

        if df is None:
            raise ValueError("Either provide output memory or a df")
        hlen = int(numpy.ceil(sample_frequencies.max()/df+1))
        out = FrequencySeries(numpy.zeros(hlen,
            dtype=_complex_dtypes[precision]), copy=False,
            delta_f=df)
    else:
        # check for precision compatibility
        if out.precision == 'double' and precision == 'single':
            raise ValueError("cannot cast single precision to double")
        df = out.delta_f
        hlen = len(out)
    if f_lower is None:
        imin = 0 # pylint:disable=unused-variable
        f_lower = sample_frequencies[0]
        start_index = 0
    else:
        if f_lower >= sample_frequencies.max():
            raise ValueError("f_lower is > than the maximum sample frequency")
        if f_lower < sample_frequencies.min():
            raise ValueError("f_lower is < than the minimum sample frequency")
        imin = int(numpy.searchsorted(sample_frequencies, f_lower,
            side='right')) - 1 # pylint:disable=unused-variable
        start_index = int(numpy.ceil(f_lower/df))
    if start_index >= hlen:
        raise ValueError('requested f_lower >= largest frequency in out')
    # interpolate the amplitude and the phase
    if interpolation == "inline_linear":
        # Call the scheme-dependent function
        inline_linear_interp(amp, phase, sample_frequencies, out,
                             df, f_lower, imin, start_index)
    else:
        # use scipy for fancier interpolation
        sample_frequencies = numpy.array(sample_frequencies)
        amp = numpy.array(amp)
        phase = numpy.array(phase)
        outfreq = out.sample_frequencies.numpy()
        amp_interp = interpolate.interp1d(sample_frequencies, amp,
                                          kind=interpolation,
                                          bounds_error=False,
                                          fill_value=0.,
                                          assume_sorted=True)
        phase_interp = interpolate.interp1d(sample_frequencies, phase,
                                            kind=interpolation,
                                            bounds_error=False,
                                            fill_value=0.,
                                            assume_sorted=True)
        A = amp_interp(outfreq)
        phi = phase_interp(outfreq)
        out.data[:] = A*numpy.cos(phi) + (1j)*A*numpy.sin(phi)
    return out

class CompressedWaveform(object):
    """Class that stores information about a compressed waveform.

    sample_points : {array, h5py.Dataset}
        The frequency points at which the compressed waveform is sampled.
    amplitude : {array, h5py.Dataset}
        The amplitude of the waveform at the given `sample_points`.
    phase : {array, h5py.Dataset}
        The phase of the waveform at the given `sample_points`.
    interpolation : {None, str}
        The interpolation that was used when compressing the waveform for
        computing tolerance. This is also the default interpolation used when
        decompressing; see `decompress` for details.
    tolerance : {None, float}
        The tolerance that was used when compressing the waveform.
    mismatch : {None, float}
        The actual mismatch between the decompressed waveform (using the given
        `interpolation`) and the full waveform.
    precision : {'double', str}
        The precision used to generate the compressed waveform's amplitude and
        phase points. Default is 'double'.
    load_to_memory : {True, bool}
        If `sample_points`, `amplitude`, and/or `phase` is an hdf dataset, they
        will be cached in memory the first time they are accessed. Default is
        True.

    sample_points
    amplitude
    phase
    load_to_memory : bool
        Whether or not to load `sample_points`/`amplitude`/`phase` into memory
        the first time they are accessed, if they are hdf datasets. Can be
        set directly to toggle this behavior.
        The interpolation that was used when compressing the waveform, for
        checking the mismatch. Also the default interpolation used when
        decompressing.
        The mismatch between the decompressed waveform and the original
        waveform.
        The precision used to generate and store the compressed waveform
        points. Options are 'double' or 'single'; default is 'double
    """

    def __init__(self, sample_points, amplitude, phase,
                 interpolation=None, tolerance=None, mismatch=None,
                 precision='double', load_to_memory=True):
        self._sample_points = sample_points
        self._amplitude = amplitude
        self._phase = phase
        self._cache = {}
        self.load_to_memory = load_to_memory
        # if sample points, amplitude, and/or phase are hdf datasets,
        # save their filenames
        self._filenames = {}
        self._groupnames = {}
        for arrname in ['sample_points', 'amplitude', 'phase']:
            try:
                fname = getattr(self, '_{}'.format(arrname)).file.filename
                gname = getattr(self, '_{}'.format(arrname)).name
                fname = None
                gname = None
            self._filenames[arrname] = fname
            self._groupnames[arrname] = gname
        # metadata
        self.interpolation = interpolation
        self.tolerance = tolerance
        self.mismatch = mismatch
        self.precision = precision

    def _get(self, param):
        val = getattr(self, '_%s' %param)
        if isinstance(val, h5py.Dataset):
            try:
                val = self._cache[param]
                try:
                    val = val[:]
                    # this can happen if the file is closed; if so, open it
                    # and get the data
                    fp = h5py.File(self._filenames[param], 'r')
                    val = fp[self._groupnames[param]][:]
                    fp.close()
                if self.load_to_memory:
                    self._cache[param] = val
        return val

    @property
    def amplitude(self):
        """The amplitude of the waveform at the `sample_points`.

        This is always returned as an array; the same logic as for
        `sample_points` is used to determine whether or not to cache in
        memory.

        Returns
        -------
        amplitude : Array
        """
        return self._get('amplitude')

    @property
    def phase(self):
        """The phase of the waveform as the `sample_points`.

        `sample_points` returned as an array; the same logic as for
        memory.

        Returns
        -------
        phase : Array
        """
        return self._get('phase')

    @property
    def sample_points(self):
        """The frequencies at which the compressed waveform is sampled.

        This is
        always returned as an array, even if the stored `sample_points` is an
        hdf dataset. If `load_to_memory` is True and the stored points are
        an hdf dataset, the `sample_points` will cached in memory the first
        time this attribute is accessed.

        Returns
        -------
        sample_points : Array
        """
        return self._get('sample_points')

    def clear_cache(self):
        """Clear self's cache of amplitude, phase, and sample_points."""
        self._cache.clear()

    def decompress(self, out=None, df=None, f_lower=None, interpolation=None):
        """Decompress self.

        out : {None, FrequencySeries}
            Write the decompressed waveform to the given frequency series. The
            decompressed waveform will have the same `delta_f` as `out`.
            Either this or `df` must be provided.
        df : {None, float}
            Decompress the waveform such that its `delta_f` has the given
            value. Either this or `out` must be provided.
        f_lower : {None, float}
            The starting frequency at which to decompress the waveform. Cannot
            be less than the minimum frequency in `sample_points`. If `None`
            provided, will default to the minimum frequency in `sample_points`.
        interpolation : {None, str}
            The interpolation to use for decompressing the waveform. If `None`
            provided, will default to `self.interpolation`.

        Returns
        -------
        FrequencySeries
            The decompressed waveform.
        """
            # use the minimum of the samlpe points
            f_lower = self.sample_points.min()
        if interpolation is None:
            interpolation = self.interpolation
        return fd_decompress(self.amplitude, self.phase, self.sample_points,
                             out=out, df=df, f_lower=f_lower,
                             interpolation=interpolation)

    def write_to_hdf(self, fp, template_hash, root=None, precision=None):
        """Write the compressed waveform to the given hdf file handler.

        The waveform is written to:
        `fp['[{root}/]compressed_waveforms/{template_hash}/{param}']`,
        where `param` is the `sample_points`, `amplitude`, and `phase`. The
        `interpolation`, `tolerance`, `mismatch` and `precision` are saved
        to the group's attributes.

        fp : h5py.File
            An open hdf file to write the compressed waveform to.
        template_hash : {hash, int, str}
            A hash, int, or string to map the template to the waveform.
        root : {None, str}
            Put the `compressed_waveforms` group in the given directory in the
            hdf file. If `None`, `compressed_waveforms` will be the root
            directory.
        precision : {None, str}
            Cast the saved parameters to the given precision before saving. If
            None provided, will use whatever their current precision is. This
            will raise an error if the parameters have single precision but the
            requested precision is double.
        """
        if root is None:
            root = ''
        else:
            root = '%s/'%(root)
        if precision is None:
            precision = self.precision
        elif precision == 'double' and self.precision == 'single':
        outdtype = _real_dtypes[precision]
        group = '%scompressed_waveforms/%s' %(root, str(template_hash))
        for param in ['amplitude', 'phase', 'sample_points']:
            fp['%s/%s' %(group, param)] = self._get(param).astype(outdtype)
        fp_group = fp[group]
        fp_group.attrs['mismatch'] = self.mismatch
        fp_group.attrs['interpolation'] = self.interpolation
        fp_group.attrs['tolerance'] = self.tolerance
        fp_group.attrs['precision'] = precision

    def from_hdf(cls, fp, template_hash, root=None, load_to_memory=True,
                 load_now=False):
        """Load a compressed waveform from the given hdf file handler.

        The waveform is retrieved from:
        where `param` is the `sample_points`, `amplitude`, and `phase`.

            The id of the waveform.
            Retrieve the `compressed_waveforms` group from the given string.
            If `None`, `compressed_waveforms` will be assumed to be in the
            top level.
        load_to_memory : {True, bool}
            Set the `load_to_memory` attribute to the given value in the
            returned instance.
        load_now : {False, bool}
            Immediately load the `sample_points`/`amplitude`/`phase` to memory.

        Returns
        -------
        CompressedWaveform
            An instance of this class with parameters loaded from the hdf file.
        """
            root = ''
        else:
        sample_points = fp_group['sample_points']
        amp = fp_group['amplitude']
        phase = fp_group['phase']
        if load_now:
            sample_points = sample_points[:]
            amp = amp[:]
            phase = phase[:]
        return cls(sample_points, amp, phase,
            interpolation=fp_group.attrs['interpolation'],
            tolerance=fp_group.attrs['tolerance'],
            mismatch=fp_group.attrs['mismatch'],
            precision=fp_group.attrs['precision'],
            load_to_memory=load_to_memory)

#
#
#
#
#
#                                   Preamble
#
#
"""
from ..types import real_same_precision_as
from ..types import complex_same_precision_as
from .decompress_cpu_cython import decomp_ccode_double, decomp_ccode_float

    rprec = real_same_precision_as(output)
    cprec = complex_same_precision_as(output)
    sample_frequencies = numpy.array(sample_frequencies, copy=False,
                                     dtype=rprec)
    amp = numpy.array(amp, copy=False, dtype=rprec)
    phase = numpy.array(phase, copy=False, dtype=rprec)
    sflen = len(sample_frequencies)
    h = numpy.array(output.data, copy=False, dtype=cprec)
    hlen = len(output)
    delta_f = float(df)
    if output.precision == 'single':
        decomp_ccode_float(h, delta_f, hlen, start_index, sample_frequencies,
                           amp, phase, sflen, imin)
    else:
        decomp_ccode_double(h, delta_f, hlen, start_index, sample_frequencies,
                            amp, phase, sflen, imin)

# Copyright (C) 2016  Collin Capano, Alex Nitz, Christopher Biwer
#
#

#
#
#                                   Preamble
#
#
"""
This modules provides classes for generating waveforms.
"""
import os

from . import waveform
from .waveform import (NoWaveformError, FailedWaveformError)
from . import ringdown
from . import supernovae
from . import waveform_modes
from pycbc import filter
from pycbc.waveform.utils import apply_fd_time_shift, taper_timeseries, \
                                 ceilpow2

# utility functions/class
failed_counter = 0

class BaseGenerator(object):
    """A wrapper class to call a waveform generator with a set of frozen
    parameters and a set of variable parameters. The frozen parameters and
    values, along with a list of variable parameter names, are set at
    initialization. This way, repeated calls can be made to the underlying
    generator by simply passing a list of values for the variable parameters
    to this class's generate function.

    generator : function
        The function that is called for waveform generation.
    variable_args : {(), list}
        A tuple or list of strings giving the names and order of variable
        parameters that will be passed to the waveform generator when the
        generate function is called.
    record_failures : boolean
        Store output files containing the parameters of failed waveform
        generation. Default is False.
    \**frozen_params :
        These keyword arguments are the ones that will be frozen in the
        waveform generator. For a list of possible parameters, see
        pycbc.waveform.cbc_parameters.

        The list of names of variable arguments.
    frozen_params : dict
        A dictionary of the frozen keyword arguments that are always passed
        to the waveform generator function.
    current_params : dict
        A dictionary of the frozen keyword arguments and variable arguments
        that were last passed to the waveform generator.

    Methods
    -------
    generate(variable_values)
        Generates a waveform using the variable arguments and the frozen
        arguments.
    """
    def __init__(self, generator, variable_args=(), record_failures=False,
                 **frozen_params):
        self.generator = generator
        self.frozen_params = frozen_params
        # we'll keep a dictionary of the current parameters for fast
        # generation
        self.current_params = frozen_params.copy()
        # keep a list of functions to call before waveform generation
        self._pregenerate_functions = []

        # If we are under mpi, then failed waveform will be stored by
        # mpi rank to avoid file writing conflicts. We'll check for this
        # upfront
        self.record_failures = (record_failures or
                                ('PYCBC_RECORD_FAILED_WAVEFORMS' in os.environ))
        self.mpi_enabled, _, self.mpi_rank = use_mpi()

    @property
    def static_args(self):
        """Returns a dictionary of the static arguments."""
        return self.frozen_params

    def generate(self, **kwargs):
        """Generates a waveform from the keyword args. The current params
        are updated with the given kwargs, then the generator is called.
        """
        self.current_params.update(kwargs)
        return self._generate_from_current()

    def _add_pregenerate(self, func):
        """ Adds a function that will be called by the generator function
        before waveform generation.
        """
        self._pregenerate_functions.append(func)

    def _postgenerate(self, res):
        """Allows the waveform returned by the generator function to be
        manipulated before returning.
        """
        return res

    def _gdecorator(generate_func):
        """A decorator that allows for seemless pre/post manipulation of
        the waveform generator function.
        """
        def dostuff(self):
            for func in self._pregenerate_functions:
                self.current_params = func(self.current_params)
            res = generate_func(self) # pylint:disable=not-callable
            return self._postgenerate(res)
        return dostuff

    @_gdecorator
    def _generate_from_current(self):
        """Generates a waveform from the current parameters.
        """
        try:
            new_waveform = self.generator(**self.current_params)
            return new_waveform
            if self.record_failures:
                import h5py
                from pycbc.io.hdf import dump_state

                global failed_counter

                if self.mpi_enabled:
                    outname = 'failed/params_%s.hdf' % self.mpi_rank
                else:
                    outname = 'failed/params.hdf'

                if not os.path.exists('failed'):
                    os.makedirs('failed')

                with h5py.File(outname) as f:
                    dump_state(self.current_params, f,
                               dsetname=str(failed_counter))
                    failed_counter += 1

            # we'll get a RuntimeError if lalsimulation failed to generate
            # the waveform for whatever reason
            strparams = ' | '.join(['{}: {}'.format(
                p, str(val)) for p, val in self.current_params.items()])
            raise FailedWaveformError("Failed to generate waveform with "
                                      "parameters:\n{}\nError was: {}"
                                      .format(strparams, e))

class BaseCBCGenerator(BaseGenerator):
    """Adds ability to convert from various derived parameters to parameters
    needed by the waveform generators.

    possible_args : set
        The set of names of arguments that may be used in the `variable_args`
        or `frozen_params`.
    """
    possible_args = set(parameters.td_waveform_params +
                        parameters.fd_waveform_params +
                        ['taper'])
    def __init__(self, generator, variable_args=(), **frozen_params):
        super(BaseCBCGenerator, self).__init__(generator,
            variable_args=variable_args, **frozen_params)
        # decorate the generator function with a list of functions that convert
        # parameters to those used by the waveform generation interface
        all_args = set(list(self.frozen_params.keys()) +
                       list(self.variable_args))
        # compare a set of all args of the generator to the input parameters
        # of the functions that do conversions and adds to list of pregenerate
        # functions if it is needed
        params_used, cs = transforms.get_common_cbc_transforms(
                                       list(self.possible_args), variable_args)
        for c in cs:
            self._add_pregenerate(c)
        # check that there are no unused (non-calibration) parameters
        calib_args = set([a for a in self.variable_args if
                          a.startswith('calib_')])
        all_args = all_args - calib_args
        unused_args = all_args.difference(params_used) \
                              .difference(self.possible_args)
        if len(unused_args):
            logging.warning("WARNING: The following args are not being used "
                            "for waveform generation: %s",
                            ', '.join(unused_args))

class FDomainCBCGenerator(BaseCBCGenerator):
    """Generates frequency-domain CBC waveforms in the radiation frame.

    Uses `waveform.get_fd_waveform` as a generator function to create
    frequency- domain CBC waveforms in the radiation frame; i.e., with no
    detector response function applied. For more details, see `BaseGenerator`.

    Derived parameters not understood by `get_fd_waveform` may be used as
    variable args and/or frozen parameters, as long as they can be converted
    into parameters that `get_fd_waveform` can use. For example, `mchirp` and
    `eta` (currently, the only supported derived parameters) may be used as
    variable/frozen params; these are converted to `mass1` and `mass2` prior to
    calling the waveform generator function.

    Examples
    --------
    Initialize a generator:

    >>> from pycbc.waveform.generator import FDomainCBCGenerator
    >>> generator = FDomainCBCGenerator(variable_args=['mass1', 'mass2'], delta_f=1./32, f_lower=30., approximant='TaylorF2')

    Create a waveform with the variable arguments (in this case, mass1, mass2):

    >>> generator.generate(mass1=1.4, mass2=1.4)
        (<pycbc.types.frequencyseries.FrequencySeries at 0x1110c1450>,
         <pycbc.types.frequencyseries.FrequencySeries at 0x1110c1510>)

    Initialize a generator using mchirp, eta as the variable args, and generate
    a waveform:

    >>> generator = FDomainCBCGenerator(variable_args=['mchirp', 'eta'], delta_f=1./32, f_lower=30., approximant='TaylorF2')
    >>> generator.generate(mchirp=1.5, eta=0.25)
        (<pycbc.types.frequencyseries.FrequencySeries at 0x109a104d0>,
         <pycbc.types.frequencyseries.FrequencySeries at 0x109a10b50>)

    Note that the `current_params` contains the mchirp and eta values, along
    with the mass1 and mass2 they were converted to:

    >>> generator.current_params
        {'approximant': 'TaylorF2',
         'delta_f': 0.03125,
         'eta': 0.25,
         'f_lower': 30.0,
         'mass1': 1.7230475324955525,
         'mass2': 1.7230475324955525,
         'mchirp': 1.5}

    """
    def __init__(self, variable_args=(), **frozen_params):
        super(FDomainCBCGenerator, self).__init__(waveform.get_fd_waveform,

class FDomainCBCModesGenerator(BaseCBCGenerator):
    """Generates frequency-domain CBC waveform modes.

    Uses :py:func:`waveform_modes.get_fd_waveform_modes` as a generator
    function to create frequency-domain CBC waveforms mode-by-mode, without
    applying spherical harmonics.

    For details, on methods and arguments, see :py:class:`BaseGenerator`.
    """
        super(FDomainCBCModesGenerator, self).__init__(
            waveform_modes.get_fd_waveform_modes,

class TDomainCBCGenerator(BaseCBCGenerator):
    """Create time domain CBC waveforms in the radiation frame.

    Uses waveform.get_td_waveform as a generator function to create time-
    domain CBC waveforms in the radiation frame; i.e., with no detector
    response function applied. For more details, see `BaseGenerator`.

    Derived parameters not understood by `get_td_waveform` may be used as
    into parameters that `get_td_waveform` can use. For example, `mchirp` and

    Examples
    --------

    >>> from pycbc.waveform.generator import TDomainCBCGenerator
    >>> generator = TDomainCBCGenerator(variable_args=['mass1', 'mass2'], delta_t=1./4096, f_lower=30., approximant='TaylorT4')

    >>> generator.generate(mass1=2., mass2=1.3)
        (<pycbc.types.timeseries.TimeSeries at 0x10e546710>,
         <pycbc.types.timeseries.TimeSeries at 0x115f37690>)

    >>> generator = TDomainCBCGenerator(variable_args=['mchirp', 'eta'], delta_t=1./4096, f_lower=30., approximant='TaylorT4')
    >>> generator.generate(mchirp=1.75, eta=0.2)
        (<pycbc.types.timeseries.TimeSeries at 0x116ac6050>,
         <pycbc.types.timeseries.TimeSeries at 0x116ac6950>)

    """
        super(TDomainCBCGenerator, self).__init__(waveform.get_td_waveform,

        """Applies a taper if it is in current params.
        """
        hp, hc = res
        try:
            hp = taper_timeseries(hp, tapermethod=self.current_params['taper'])
            hc = taper_timeseries(hc, tapermethod=self.current_params['taper'])
            pass
        return hp, hc

class TDomainCBCModesGenerator(BaseCBCGenerator):
    """Generates time domain CBC waveform modes.

    Uses :py:func:`waveform_modes.get_td_waveform_modes` as a generator
    function to create time-domain CBC waveforms mode-by-mode, without applying
    spherical harmonics. The ``generate`` function returns a dictionary of
    modes -> (real, imag) part of the complex time series.

    """
        super(TDomainCBCModesGenerator, self).__init__(
            waveform_modes.get_td_waveform_modes,

        """
        if 'taper' in self.current_params:
            tapermethod = self.current_params['taper']
            for mode in res:
                ulm, vlm = res[mode]
                ulm = taper_timeseries(ulm, tapermethod=tapermethod)
                vlm = taper_timeseries(vlm, tapermethod=tapermethod)
                res[mode] = (ulm, vlm)
        return res

class FDomainMassSpinRingdownGenerator(BaseGenerator):
    """Uses ringdown.get_fd_from_final_mass_spin as a generator function to
    create frequency-domain ringdown waveforms with higher modes in the
    radiation frame; i.e., with no detector response function applied.
    For more details, see BaseGenerator.

    Examples
    --------

    >>> from pycbc.waveform.generator import FDomainMassSpinRingdownGenerator
    >>> generator = FDomainMassSpinRingdownGenerator(variable_args=['final_mass',
                    'final_spin','amp220','amp210','phi220','phi210'], lmns=['221','211'],
                    delta_f=1./32, f_lower=30., f_final=500)

    Create a ringdown with the variable arguments:

    >>> generator.generate(final_mass=65., final_spin=0.7,
                           amp220=1e-21, amp210=1./10, phi220=0., phi210=0.)
        (<pycbc.types.frequencyseries.FrequencySeries at 0x51614d0>,
         <pycbc.types.frequencyseries.FrequencySeries at 0x5161550>)

    """
        super(FDomainMassSpinRingdownGenerator, self).__init__(ringdown.get_fd_from_final_mass_spin,

class FDomainFreqTauRingdownGenerator(BaseGenerator):
    """Uses ringdown.get_fd_from_freqtau as a generator function to

    Examples
    --------

    >>> from pycbc.waveform.generator import FDomainFreqTauRingdownGenerator
    >>> generator = FDomainFreqTauRingdownGenerator(variable_args=['f_220',
                    'tau_220','f_210','tau_210','amp220','amp210','phi220','phi210'],
                    lmns=['221','211'], delta_f=1./32, f_lower=30., f_final=500)

    >>> generator.generate(f_220=317., tau_220=0.003, f_210=274., tau_210=0.003,

    """
        super(FDomainFreqTauRingdownGenerator, self).__init__(ringdown.get_fd_from_freqtau,

class TDomainMassSpinRingdownGenerator(BaseGenerator):
    """Uses ringdown.get_td_from_final_mass_spin as a generator function to
    create time-domain ringdown waveforms with higher modes in the

    Examples
    --------

    >>> from pycbc.waveform.generator import TDomainMassSpinRingdownGenerator
    >>> generator = TDomainMassSpinRingdownGenerator(variable_args=['final_mass',
                    delta_t=1./2048)

    """
        super(TDomainMassSpinRingdownGenerator, self).__init__(ringdown.get_td_from_final_mass_spin,

class TDomainFreqTauRingdownGenerator(BaseGenerator):
    """Uses ringdown.get_td_from_freqtau as a generator function to

    Examples
    --------

    >>> generator = TDomainFreqTauRingdownGenerator(variable_args=['f_220',
                    lmns=['221','211'], delta_t=1./2048)

    """
        super(TDomainFreqTauRingdownGenerator, self).__init__(ringdown.get_td_from_freqtau,

class TDomainSupernovaeGenerator(BaseGenerator):
    """Uses supernovae.py to create time domain core-collapse supernovae waveforms
    using a set of Principal Components provided in a .hdf file.
    """
        super(TDomainSupernovaeGenerator,
              self).__init__(supernovae.get_corecollapse_bounce,
           variable_args=variable_args, **frozen_params)

#
#
#                        Detector-frame generators
#
#

class BaseFDomainDetFrameGenerator(object):
    """Base generator for frquency-domain waveforms in a detector frame.

    rFrameGeneratorClass : class
        The class to use for generating the waveform in the radiation frame,
        e.g., FDomainCBCGenerator. This should be the class, not an
        instance of the class (the class will be initialized with the
        appropriate arguments internally).
    detectors : {None, list of strings}
        The names of the detectors to use. If provided, all location parameters
        must be included in either the variable args or the frozen params. If
        None, the generate function will just return the plus polarization
        returned by the rFrameGeneratorClass shifted by any desired time shift.
    epoch : {float, lal.LIGOTimeGPS
        The epoch start time to set the waveform to. A time shift = tc - epoch is
        applied to waveforms before returning.
    variable_args : {(), list or tuple}
        A list or tuple of strings giving the names and order of parameters
        that will be passed to the generate function.
    \**frozen_params
        Keyword arguments setting the parameters that will not be changed from
        call-to-call of the generate function.

    location_args : set([])
        Should be overriden by childern classes with a set of parameters
        that set the binary's location.

    detectors : dict
        The dictionary of detectors that antenna patterns are calculated for
        on each call of generate. If no detectors were provided, will be
        ``{'RF': None}``, where "RF" means "radiation frame".
    detector_names : list
        The list of detector names. If no detectors were provided, then this
        will be ['RF'] for "radiation frame".
        The GPS start time of the frequency series returned by the generate function.
        A time shift is applied to the waveform equal to tc-epoch. Update by using
        ``set_epoch``.
        A dictionary of name, value pairs of the arguments that were last
        used by the generate function.
    rframe_generator : instance of rFrameGeneratorClass
        The instance of the radiation-frame generator that is used for waveform
        generation. All parameters in current_params except for the
        location params are passed to this class's generate function.
    frozen_location_args : dict
        Any location parameters that were included in the frozen_params.
        The list of names of arguments that are passed to the generate
        function.

    """
    location_args = set([])

    def __init__(self, rFrameGeneratorClass, epoch, detectors=None,
                 variable_args=(), recalib=None, gates=None, **frozen_params):
        # initialize frozen & current parameters:
        self._static_args = frozen_params.copy()
        # we'll separate out frozen location parameters from the frozen
        # parameters that are sent to the rframe generator
        self.frozen_location_args = {}
        loc_params = set(frozen_params.keys()) & self.location_args
        for param in loc_params:
            self.frozen_location_args[param] = frozen_params.pop(param)
        # set the order of the variable parameters
        # variables that are sent to the rFrame generator
        rframe_variables = list(set(self.variable_args) - self.location_args)
        # initialize the radiation frame generator
        self.rframe_generator = rFrameGeneratorClass(
            variable_args=rframe_variables, **frozen_params)
        self.set_epoch(epoch)
        # set calibration model
        self.recalib = recalib
        # if detectors are provided, convert to detector type; also ensure that
        # location variables are specified
        if detectors is not None:
            self.detectors = {det: Detector(det) for det in detectors}
            missing_args = [arg for arg in self.location_args if not
                (arg in self.current_params or arg in self.variable_args)]
            if any(missing_args):
                raise ValueError("detectors provided, but missing location "
                    "parameters %s. " %(', '.join(missing_args)) +
                    "These must be either in the frozen params or the "
                    "variable args.")
        else:
            self.detectors = {'RF': None}
        self.detector_names = sorted(self.detectors.keys())

    def set_epoch(self, epoch):
        """Sets the epoch; epoch should be a float or a LIGOTimeGPS."""
        self._epoch = float(epoch)

    @property
        return self._static_args

    @property
    def epoch(self):
        return _lal.LIGOTimeGPS(self._epoch)

        """The function that generates the waveforms.
        """
        pass

    def select_rframe_generator(self, approximant):
        """Method to select waveform generator based on an approximant."""
        pass

class FDomainDetFrameGenerator(BaseFDomainDetFrameGenerator):
    """Generates frequency-domain waveform in a specific frame.

    Generates a waveform using the given radiation frame generator class,
    and applies the detector response function and appropriate time offset.

    location_args : set(['tc', 'ra', 'dec', 'polarization'])
        The set of location parameters. These are not passed to the rFrame
        generator class; instead, they are used to apply the detector response
        function and/or shift the waveform in time. The parameters are:

          * tc: The GPS time of coalescence (should be geocentric time).
          * ra: Right ascension.
          * dec: declination
          * polarization: polarization.

        All of these must be provided in either the variable args or the
        frozen params if detectors is not None. If detectors
        is None, tc may optionally be provided.

        function.

    Examples
    --------

    >>> from pycbc.waveform.generator import FDomainDetFrameGenerator
    >>> generator = FDomainDetFrameGenerator(waveform.FDomainCBCGenerator, 0., variable_args=['mass1', 'mass2', 'spin1z', 'spin2z', 'tc', 'ra', 'dec', 'polarization'], detectors=['H1', 'L1'], delta_f=1./64, f_lower=20., approximant='SEOBNRv2_ROM_DoubleSpin')

    Generate a waveform:

    >>> generator.generate(mass1=38.6, mass2=29.3, spin1z=0.33, spin2z=-0.94, tc=2.43, ra=1.37, dec=-1.26, polarization=2.76)
    {'H1': <pycbc.types.frequencyseries.FrequencySeries at 0x116637350>,
     'L1': <pycbc.types.frequencyseries.FrequencySeries at 0x116637a50>}

    """
    location_args = set(['tc', 'ra', 'dec', 'polarization'])

        """Generates a waveform, applies a time shift and the detector response
        function from the given kwargs.
        """
        rfparams = {param: self.current_params[param]
            for param in kwargs if param not in self.location_args}
        hp, hc = self.rframe_generator.generate(**rfparams)
        if isinstance(hp, TimeSeries):
            df = self.current_params['delta_f']
            hp = hp.to_frequencyseries(delta_f=df)
            hc = hc.to_frequencyseries(delta_f=df)
            # time-domain waveforms will not be shifted so that the peak amp
            # happens at the end of the time series (as they are for f-domain),
            # so we add an additional shift to account for it
            tshift = 1./df - abs(hp._epoch)
        else:
            tshift = 0.
        hp._epoch = hc._epoch = self._epoch
        h = {}
        if self.detector_names != ['RF']:
            for detname, det in self.detectors.items():
                # apply detector response function
                fp, fc = det.antenna_pattern(self.current_params['ra'],
                            self.current_params['dec'],
                            self.current_params['polarization'],
                            self.current_params['tc'])
                thish = fp*hp + fc*hc
                # apply the time shift
                tc = self.current_params['tc'] + \
                    det.time_delay_from_earth_center(self.current_params['ra'],
                         self.current_params['dec'], self.current_params['tc'])
                h[detname] = apply_fd_time_shift(thish, tc+tshift, copy=False)
                if self.recalib:
                    # recalibrate with given calibration model
                    h[detname] = \
                        self.recalib[detname].map_to_adjust(h[detname],
                            **self.current_params)
        else:
            # no detector response, just use the + polarization
            if 'tc' in self.current_params:
                hp = apply_fd_time_shift(hp, self.current_params['tc']+tshift,
                                         copy=False)
            h['RF'] = hp
        if self.gates is not None:
            # resize all to nearest power of 2
            for d in h.values():
                d.resize(ceilpow2(len(d)-1) + 1)
            h = strain.apply_gates_to_fd(h, self.gates)
        return h

    def select_rframe_generator(approximant):
        """Returns a radiation frame generator class based on the approximant
        string.
        """
        return select_waveform_generator(approximant)

class FDomainDetFrameTwoPolGenerator(BaseFDomainDetFrameGenerator):

    Generates both polarizations of a waveform using the given radiation frame
    generator class, and applies the time shift. Detector response functions
    are not applied.

    location_args : set(['tc', 'ra', 'dec'])

        function.

    """
    location_args = set(['tc', 'ra', 'dec'])

        """Generates a waveform polarizations and applies a time shift.

        Returns
        -------
        dict :
            Dictionary of ``detector names -> (hp, hc)``, where ``hp, hc`` are
            the plus and cross polarization, respectively.
        """
        else:
            tshift = 0.
        h = {}
                dethp = apply_fd_time_shift(hp, tc+tshift, copy=True)
                dethc = apply_fd_time_shift(hc, tc+tshift, copy=True)
                    dethp = self.recalib[detname].map_to_adjust(
                        dethp, **self.current_params)
                    dethc = self.recalib[detname].map_to_adjust(
                        dethc, **self.current_params)
                h[detname] = (dethp, dethc)
        else:
                hc = apply_fd_time_shift(hc, self.current_params['tc']+tshift,
            h['RF'] = (hp, hc)
            hps = {}
            hcs = {}
            for det in h:
                hp = h[det]
                hc = h[det]
                hp.resize(ceilpow2(len(hp)-1) + 1)
                hc.resize(ceilpow2(len(hc)-1) + 1)
                hps[det] = hp
                hcs[det] = hc
            hps = strain.apply_gates_to_fd(hps, self.gates)
            hcs = strain.apply_gates_to_fd(hps, self.gates)
            h = {det: (hps[det], hcs[det]) for det in h}
        return h

        string.
        """

class FDomainDetFrameModesGenerator(BaseFDomainDetFrameGenerator):
    """Generates frequency-domain waveform modes in a specific frame.

    Generates both polarizations of every waveform mode using the given
    radiation frame generator class, and applies the time shift. Detector
    response functions are not applied.

        The class to use for generating the waveform modes in the radiation
        frame, e.g., :py:class:`FDomainCBCModesGenerator`. This should be the
        class, not an instance of the class (the class will be initialized with
        the appropriate arguments internally). The class should have a generate
        function that returns a dictionary of waveforms keyed by the modes.

        The GPS start time of the frequency series returned by the generate
        function. A time shift is applied to the waveform equal to tc-epoch.
        Update by using ``set_epoch``.
        function.

    """

        """Generates and returns a waveform decompsed into separate modes.

        Returns
        -------
        dict :
            Dictionary of ``detector names -> modes -> (ulm, vlm)``, where
            ``ulm, vlm`` are the frequency-domain representations of the real
            and imaginary parts, respectively, of the complex time series
            representation of the ``hlm``.
        """
        hlms = self.rframe_generator.generate(**rfparams)
        h = {det: {} for det in self.detectors}
        for mode in hlms:
            ulm, vlm = hlms[mode]
            if isinstance(ulm, TimeSeries):
                df = self.current_params['delta_f']
                ulm = ulm.to_frequencyseries(delta_f=df)
                vlm = vlm.to_frequencyseries(delta_f=df)
                # time-domain waveforms will not be shifted so that the peak
                # amplitude happens at the end of the time series (as they are
                # for f-domain), so we add an additional shift to account for
                # it
                tshift = 1./df - abs(ulm._epoch)
            else:
                tshift = 0.
            ulm._epoch = vlm._epoch = self._epoch
            if self.detector_names != ['RF']:
                for detname, det in self.detectors.items():
                    # apply the time shift
                    tc = self.current_params['tc'] + \
                        det.time_delay_from_earth_center(
                            self.current_params['ra'],
                    detulm = apply_fd_time_shift(ulm, tc+tshift, copy=True)
                    detvlm = apply_fd_time_shift(vlm, tc+tshift, copy=True)
                    if self.recalib:
                        # recalibrate with given calibration model
                        detulm = self.recalib[detname].map_to_adjust(
                            detulm, **self.current_params)
                        detvlm = self.recalib[detname].map_to_adjust(
                            detvlm, **self.current_params)
                    h[detname][mode] = (detulm, detvlm)
            else:
                # no detector response, just apply time shift
                if 'tc' in self.current_params:
                    ulm = apply_fd_time_shift(ulm,
                                              self.current_params['tc']+tshift,
                                              copy=False)
                    vlm = apply_fd_time_shift(vlm,
                h['RF'][mode] = (ulm, vlm)
            if self.gates is not None:
                # resize all to nearest power of 2
                ulms = {}
                vlms = {}
                for det in h:
                    ulm, vlm = h[det][mode]
                    ulm.resize(ceilpow2(len(ulm)-1) + 1)
                    vlm.resize(ceilpow2(len(vlm)-1) + 1)
                    ulms[det] = ulm
                    vlms[det] = vlm
                ulms = strain.apply_gates_to_fd(ulms, self.gates)
                vlms = strain.apply_gates_to_fd(ulms, self.gates)
                for det in ulms:
                    h[det][mode] = (ulms[det], vlms[det])
        return h

        string.
        """
        return select_waveform_modes_generator(approximant)

#
#
#                        Helper functions
#
#

def select_waveform_generator(approximant):
    """Returns the single-IFO generator for the approximant.

        Name of waveform approximant. Valid names can be found using
        ``pycbc.waveform`` methods.

    Returns
    -------
    generator : (PyCBC generator instance)
        A waveform generator object.

    Examples
    --------
    Get a list of available approximants:
    >>> from pycbc import waveform
    >>> waveform.fd_approximants()
    >>> waveform.td_approximants()
    >>> from pycbc.waveform import ringdown
    >>> ringdown.ringdown_fd_approximants.keys()

    Get generator object:
    >>> from pycbc.waveform.generator import select_waveform_generator
    >>> select_waveform_generator(waveform.fd_approximants()[0])
    """
    # check if frequency-domain CBC waveform
    if approximant in waveform.fd_approximants():
        return FDomainCBCGenerator
    # check if time-domain CBC waveform
    elif approximant in waveform.td_approximants():
        return TDomainCBCGenerator
    # check if frequency-domain ringdown waveform
    elif approximant in ringdown.ringdown_fd_approximants:
        if approximant == 'FdQNMfromFinalMassSpin':
            return FDomainMassSpinRingdownGenerator
        elif approximant == 'FdQNMfromFreqTau':
            return FDomainFreqTauRingdownGenerator
    elif approximant in ringdown.ringdown_td_approximants:
        if approximant == 'TdQNMfromFinalMassSpin':
            return TDomainMassSpinRingdownGenerator
        elif approximant == 'TdQNMfromFreqTau':
            return TDomainFreqTauRingdownGenerator
    # check if supernovae waveform:
    elif approximant in supernovae.supernovae_td_approximants:
        if approximant == 'CoreCollapseBounce':
            return TDomainSupernovaeGenerator
    # otherwise waveform approximant is not supported
    else:
        raise ValueError("%s is not a valid approximant." % approximant)

def select_waveform_modes_generator(approximant):
    """Returns the single-IFO modes generator for the approximant.

    Returns
    -------
    """
        return FDomainCBCModesGenerator
        return TDomainCBCModesGenerator
    raise ValueError("%s is not a valid approximant." % approximant)

""" Tools and functions to calculate interpolate waveforms using multi-banding
"""

def multiband_fd_waveform(bands=None, lengths=None, overlap=0, **p):
    """ Generate a fourier domain waveform using multibanding

    Speed up generation of a fouerier domain waveform using multibanding. This
    allows for multi-rate sampling of the frequeny space. Each band is
    smoothed and stitched together to produce the final waveform. The base
    approximant must support 'f_ref' and 'f_final'. The other parameters
    must be chosen carefully by the user.

    bands: list or str
        The frequencies to split the waveform by. These should be chosen
        so that the corresponding length include all the waveform's frequencies
        within this band.
    lengths: list or str
        The corresponding length for each frequency band. This sets the
        resolution of the subband and should be chosen carefully so that it is
        sufficiently long to include all of the bands frequency content.
        The frequency width to apply tapering between bands.
    params: dict
        The remaining keyworkd arguments passed to the base approximant
        waveform generation.

    Returns
    -------
    hp: pycbc.types.FrequencySeries
        Plus polarization
    hc: pycbc.type.FrequencySeries
        Cross polarization
    """
    from pycbc.waveform import get_fd_waveform

    if isinstance(bands, str):
        bands = [float(s) for s in bands.split(' ')]

    if isinstance(lengths, str):
        lengths = [float(s) for s in lengths.split(' ')]

    p['approximant'] = p['base_approximant']
    df = p['delta_f']
    fmax = p['f_final']
    flow = p['f_lower']

    bands = [flow] + bands + [fmax]
    dfs = [df] + [1.0 / l for l in lengths]

    dt = 1.0 / (2.0 * fmax)
    tlen = int(1.0 / dt / df)
    flen = tlen / 2 + 1
    wf_plus = TimeSeries(zeros(tlen, dtype=numpy.float32),
                         copy=False, delta_t=dt, epoch=-1.0/df)
    wf_cross = TimeSeries(zeros(tlen, dtype=numpy.float32),
                          copy=False, delta_t=dt, epoch=-1.0/df)

    # Iterate over the sub-bands
    for i in range(len(lengths)+1):
        taper_start = taper_end = False
        if i != 0:
            taper_start = True
        if i != len(lengths):
            taper_end = True

        # Generate waveform for sub-band of full waveform
        start = bands[i]
        stop = bands[i+1]
        p2 = p.copy()
        p2['delta_f'] = dfs[i]
        p2['f_lower'] = start
        p2['f_final'] = stop

        if taper_start:
            p2['f_lower'] -= overlap / 2.0

        if taper_end:
            p2['f_final'] += overlap / 2.0

        tlen = int(1.0 / dt / dfs[i])
        flen = tlen / 2 + 1

        hp, hc = get_fd_waveform(**p2)

        # apply window function to smooth over transition regions
        kmin = int(p2['f_lower'] / dfs[i])
        kmax = int(p2['f_final'] / dfs[i])
        taper = numpy.hanning(int(overlap * 2 / dfs[i]))

        for wf, h in zip([wf_plus, wf_cross], [hp, hc]):
            h = h.astype(numpy.complex64)

            if taper_start:
                h[kmin:kmin + len(taper) // 2] *= taper[:len(taper)//2]

            if taper_end:
                l, r = kmax - (len(taper) - len(taper) // 2), kmax
                h[l:r] *= taper[len(taper)//2:]

            # add frequency band to total and use fft to interpolate
            h.resize(flen)
            h = h.to_timeseries()
            wf[len(wf)-len(h):] += h

    return (wf_plus.to_frequencyseries().astype(hp.dtype),
            wf_cross.to_frequencyseries().astype(hp.dtype))

""" Utilities for introducing nonlinear tidal effects into waveform approximants
"""
import lal

def nltides_fourier_phase_difference(f, delta_f, f0, amplitude, n, m1, m2):
    """Calculate the change to the Fourier phase change due
    to non-linear tides. Note that the Fourier phase Psi(f)
    is not the same as the gravitational-wave phase phi(f) and
    is computed by
    Delta Psi(f) = 2 \pi f Delta t(f) - Delta phi(f)

    f: numpy.array
        Array of frequency values to calculate the fourier phase difference
    delta_f: float
        Frequency resolution of f array
    f0: float
    n: float
    m1: float
    m2: float

    Returns
    -------
    delta_psi: numpy.array
        Fourier phase as a function of frequency
    """

    kmin = int(f0/delta_f)
    kmax = len(f)

    f_ref, t_of_f_factor, phi_of_f_factor = \
        pycbc.conversions.nltides_coefs(amplitude, n, m1, m2)

    # Fourier phase shift below f0 from \Delta \phi(f)
    delta_psi_f_le_f0 = numpy.ones(kmin)
    delta_psi_f_le_f0 *= - phi_of_f_factor * (f0/f_ref)**(n-3.)

    # Fourier phase shift above f0 from \Delta \phi(f)
    delta_psi_f_gt_f0 = - phi_of_f_factor * (f[kmin:kmax]/f_ref)**(n-3.)

    # Fourier phase shift below f0 from 2 pi f \Delta t(f)
    delta_psi_f_le_f0 += 2.0 * lal.lal.PI * f[0:kmin] * t_of_f_factor * \
        (f0/f_ref)**(n-4.)

    # Fourier phase shift above f0 from 2 pi f \Delta t(f)
    delta_psi_f_gt_f0 += 2.0 * lal.lal.PI * f[kmin:kmax] * t_of_f_factor * \
        (f[kmin:kmax]/f_ref)**(n-4.)

    # Return the shift to the Fourier phase
    return numpy.concatenate((delta_psi_f_le_f0, delta_psi_f_gt_f0), axis=0)

def nonlinear_tidal_spa(**kwds):
    """Generates a frequency-domain waveform that implements the
    TaylorF2+NL tide model described in https://arxiv.org/abs/1808.07013
    """

    from pycbc import waveform
    from pycbc.types import Array

    # We start with the standard TaylorF2 based waveform
    kwds.pop('approximant')
    hp, hc = waveform.get_fd_waveform(approximant="TaylorF2", **kwds)

    # Add the phasing difference from the nonlinear tides
    f = numpy.arange(len(hp)) * hp.delta_f
    pd =  Array(numpy.exp(-1.0j * nltides_fourier_phase_difference(f,
               hp.delta_f,
               kwds['f0'], kwds['amplitude'], kwds['n'],
               kwds['mass1'], kwds['mass2'])),
               dtype=hp.dtype)
    hp *= pd
    hc *= pd
    return hp, hc

#
#
#

#
#
#                                   Preamble
#
#
"""Classes to define common parameters used for waveform generation.
"""

try:
    from collections import UserList
    from UserList import UserList

#
#
#                                  Base definitions
#
#

class Parameter(str):
    """A class that stores information about a parameter. This is done by
    sub-classing string, adding additional attributes.
    """

    def __new__(cls, name, dtype=None, default=None, label=None,
                 description="No description."):
        obj = str.__new__(cls, name)
        obj.dtype = dtype
        obj.default = default
        obj.label = label
        obj.description = description
        return obj

    def docstr(self, prefix='', include_label=True):
        """Returns a string summarizing the parameter. Format is:
        <prefix>``name`` : {``default``, ``dtype``}
        <prefix>   ``description`` Label: ``label``.
        """
        outstr = "%s%s : {%s, %s}\n" %(prefix, self.name, str(self.default),
            str(self.dtype).replace("<type '", '').replace("'>", '')) + \
            "%s    %s" %(prefix, self.description)
        if include_label:
            outstr += " Label: %s" %(self.label)
        return outstr

class ParameterList(UserList):
    """A list of parameters. Each element in the list is expected to be a
    Parameter instance.
    """

    @property
    def names(self):
        """Returns a list of the names of each parameter."""
        return [x.name for x in self]

    @property
    def aslist(self):
        """Cast to basic list."""
        return list(self)

    @property
    def asdict(self):
        """Returns a dictionary of the parameters keyed by the parameters."""
        return dict([[x, x] for x in self])

    def defaults(self):
        """Returns a list of the name and default value of each parameter,
        as tuples.
        """
        return [(x, x.default) for x in self]

    def default_dict(self):
        """Returns a dictionary of the name and default value of each
        """
        return OrderedDict(self.defaults())

    @property
    def nodefaults(self):
        """Returns a ParameterList of the parameters that have None for
        defaults.
        """
        return ParameterList([x for x in self if x.default is None])

    @property
    def dtypes(self):
        """Returns a list of the name and dtype of each parameter,
        as tuples.
        """
        return [(x, x.dtype) for x in self]

    @property
    def dtype_dict(self):
        """Returns a dictionary of the name and dtype of each parameter."""
        return OrderedDict(self.dtypes)

    @property
    def descriptions(self):
        """Returns a list of the name and description of each parameter,
        as tuples.
        """
        return [(x, x.description) for x in self]

    @property
    def description_dict(self):
        """Return a dictionary of the name and description of each parameter.
        """
        return OrderedDict(self.descriptions)

    @property
    def labels(self):
        """Returns a list of each parameter and its label, as tuples."""
        return [(x, x.label) for x in self]

    @property
    def label_dict(self):
        """Return a dictionary of the name and label of each parameter.
        """
        return OrderedDict(self.labels)

        """Returns the ``docstr`` of each parameter joined together."""
        return '\n'.join([x.docstr(prefix, include_label) for x in self])

#
#
#                        Parameter definitions
#
#

#
#   CBC intrinsic parameters
#
mass1 = Parameter("mass1",
                dtype=float, default=None, label=r"$m_1~(\mathrm{M}_\odot)$",
                description="The mass of the first component object in the "
                          "binary (in solar masses).")
mass2 = Parameter("mass2",
                dtype=float, default=None, label=r"$m_2~(\mathrm{M}_\odot)$",
                description="The mass of the second component object in the "
                            "binary (in solar masses).")
spin1x = Parameter("spin1x",
                dtype=float, default=0., label=r"$\chi_{1x}$",
                description="The x component of the first binary component's "
                            "dimensionless spin.")
spin1y = Parameter("spin1y",
                dtype=float, default=0., label=r"$\chi_{1y}$",
                description="The y component of the first binary component's "
spin1z = Parameter("spin1z",
                dtype=float, default=0., label=r"$\chi_{1z}$",
                description="The z component of the first binary component's "
spin2x = Parameter("spin2x",
                dtype=float, default=0., label=r"$\chi_{2x}$",
                description="The x component of the second binary component's "
spin2y = Parameter("spin2y",
                dtype=float, default=0., label=r"$\chi_{2y}$",
                description="The y component of the second binary component's "
spin2z = Parameter("spin2z",
                dtype=float, default=0., label=r"$\chi_{2z}$",
                description="The z component of the second binary component's "
eccentricity = Parameter("eccentricity",
                dtype=float, default=0., label=r"$e$",
                description="Eccentricity.")

# derived parameters (these are not used for waveform generation)
mchirp = Parameter("mchirp",
                dtype=float, label=r"$\mathcal{M}~(\mathrm{M}_\odot)$",
                description="The chirp mass of the binary (in solar masses).")
eta = Parameter("eta",
                dtype=float, label=r"$\eta$",
                description="The symmetric mass ratio of the binary.")
mtotal = Parameter("mtotal",
                dtype=float, label=r"$M~(\mathrm{M}_\odot)$",
                description="The total mass of the binary (in solar masses).")
q = Parameter("q",
              dtype=float, label=r"$q$",
              description="The mass ratio, m1/m2, where m1 >= m2.")
srcmass1 = Parameter("srcmass1", dtype=float,
                     label=r"$m_1^{\rm{src}}~(\mathrm{M}_\odot)$",
                     description="The mass of the first component object in "
                                 "the source frame (in solar masses).")
srcmass2 = Parameter("srcmass1", dtype=float,
                     label=r"$m_2^{\rm{src}}~(\mathrm{M}_\odot)$",
                     description="The mass of the second component object in "
srcmchirp = Parameter("srcmchirp", dtype=float,
                      label=r"$\mathcal{M}^{\rm{src}}~(\mathrm{M}_\odot)$",
                      description="The chirp mass of the binary in the "
                                  "source frame (in solar masses).")
srcmtotal = Parameter("mtotal", dtype=float,
                      label=r"$M^{\rm{src}}~(\mathrm{M}_\odot)$",
                      description="The total mass of the binary in the "
primary_mass = Parameter("primary_mass",
                dtype=float, label=r"$m_{1}$",
                description="Mass of the primary object (in solar masses).")
secondary_mass = Parameter("secondary_mass",
                dtype=float, label=r"$m_{2}$",
                description="Mass of the secondary object (in solar masses).")

# derived parameters for component spins
chi_eff = Parameter("chi_eff",
                dtype=float, label=r"$\chi_\mathrm{eff}$",
                description="Effective spin of the binary.")
chi_p = Parameter("chi_p",
                dtype=float, label=r"$\chi_p$",
                description="Effective precessing spin of the binary.")
spin_px = Parameter("spin_px",
                dtype=float, label=r"$\chi_{1x}$",
                description="The x component of the dimensionless spin of the "
                            "primary object.")
spin_py = Parameter("spin_py",
                dtype=float, label=r"$\chi_{1y}$",
                description="The y component of the dimensionless spin of the "
spin_pz = Parameter("spin_pz",
                dtype=float, label=r"$\chi_{1z}$",
                description="The z component of the dimensionless spin of the "
spin_sx = Parameter("spin_sx",
                dtype=float, label=r"$\chi_{2x}$",
                            "secondary object.")
spin_sy = Parameter("spin_sy",
                dtype=float, label=r"$\chi_{2y}$",
spin_sz = Parameter("spin_sz",
                dtype=float, label=r"$\chi_{2z}$",
lambda1 = Parameter("lambda1",
                dtype=float, default=None, label=r"$\Lambda_1$",
                description="The dimensionless tidal deformability parameter of object 1.")
lambda2 = Parameter("lambda2",
                dtype=float, default=None, label=r"$\Lambda_2$",
                description="The dimensionless tidal deformability parameter of object 2.")
dquad_mon1 = Parameter("dquad_mon1",
                dtype=float, default=None, label=r"$qm_1$",
                description="Quadrupole-monopole parameter / m_1^5 -1.")
dquad_mon2 = Parameter("dquad_mon2",
                dtype=float, default=None, label=r"$qm_2$",
                description="Quadrupole-monopole parameter / m_2^5 -1.")
lambda_octu1 = Parameter("lambda_octu1",
                dtype=float, default=None, label=r"$\Lambda_3^{(1)}$",
                description="The octupolar tidal deformability parameter of "
                            "object 1.")
lambda_octu2 = Parameter("lambda_octu2",
                dtype=float, default=None, label=r"$\Lambda_3^{(2)}$",
                            "object 2.")
quadfmode1 = Parameter("quadfmode1",
                dtype=float, default=None, label=r"$m_1 \omega_{02}^{(1)}$",
                description="The quadrupolar f-mode angular frequency of "
quadfmode2 = Parameter("quadfmode2",
                dtype=float, default=None, label=r"$m_ \omega_{02}^{(2)}$",
octufmode1 = Parameter("octufmode1",
                dtype=float, default=None, label=r"$m_1 \omega_{03}^{(1)}$",
                description="The octupolar f-mode angular frequency of "
octufmode2 = Parameter("octufmode2",
                dtype=float, default=None, label=r"$m_ \omega_{03}^{(2)}$",

# derived parameters for component spin magnitude and angles
spin1_a = Parameter("spin1_a",
                    dtype=float, label=r"$a_{1}$",
                    description="The dimensionless spin magnitude "
                                r"$|\vec{s}/m_{1}^2|$.")
spin2_a = Parameter("spin2_a",
                    dtype=float, label=r"$a_{2}$",
                                r"$|\vec{s}/m_{2}^2|$.")
spin1_azimuthal = Parameter(
                      "spin1_azimuthal",
                      dtype=float, label=r"$\theta_1^\mathrm{azimuthal}$",
                      description="The azimuthal spin angle for mass 1.")
spin2_azimuthal = Parameter(
                      "spin2_azimuthal",
                      dtype=float, label=r"$\theta_2^\mathrm{azimuthal}$",
                      description="The azimuthal spin angle for mass 2.")
spin1_polar = Parameter("spin1_polar",
                        dtype=float, label=r"$\theta_1^\mathrm{polar}$",
                        description="The polar spin angle for mass 1.")
spin2_polar = Parameter("spin2_polar",
                        dtype=float, label=r"$\theta_2^\mathrm{polar}$",
                        description="The polar spin angle for mass 2.")

#
#   Parameters needed for CBC waveform generation
#
f_lower = Parameter("f_lower",
                dtype=float, default=None, label=r"$f_0$ (Hz)",
                description="The starting frequency of the waveform (in Hz).")
f_final = Parameter("f_final",
                dtype=float, default=0, label=r"$f_{\mathrm{final}}$ (Hz)",
                description="The ending frequency of the waveform. The "
                            "default (0) indicates that the choice is made by "
                            "the respective approximant.")
f_final_func = Parameter("f_final_func",
                dtype=str, default="", label=None,
                description="Use the given frequency function to compute f_final "
                            "based on the parameters of the waveform.")
f_ref = Parameter("f_ref",
                dtype=float, default=0, label=r"$f_{\mathrm{ref}}$ (Hz)",
                description="The reference frequency.")
delta_f = Parameter("delta_f",
                dtype=float, default=None, label=r"$\Delta f$ (Hz)",
                description="The frequency step used to generate the waveform "
                            "(in Hz).")
delta_t = Parameter("delta_t",
                dtype=float, default=None, label=r"$\Delta t$ (s)",
                description="The time step used to generate the waveform "
                            "(in s).")
sample_points = Parameter("sample_points",
                dtype="Array", default=None, label=None,
                description="An array of the frequencies (in Hz) at which to "
                            "generate the waveform.")
approximant = Parameter("approximant",
                dtype=str, default=None, label=None,
                description="A string that indicates the chosen approximant.")
phase_order = Parameter("phase_order",
                dtype=int, default=-1, label=None,
                description="The pN order of the orbital phase. The default "
                            "of -1 indicates that all implemented orders are "
                            "used.")
spin_order = Parameter("spin_order",
                description="The pN order of the spin corrections. The "
                            "default of -1 indicates that all implemented "
                            "orders are used.")
tidal_order = Parameter("tidal_order",
                description="The pN order of the tidal corrections. The "
amplitude_order = Parameter("amplitude_order",
                description="The pN order of the amplitude. The default of -1 "
                            "indicates that all implemented orders are used.")
eccentricity_order = Parameter("eccentricity_order",
                description="The pN order of the eccentricity corrections."
                        "The default of -1 indicates that all implemented orders are used.")
numrel_data = Parameter("numrel_data",
                description="Sets the NR flags; only needed for NR waveforms.")

#
#   General location parameters
#
distance = Parameter("distance",
                dtype=float, default=1., label=r"$d_L$ (Mpc)",
                description="Luminosity distance to the binary (in Mpc).")
chirp_distance = Parameter("chirp_distance",
                dtype=float, default=1., label=r"$d_c$ (Mpc)",
                description="Chirp distance to the binary (in Mpc).")
coa_phase = Parameter("coa_phase",
                dtype=float, default=0., label=r"$\phi_c$",
                description="Coalesence phase of the binary (in rad).")
inclination = Parameter("inclination",
                dtype=float, default=0., label=r"$\iota$",
                description="Inclination (rad), defined as the angle between "
                            "the orbital angular momentum L and the "
                            "line-of-sight at the reference frequency.")
thetajn = Parameter("thetajn",
                    dtype=float, default=0., label=r"$\theta_{JN}$",
                    description="The angle between the total angular momentum "
                                "J and the line-of-sight.")
long_asc_nodes = Parameter("long_asc_nodes",
                dtype=float, default=0., label=r"$\Omega$",
                description="Longitude of ascending nodes axis (rad).")
mean_per_ano = Parameter("mean_per_ano",
                dtype=float, default=0., label=r"$\delta$",
                description="Mean anomaly of the periastron (rad).")
tc = Parameter("tc",
                dtype=float, default=None, label=r"$t_c$ (s)",
                description="Coalescence time (s).")
delta_tc = Parameter("delta_tc", dtype=float,
                     label=r"$\Delta t_c~(\rm{s})$",
                     description="Coalesence time offset.")
ra = Parameter("ra",
                dtype=float, default=None, label=r"$\alpha$",
                description="Right ascension (rad).")
dec = Parameter("dec",
                dtype=float, default=None, label=r"$\delta$",
                description="Declination (rad).")
polarization = Parameter("polarization",
                dtype=float, default=None, label=r"$\psi$",
                description="Polarization (rad).")
redshift = Parameter("redshift",
                dtype=float, default=None, label=r"$z$",
                description="Redshift.")
comoving_volume = Parameter("comoving_volume", dtype=float,
                            label=r"$V_C~(\rm{Mpc}^3)$",
                            description="Comoving volume (in cubic Mpc).")

#
#   Calibration parameters
#
delta_fs = Parameter("calib_delta_fs",
                     dtype=float,
                     description="Change in optical spring freq (Hz).")
delta_fc = Parameter("calib_delta_fc",
                     description="Change in cavity pole freq (Hz).")
delta_qinv = Parameter("calib_delta_qinv",
                       dtype=float,
                       description="Change in inverse quality factor.")
kappa_c = Parameter("calib_kappa_c",
                    dtype=float)
kappa_tst_re = Parameter("calib_kappa_tst_re",
                         dtype=float)
kappa_tst_im = Parameter("calib_kappa_tst_im",
kappa_pu_re = Parameter("calib_kappa_pu_re",
                        dtype=float)
kappa_pu_im = Parameter("calib_kappa_pu_im",

#
#   Non mandatory flags with default values
#
frame_axis = Parameter("frame_axis",
                dtype=int, default=0,
                description="Allow to choose among orbital_l, view and total_j")
modes_choice = Parameter("modes_choice",
                description="Allow to turn on  among orbital_l, view and total_j")
side_bands = Parameter("side_bands",
                description="Flag for generating sidebands")
mode_array = Parameter("mode_array",
                dtype=list, default=None,
                description="Choose which (l,m) modes to include when "
                            "generating a waveform. "
                            "Only if approximant supports this feature."
                            "By default pass None and let lalsimulation "
                            "use it's default behaviour."
                            "Example: mode_array = [ [2,2], [2,-2] ]")

#
#   Parametrized testing general relativity parameters
#
dchi0 = Parameter("dchi0",
                dtype=float, default=0., label=r"$d\chi_0$",
                description="0PN testingGR parameter.")
dchi1 = Parameter("dchi1",
                dtype=float, default=0., label=r"$d\chi_1$",
                description="0.5PN testingGR parameter.")
dchi2 = Parameter("dchi2",
                dtype=float, default=0., label=r"$d\chi_2$",
                description="1PN testingGR parameter.")
dchi3 = Parameter("dchi3",
                dtype=float, default=0., label=r"$d\chi_3$",
                description="1.5PN testingGR parameter.")
dchi4 = Parameter("dchi4",
                dtype=float, default=0., label=r"$d\chi_4$",
                description="2PN testingGR parameter.")
dchi5 = Parameter("dchi5",
                dtype=float, default=0., label=r"$d\chi_5$",
                description="2.5PN testingGR parameter.")
dchi5l = Parameter("dchi5l",
                dtype=float, default=0., label=r"$d\chi_5{l}$",
                description="2.5PN logrithm testingGR parameter.")
dchi6 = Parameter("dchi6",
                dtype=float, default=0., label=r"$d\chi_6$",
                description="3PN testingGR parameter.")
dchi6l = Parameter("dchi6l",
                dtype=float, default=0., label=r"$d\chi_{6l}$",
                description="3PN logrithm testingGR parameter.")
dchi7 = Parameter("dchi7",
                dtype=float, default=0., label=r"$d\chi_7$",
                description="3.5PN testingGR parameter.")
dalpha1 = Parameter("dalpha1",
                dtype=float, default=0., label=r"$d\alpha_1$",
                description="Merger-ringdown testingGR parameter.")
dalpha2 = Parameter("dalpha2",
                dtype=float, default=0., label=r"$d\alpha_2$",
dalpha3 = Parameter("dalpha3",
                dtype=float, default=0., label=r"$d\alpha_3$",
dalpha4 = Parameter("dalpha4",
                dtype=float, default=0., label=r"$d\alpha_4$",
dalpha5 = Parameter("dalpha5",
                dtype=float, default=0., label=r"$d\alpha_5$",
dbeta1 = Parameter("dbeta1",
                dtype=float, default=0., label=r"$d\beta_1$",
                description="Intermediate testingGR parameter.")
dbeta2 = Parameter("dbeta2",
                dtype=float, default=0., label=r"$d\beta_2$",
dbeta3 = Parameter("dbeta3",
                dtype=float, default=0., label=r"$d\beta_3$",
#
#
#                        Parameter list definitions
#
#

# parameters describing the location of a binary w.r.t. the Earth. Note: we
# do not include distance here. This is because these parameters are not
# passed to the waveform generators in lalsimulation, but are instead applied
# after a waveform is generated. Distance, however, is a parameter used by
# the waveform generators.
location_params = ParameterList([tc, ra, dec, polarization])

# parameters describing the orientation of a binary w.r.t. the radiation
# frame. Note: we include distance here, as it is typically used for generating
# waveforms.
orientation_params = ParameterList\
    ([distance, coa_phase, inclination, long_asc_nodes, mean_per_ano])

# the extrinsic parameters of a waveform
extrinsic_params = orientation_params + location_params

# testing GR parameters
testingGR_params = ParameterList\
    ([dchi0, dchi1, dchi2, dchi3, dchi4, dchi5, dchi5l, dchi6, dchi6l,
      dchi7, dalpha1, dalpha2, dalpha3, dalpha4, dalpha5,
      dbeta1, dbeta2, dbeta3])

# intrinsic parameters of a CBC waveform. Some of these are not recognized
# by every waveform model
cbc_intrinsic_params = ParameterList\
    ([mass1, mass2, spin1x, spin1y, spin1z, spin2x, spin2y, spin2z,
      eccentricity, lambda1, lambda2, dquad_mon1, dquad_mon2, lambda_octu1,
      lambda_octu2, quadfmode1, quadfmode2, octufmode1, octufmode2]) + \
    testingGR_params

# the parameters of a cbc in the radiation frame
cbc_rframe_params = cbc_intrinsic_params + orientation_params

# calibration parameters
calibration_params = ParameterList([
    delta_fc, delta_fs, delta_qinv, kappa_c, kappa_tst_re, kappa_tst_im,
    kappa_pu_re, kappa_pu_im])

# common generation parameters are parameters needed to generate either
# a TD, FD, or frequency sequence waveform
common_generation_params = ParameterList([
    approximant, f_ref, phase_order, spin_order, tidal_order, amplitude_order, eccentricity_order])

# Flags having discrete values, optional to generate either
flags_generation_params = ParameterList([frame_axis, modes_choice, side_bands, mode_array])

# the following are parameters needed to generate an FD or TD waveform that
# is equally sampled
common_gen_equal_sampled_params = ParameterList([f_lower]) + \
    common_generation_params + flags_generation_params

# the following are parameters that can be used to generate an FD waveform
fd_waveform_params = cbc_rframe_params + ParameterList([delta_f]) + \
    common_gen_equal_sampled_params + ParameterList([f_final, f_final_func])

# the following are parameters that can be used to generate a TD waveform
td_waveform_params = cbc_rframe_params + ParameterList([delta_t]) + \
    common_gen_equal_sampled_params + ParameterList([numrel_data]) + \
    flags_generation_params

# The following are the minimum set of parameters that are required to
# generate a FD or TD waveform. All other parameters have some default value as
# defined above. Defaults of None simply mean that the value is not passed into
# the lal_dict structure and the waveform generator will take whatever default
# behaviour
td_required = ParameterList([f_lower, delta_t, approximant])
fd_required = ParameterList([f_lower, delta_f, approximant])

####
cbc_td_required = ParameterList([mass1, mass2, f_lower, delta_t, approximant])
cbc_fd_required = ParameterList([mass1, mass2, f_lower, delta_f, approximant])

# the following are parameters that can be used to generate a
# frequency series waveform
fd_waveform_sequence_params = cbc_rframe_params + \
    ParameterList([sample_points]) + common_generation_params + \

""" Utilities for handling waveform plugins
"""

def add_custom_waveform(approximant, function, domain,
                        sequence=False, force=False):
    """ Make custom waveform available to pycbc

        The name of the waveform
    function : function
        The function to generate the waveform
    domain : str
        Either 'frequency' or 'time' to indicate the domain of the waveform.
    sequence : bool, False
        Function evaluates waveform at only chosen points (instead of a
        equal-spaced grid).
    """
    from pycbc.waveform.waveform import cpu_fd, cpu_td, fd_sequence

    used = RuntimeError("Can't load plugin waveform {}, the name is"
                        " already in use.".format(approximant))

    if domain == 'time':
        if not force and (approximant in cpu_td):
            raise used
        cpu_td[approximant] = function
    elif domain == 'frequency':
        if sequence:
            if not force and (approximant in fd_sequence):
                raise used
            fd_sequence[approximant] = function
        else:
            if not force and (approximant in cpu_fd):
                raise used
            cpu_fd[approximant] = function
    else:
        raise ValueError("Invalid domain ({}), should be "
                         "'time' or 'frequency'".format(domain))

def add_length_estimator(approximant, function):
    """ Add length estimator for an approximant

        Name of approximant
        A function which takes kwargs and returns the waveform length
    """
    from pycbc.waveform.waveform import _filter_time_lengths
    if approximant in _filter_time_lengths:
        raise RuntimeError("Can't load length estimator {}, the name is"
                           " already in use.".format(approximant))
    _filter_time_lengths[approximant] = function

def retrieve_waveform_plugins():
    """ Process external waveform plugins
    """
    import pkg_resources

    # Check for fd waveforms
    for plugin in pkg_resources.iter_entry_points('pycbc.waveform.fd'):
        add_custom_waveform(plugin.name, plugin.resolve(), 'frequency')

    # Check for fd sequence waveforms
    for plugin in pkg_resources.iter_entry_points('pycbc.waveform.fd_sequence'):
        add_custom_waveform(plugin.name, plugin.resolve(), 'frequency',
                            sequence=True)

    # Check for td waveforms
    for plugin in pkg_resources.iter_entry_points('pycbc.waveform.td'):
        add_custom_waveform(plugin.name, plugin.resolve(), 'time')

    # Check for waveform length estimates
    for plugin in pkg_resources.iter_entry_points('pycbc.waveform.length'):
        add_length_estimator(plugin.name, plugin.resolve())

""" Waveform approximants for the pre-merger detection of gravitational waves
"""

def premerger_taylorf2(**p):
    """ Generate time-shifted TaylorF2"""
    from pycbc.waveform.spa_tmplt import spa_length_in_time
    from pycbc.waveform.utils import fd_taper

    p.pop('approximant')
    hp, hc = get_fd_waveform(approximant="TaylorF2", **p)

    removed = spa_length_in_time(mass1=p['mass1'],
                                 mass2=p['mass2'],
                                 f_lower=p['f_final'],
                                 phase_order=-1)

    hp = hp.cyclic_time_shift(removed)
    hp.start_time += removed

    hc = hc.cyclic_time_shift(removed)
    hc.start_time += removed

    logging.info("PreTaylorF2, m1=%.1f, m2=%.1f, fmax=%.1f, timeshift=%.1f",
                 p['mass1'], p['mass2'], p['f_final'], removed)
    kmin = int(p['f_lower'] / p['delta_f'])
    hp[0:kmin] = 0
    hc[0:kmin] = 0

    if 'final_taper' in p:
        taper_size = p['final_taper']
        hp = fd_taper(hp, p['f_final'] - taper_size, p['f_final'], side='right')
        hc = fd_taper(hc, p['f_final'] - taper_size, p['f_final'], side='right')

    hp.time_offset = removed
    hc.time_offset = removed

# Copyright (C) 2016 Miriam Cabero Mueller
#
#
#

#
#
#                                   Preamble
#
#
"""Generate ringdown templates in the time and frequency domain.
"""

import re, numpy, lal
from pycbc.types import TimeSeries, FrequencySeries, float64, complex128, zeros
from pycbc.waveform.waveform import get_obj_attrs
from pycbc.conversions import get_lm_f0tau_allmodes

qnm_required_args = ['f_0', 'tau', 'amp', 'phi']
mass_spin_required_args = ['final_mass','final_spin', 'lmns', 'inclination']
freqtau_required_args = ['lmns']
td_args = {'delta_t':None, 't_final':None, 'taper':False}
fd_args = {'t_0':0, 'delta_f':None, 'f_lower':None, 'f_final':None}

max_freq = 16384/2.
min_dt = 1. / (2 * max_freq)
pi = numpy.pi
two_pi = 2 * numpy.pi
pi_sq = numpy.pi * numpy.pi

# Input parameters ############################################################

def props(obj, required, domain_args, **kwargs):
    """ Return a dictionary built from the combination of defaults, kwargs,
    and the attributes of the given object.
    """
    # Get the attributes of the template object
    pr = get_obj_attrs(obj)

    # Get the parameters to generate the waveform
    # Note that keyword arguments override values in the template object
    input_params = domain_args.copy()
    input_params.update(pr)
    input_params.update(kwargs)

    # Check if the required arguments are given
    for arg in required:
        if arg not in input_params:
            raise ValueError('Please provide ' + str(arg))

    return input_params

def format_lmns(lmns):
    """ Checks if the format of the parameter lmns is correct, returning the
    appropriate format if not, and raise an error if nmodes=0.
    The required format for the ringdown approximants is a list of lmn modes
    as a single whitespace-separated string, with n the number
    of overtones desired. Alternatively, a list object may be used,
    containing individual strings as elements for the lmn modes.
    For instance, lmns = '223 331' are the modes 220, 221, 222, and 330.
    Giving lmns = ['223', '331'] is equivalent.
    The ConfigParser of a workflow might convert that to a single string
    (case 1 below) or a list with a single string (case 2), and this function
    will return the appropriate list of strings. If a different format is
    given, raise an error.
    """

    # Catch case of lmns given as float (as int injection values are cast
    # to float by pycbc_create_injections), cast to int, then string
    if isinstance(lmns, float):
        lmns = str(int(lmns))
    # Case 1: the lmns are given as a string, e.g. '221 331'
    if isinstance(lmns, str):
        lmns = lmns.split(' ')
    # Case 2: the lmns are given as strings in a list, e.g. ['221', '331']
    elif isinstance(lmns, list):
        pass
    else:
        raise ValueError('Format of parameter lmns not recognized. See '
                         'approximant documentation for more info.')

    out = []
    # Cycle over the lmns to ensure that we get back a list of strings that
    # are three digits long, and that nmodes!=0
    for lmn in lmns:
        # The following line is to be used with Python3 if the lmns are stored
        # as a list of strings in the HDF files and the workflow converts that
        # to a string
        # lmn = lmn.strip(" b'")
        # Try to convert to int and then str, to ensure the right format
        lmn = str(int(lmn))
        if len(lmn) != 3:
            raise ValueError('Format of parameter lmns not recognized. See '
                             'approximant documentation for more info.')
        elif int(lmn[2]) == 0:
            raise ValueError('Number of overtones (nmodes) must be greater '
                             'than zero in lmn={}.'.format(lmn))
        out.append(lmn)

    return out

def lm_amps_phases(**kwargs):
    """ Take input_params and return dictionaries with amplitudes and phases
    of each overtone of a specific lm mode, checking that all of them are given.
    """
    lmns = format_lmns(kwargs['lmns'])
    amps, phis = {}, {}
    # amp220 is always required, because the amplitudes of subdominant modes
    # are given as fractions of amp220.
    try:
        amps['220'] = kwargs['amp220']
        raise ValueError('amp220 is always required')

    # Get amplitudes of subdominant modes and all phases
        lm, nmodes = lmn[0:-1], int(lmn[2])
        for n in range(nmodes):
            # If it is the 22 mode, skip 220 amplitude
            mode = lm + '{}'.format(n)
            if mode != '220':
                try:
                    amps[mode] = kwargs['amp' + mode] * amps['220']
                    raise ValueError('amp{} is required'.format(mode))
            try:
                phis[mode] = kwargs['phi' + mode]
                raise ValueError('phi{} is required'.format(mode))

    return amps, phis

def lm_freqs_taus(**kwargs):
    """ Take input_params and return dictionaries with frequencies and damping
    times of each overtone of a specific lm mode, checking that all of them
    are given.
    """
    freqs, taus = {}, {}

            try:
                freqs[mode] = kwargs['f_' + mode]
                raise ValueError('f_{} is required'.format(mode))
            try:
                taus[mode] = kwargs['tau_' + mode]
                raise ValueError('tau_{} is required'.format(mode))

    return freqs, taus

# Functions to obtain t_final, f_final and output vector ######################

def qnm_time_decay(tau, decay):
    """Return the time at which the amplitude of the
    ringdown falls to decay of the peak amplitude.

    tau : float
        The damping time of the sinusoid.
    decay : float
        The fraction of the peak amplitude.

    Returns
    -------
    t_decay : float
        The time at which the amplitude of the time-domain
        ringdown falls to decay of the peak amplitude.
    """

    return - tau * numpy.log(decay)

def qnm_freq_decay(f_0, tau, decay):
    """Return the frequency at which the amplitude of the

    f_0 : float
        The ringdown-frequency, which gives the peak amplitude.
    tau : float

    Returns
    -------
    f_decay : float
        The frequency at which the amplitude of the frequency-domain
    """

    q_0 = pi * f_0 * tau
    alpha = 1. / decay
    alpha_sq = 1. / decay / decay

    # Expression obtained analytically under the assumption
    # that 1./alpha_sq, q_0^2 >> 1
    q_sq = (alpha_sq + 4*q_0*q_0 + alpha*numpy.sqrt(alpha_sq + 16*q_0*q_0)) / 4.
    return numpy.sqrt(q_sq) / pi / tau

def lm_tfinal(damping_times):
    """Return the maximum t_final of the modes given, with t_final the time
    at which the amplitude falls to 1/1000 of the peak amplitude
    """

    if isinstance(damping_times, dict):
        t_max = {}
        for lmn in damping_times.keys():
            t_max[lmn] = qnm_time_decay(damping_times[lmn], 1./1000)
        t_final = max(t_max.values())
    else:
        t_final = qnm_time_decay(damping_times, 1./1000)

    return t_final

def lm_deltat(freqs, damping_times):
    """Return the minimum delta_t of all the modes given, with delta_t given by
    the inverse of the frequency at which the amplitude of the ringdown falls to
    1/1000 of the peak amplitude.
    """

    if isinstance(freqs, dict) and isinstance(damping_times, dict):
        dt = {}
        for lmn in freqs.keys():
            dt[lmn] = 1. / qnm_freq_decay(freqs[lmn],
                               damping_times[lmn], 1./1000)
        delta_t = min(dt.values())
    elif isinstance(freqs, dict) and not isinstance(damping_times, dict):
        raise ValueError('Missing damping times.')
    elif isinstance(damping_times, dict) and not isinstance(freqs, dict):
        raise ValueError('Missing frequencies.')
    else:
        delta_t = 1. / qnm_freq_decay(freqs, damping_times, 1./1000)

    if delta_t < min_dt:
        delta_t = min_dt

    return delta_t

def lm_ffinal(freqs, damping_times):
    """Return the maximum f_final of the modes given, with f_final the frequency
    """

        f_max = {}
            f_max[lmn] = qnm_freq_decay(freqs[lmn],
                              damping_times[lmn], 1./1000)
        f_final = max(f_max.values())
    else:
        f_final = qnm_freq_decay(freqs, damping_times, 1./1000)

    if f_final > max_freq:
        f_final = max_freq

    return f_final

def lm_deltaf(damping_times):
    """Return the minimum delta_f of all the modes given, with delta_f given by
    the inverse of the time at which the amplitude of the ringdown falls to
    """

        df = {}
            df[lmn] = 1. / qnm_time_decay(damping_times[lmn], 1./1000)
        delta_f = min(df.values())
    else:
        delta_f = 1. / qnm_time_decay(damping_times, 1./1000)

    return delta_f

def td_output_vector(freqs, damping_times, taper=False,
                     delta_t=None, t_final=None):
    """Return an empty TimeSeries with the appropriate size to fit all
    the quasi-normal modes present in freqs, damping_times
    """

    if not delta_t:
        delta_t = lm_deltat(freqs, damping_times)
    if not t_final:
        t_final = lm_tfinal(damping_times)
    kmax = int(t_final / delta_t) + 1

    # Different modes will have different tapering window-size
    # Find maximum window size to create long enough output vector
    if taper:
        max_tau = max(damping_times.values()) if \
                  isinstance(damping_times, dict) else damping_times
        kmax += int(max_tau/delta_t)

    outplus = TimeSeries(zeros(kmax, dtype=float64), delta_t=delta_t)
    outcross = TimeSeries(zeros(kmax, dtype=float64), delta_t=delta_t)
    if taper:
        # Change epoch of output vector if tapering will be applied
        start = - max_tau
        # To ensure that t=0 is still in output vector
        start -= start % delta_t
        outplus._epoch, outcross._epoch = start, start

    return outplus, outcross

def fd_output_vector(freqs, damping_times, delta_f=None, f_final=None):
    """Return an empty FrequencySeries with the appropriate size to fit all
    """

    if not delta_f:
        delta_f = lm_deltaf(damping_times)
    if not f_final:
        f_final = lm_ffinal(freqs, damping_times)
    kmax = int(f_final / delta_f) + 1

    outplus = FrequencySeries(zeros(kmax, dtype=complex128), delta_f=delta_f)
    outcross = FrequencySeries(zeros(kmax, dtype=complex128), delta_f=delta_f)

# Spherical harmonics and Kerr factor #########################################

def spher_harms(l, m, inclination):
    """Return spherical harmonic polarizations
    """

    # FIXME: we are using spin -2 weighted spherical harmonics for now,
    # when possible switch to spheroidal harmonics.
    Y_lm = lal.SpinWeightedSphericalHarmonic(inclination, 0., -2, l, m).real
    Y_lminusm = lal.SpinWeightedSphericalHarmonic(inclination, 0., -2, l, -m).real
    Y_plus = Y_lm + (-1)**l * Y_lminusm
    Y_cross = Y_lm - (-1)**l * Y_lminusm

    return Y_plus, Y_cross

def Kerr_factor(final_mass, distance):
    """Return the factor final_mass/distance (in dimensionless units) for Kerr
    ringdowns
    """

    # Convert solar masses to meters
    mass = final_mass * lal.MSUN_SI * lal.G_SI / lal.C_SI**2
    # Convert Mpc to meters
    dist = distance * 1e6 * lal.PC_SI

    return mass / dist

# Functions for tapering ######################################################

def apply_taper(tau, amp, phi, delta_t, l=2, m=2, inclination=None):
    """Return a tapering window of the form exp(10*t/tau).
    """

    taper_times = -numpy.arange(0, int(tau/delta_t))[::-1] * delta_t
    if inclination is not None:
        Y_plus, Y_cross = spher_harms(l, m, inclination)
    else:
        Y_plus, Y_cross = 1, 1
    hp_amp = amp * Y_plus * numpy.cos(phi)
    hc_amp = amp * Y_cross * numpy.sin(phi)

    taper_hp = hp_amp * numpy.exp(10*taper_times/tau)
    taper_hc = hc_amp * numpy.exp(10*taper_times/tau)

    return taper_hp, taper_hc

# Functions to generate ringdown waveforms ####################################

######################################################
#### Basic functions to generate damped sinusoid

def td_damped_sinusoid(f_0, tau, amp, phi, delta_t, t_final,
                       l=2, m=2, inclination=None):
    """Return a time domain damped sinusoid (plus and cross polarizations)
    with central frequency f_0, damping time tau, amplitude amp and phase phi.
    The l, m, and inclination parameters are used for the spherical harmonics.
    """

    tlen = int(t_final/delta_t) + 1
    times = numpy.linspace(0, t_final, num=tlen)

    else:

    common_factor = amp * numpy.exp(-times/tau)
    common_angle = (two_pi * f_0 * times) + phi

    hplus = Y_plus * common_factor * numpy.cos(common_angle)
    hcross = Y_cross * common_factor * numpy.sin(common_angle)

    return hplus, hcross

def fd_damped_sinusoid(f_0, tau, amp, phi, delta_f, f_lower, f_final, t_0=0.,
    """Return a frequency domain damped sinusoid (plus and cross polarizations)
    """

    if not f_lower:
        f_lower = delta_f
        kmin = 0
    else:
        kmin = int(f_lower / delta_f)

    # Create output vector with appropriate size
    outplus, outcross = fd_output_vector(f_0, tau, delta_f, f_final)
    freqs = outplus.sample_frequencies[kmin:]

    else:

    denominator = 1 + (4j * pi * freqs * tau) - \
        (4 * pi_sq * (freqs*freqs - f_0*f_0) * tau*tau)
    norm = amp * tau / denominator
    if t_0 != 0:
        time_shift = numpy.exp(-1j * two_pi * freqs * t_0)
        norm *= time_shift
    A1 = (1 + 2j * pi * freqs * tau)
    A2 = two_pi * f_0 * tau

    # Analytical expression for the Fourier transform of the ringdown (damped sinusoid)
    hp_tilde = norm * Y_plus * (A1 * numpy.cos(phi) - A2 * numpy.sin(phi))
    hc_tilde = norm * Y_cross * (A1 * numpy.sin(phi) + A2 * numpy.cos(phi))

    outplus.data[kmin:] = hp_tilde
    outcross.data[kmin:] = hc_tilde

#### Base multi-mode for all approximants

def multimode_base(input_params, domain, freq_tau_approximant=False):
    """Return a superposition of damped sinusoids in either time or frequency
    domains with parameters set by input_params.

    domain : string
        Choose domain of the waveform, either 'td' for time domain
        or 'fd' for frequency domain.
    freq_tau_approximant : {False, bool}, optional
        Choose choose the waveform approximant to use. Either based on
        mass/spin (set to False, default), or on frequencies/damping times
        of the modes (set to True).

    Returns
    -------
        The plus phase of a ringdown with the lm modes specified and
        n overtones in the chosen domain (time or frequency).
        The cross phase of a ringdown with the lm modes specified and
    """
    input_params['lmns'] = format_lmns(input_params['lmns'])
    amps, phis = lm_amps_phases(**input_params)
    if freq_tau_approximant:
        freqs, taus = lm_freqs_taus(**input_params)
        norm = 1.
    else:
        freqs, taus = get_lm_f0tau_allmodes(input_params['final_mass'],
                        input_params['final_spin'], input_params['lmns'])
        norm = Kerr_factor(input_params['final_mass'],
            input_params['distance']) if 'distance' in input_params.keys() \
            else 1.
        for mode in freqs:
            if 'delta_f{}'.format(mode) in input_params:
                freqs[mode] += input_params['delta_f{}'.format(mode)] * freqs[mode]
        for mode in taus:
            if 'delta_tau{}'.format(mode) in input_params:
                taus[mode] += input_params['delta_tau{}'.format(mode)] * taus[mode]

    if domain == 'td':
        outplus, outcross = td_output_vector(freqs, taus,
                            input_params['taper'], input_params['delta_t'],
                            input_params['t_final'])
        for lmn in freqs:
            hplus, hcross = td_damped_sinusoid(freqs[lmn], taus[lmn],
                            amps[lmn], phis[lmn], outplus.delta_t,
                            outplus.sample_times[-1], int(lmn[0]), int(lmn[1]),
                            input_params['inclination'])
            if input_params['taper'] and outplus.delta_t < taus[lmn]:
                taper_hp, taper_hc = apply_taper(taus[lmn], amps[lmn],
                                     phis[lmn], outplus.delta_t, int(lmn[0]),
                                     int(lmn[1]), input_params['inclination'])
                t0 = -int(outplus.start_time * outplus.sample_rate)
                outplus[t0-len(taper_hp):t0].data += taper_hp
                outplus[t0:].data += hplus
                outcross[t0-len(taper_hc):t0].data += taper_hc
                outcross[t0:].data += hcross
            elif input_params['taper'] and outplus.delta_t > taus[lmn]:
                # This mode has taper duration < delta_t, do not apply taper
            else:
                outplus.data += hplus
                outcross.data += hcross
    elif domain == 'fd':
        outplus, outcross = fd_output_vector(freqs, taus,
                            input_params['delta_f'], input_params['f_final'])
            hplus, hcross = fd_damped_sinusoid(freqs[lmn], taus[lmn],
                            amps[lmn], phis[lmn], outplus.delta_f,
                            input_params['f_lower'],
                            input_params['f_final'],
                            l=int(lmn[0]), m=int(lmn[1]),
                            inclination=input_params['inclination'])
            outplus.data += hplus.data
            outcross.data += hcross.data
    else:
        raise ValueError('unrecognised domain argument {}; '
                         'must be either fd or td'.format(domain))

    return norm * outplus, norm * outcross

#### Approximants

def get_td_from_final_mass_spin(template=None, **kwargs):
    """Return time domain ringdown with all the modes specified.

    template : object
        An object that has attached properties. This can be used to substitute
        for keyword arguments. A common example would be a row in an xml table.
    taper : {False, bool}, optional
        Add a rapid ringup with timescale tau/10 at the beginning of the
        waveform to avoid the abrupt turn on of the ringdown.
        Each mode and overtone will have a different taper depending on its tau,
        the final taper being the superposition of all the tapers.
    distance : {None, float}, optional
        Luminosity distance of the system. If specified, the returned ringdown
        will include the Kerr factor (final_mass/distance).
        Mass of the final black hole in solar masses.
    lmns : list
        Desired lmn modes as strings (lm modes available: 22, 21, 33, 44, 55).
        The n specifies the number of overtones desired for the corresponding
        lm pair (maximum n=8).
        Example: lmns = ['223','331'] are the modes 220, 221, 222, and 330
    amp220 : float
        Amplitude of the fundamental 220 mode.  Always required, even if 220
        mode has not been selected. Note that if distance is given,
        this parameter will have a completely different order of magnitude.
        See table II in https://arxiv.org/abs/1107.0854 for an estimate.
    amplmn : float
        Fraction of the amplitude of the lmn overtone relative to the
        fundamental mode, i.e. amplmn/amp220. Provide as many as the number
        of selected subdominant modes.
    philmn : float
        Phase of the lmn overtone, as many as the number of modes. Should also
        include the information from the azimuthal angle, philmn=(phi + m*Phi).
    inclination : float
        Inclination of the system in radians (for the spherical harmonics).
    delta_flmn: {None, float}, optional
        GR deviation for the frequency of the lmn mode. If given, the lmn
        frequency will be converted to new_flmn = flmn + delta_flmn * flmn,
        with flmn the GR predicted value for the corresponding mass and spin.
    delta_taulmn: {None, float}, optional
        GR deviation for the damping time of the lmn mode. If given, the lmn
        tau will be converted to new_taulmn = taulmn + delta_taulmn * taulmn,
        with taulmn the GR predicted value for the corresponding mass and spin.
    delta_t : {None, float}, optional
        The time step used to generate the ringdown.
        If None, it will be set to the inverse of the frequency at which the
        amplitude is 1/1000 of the peak amplitude (the minimum of all modes).
    t_final : {None, float}, optional
        The ending time of the output frequency series.
        If None, it will be set to the time at which the amplitude
        is 1/1000 of the peak amplitude (the maximum of all modes).

    Returns
    -------
        n overtones in time domain.
    """

    input_params = props(template, mass_spin_required_args, td_args, **kwargs)

    return multimode_base(input_params, domain='td')

def get_fd_from_final_mass_spin(template=None, **kwargs):
    """Return frequency domain ringdown with all the modes specified.

    lmns : list
    delta_f : {None, float}, optional
        The frequency step used to generate the ringdown (not to be confused
        with the delta_flmn parameter that simulates GR violations).
        If None, it will be set to the inverse of the time at which the
    f_lower : {None, float}, optional
        The starting frequency of the output frequency series.
        If None, it will be set to delta_f.
    f_final : {None, float}, optional
        The ending frequency of the output frequency series.
        If None, it will be set to the frequency at which the amplitude

    Returns
    -------
    hplustile : FrequencySeries
        n overtones in frequency domain.
    hcrosstilde : FrequencySeries
    """

    input_params = props(template, mass_spin_required_args, fd_args, **kwargs)

    return multimode_base(input_params, domain='fd')

def get_td_from_freqtau(template=None, **kwargs):

    lmns : list
    f_lmn : float
        Central frequency of the lmn overtone, as many as number of modes.
    tau_lmn : float
        Damping time of the lmn overtone, as many as number of modes.
        Amplitude of the fundamental 220 mode. Note that if distance is given,
        Always required, even if 220 mode has not been selected.
        If None, the spherical harmonics will be set to 1.

    Returns
    -------
    """

    input_params = props(template, freqtau_required_args, td_args, **kwargs)

    return multimode_base(input_params, domain='td', freq_tau_approximant=True)

def get_fd_from_freqtau(template=None, **kwargs):

    lmns : list
        Amplitude of the fundamental 220 mode. Always required, even if 220
        mode has not been selected.
        include the information from the azimuthal angle (phi + m*Phi).
    inclination : {None, float}, optional
        Inclination of the system in radians. If None, the spherical harmonics
        will be set to 1.
        The frequency step used to generate the ringdown.

    Returns
    -------
    hplustilde : FrequencySeries
    """

    input_params = props(template, freqtau_required_args, fd_args, **kwargs)

    return multimode_base(input_params, domain='fd', freq_tau_approximant=True)

# Approximant names ###########################################################
ringdown_fd_approximants = {'FdQNMfromFinalMassSpin': get_fd_from_final_mass_spin,
                            'FdQNMfromFreqTau': get_fd_from_freqtau}
ringdown_td_approximants = {'TdQNMfromFinalMassSpin': get_td_from_final_mass_spin,
                            'TdQNMfromFreqTau': get_td_from_freqtau}

""" Generation of sine-Gaussian bursty type things
"""

def fd_sine_gaussian(amp, quality, central_frequency, fmin, fmax, delta_f):
    """ Generate a Fourier domain sine-Gaussian

    amp: float
        Amplitude of the sine-Gaussian
    quality: float
        The quality factor
    central_frequency: float
        The central frequency of the sine-Gaussian
    fmin: float
        The minimum frequency to generate the sine-Gaussian. This determines
        the length of the output vector.
    fmax: float
        The maximum frequency to generate the sine-Gaussian
        The size of the frequency step

    Returns
    -------
    sg: pycbc.types.Frequencyseries
        A Fourier domain sine-Gaussian
    """
    kmin = int(round(fmin / delta_f))
    kmax = int(round(fmax / delta_f))
    f = numpy.arange(kmin, kmax) * delta_f
    tau = quality / 2 / numpy.pi / central_frequency
    A = amp * numpy.pi ** 0.5 / 2 * tau
    d = A * numpy.exp(-(numpy.pi  * tau  * (f - central_frequency))**2.0)
    d *= (1 + numpy.exp(-quality ** 2.0 * f / central_frequency))
    v = numpy.zeros(kmax, dtype=numpy.complex128)
    v[kmin:kmax] = d[:]
    return pycbc.types.FrequencySeries(v, delta_f=delta_f)

#  Adapted from code in LALSimInspiralTaylorF2.c
#
#  Copyright (C) 2007 Jolien Creighton, B.S. Sathyaprakash, Thomas Cokelaer
#  Copyright (C) 2012 Leo Singer, Alex Nitz
#
#  This program is free software you can redistribute it and/or modify
#  the Free Software Foundation either version 2 of the License, or
#
#  but WITHOUT ANY WARRANTY without even the implied warranty of
#
#  along with with program see the file COPYING. If not, write to the

"""This module contains functions for generating common SPA template precalculated
   vectors.
"""
from math import sqrt, log
import numpy, lal, lalsimulation, pycbc.pnutils
from pycbc.types import FrequencySeries, Array, complex64, float32, zeros
from pycbc.waveform.utils import ceilpow2

def findchirp_chirptime(m1, m2, fLower, porder):
    # variables used to compute chirp time
    m1 = float(m1)
    m2 = float(m2)
    m = m1 + m2
    eta = m1 * m2 / m / m
    c0T = c2T = c3T = c4T = c5T = c6T = c6LogT = c7T = 0.

    # All implemented option
    if porder == -1:
        porder = 7

    if porder >= 7:
        c7T = lal.PI * (14809.0 * eta * eta / 378.0 - 75703.0 * eta / 756.0 - 15419335.0 / 127008.0)

    if porder >= 6:
        c6T = lal.GAMMA * 6848.0 / 105.0 - 10052469856691.0 / 23471078400.0 +\
            lal.PI * lal.PI * 128.0 / 3.0 + \
            eta * (3147553127.0 / 3048192.0 - lal.PI * lal.PI * 451.0 / 12.0) -\
            eta * eta * 15211.0 / 1728.0 + eta * eta * eta * 25565.0 / 1296.0 +\
            eta * eta * eta * 25565.0 / 1296.0 + numpy.log(4.0) * 6848.0 / 105.0
        c6LogT = 6848.0 / 105.0

    if porder >= 5:
        c5T = 13.0 * lal.PI * eta / 3.0 - 7729.0 * lal.PI / 252.0

    if porder >= 4:
        c4T = 3058673.0 / 508032.0 + eta * (5429.0 / 504.0 + eta * 617.0 / 72.0)
        c3T = -32.0 * lal.PI / 5.0
        c2T = 743.0 / 252.0 + eta * 11.0 / 3.0
        c0T = 5.0 * m * lal.MTSUN_SI / (256.0 * eta)

    # This is the PN parameter v evaluated at the lower freq. cutoff
    xT = pow (lal.PI * m * lal.MTSUN_SI * fLower, 1.0 / 3.0)
    x2T = xT * xT
    x3T = xT * x2T
    x4T = x2T * x2T
    x5T = x2T * x3T
    x6T = x3T * x3T
    x7T = x3T * x4T
    x8T = x4T * x4T

    # Computes the chirp time as tC = t(v_low)
    # tC = t(v_low) - t(v_upper) would be more
    # correct, but the difference is negligble.

    # This formula works for any PN order, because
    # higher order coeffs will be set to zero.
    return c0T * (1 + c2T * x2T + c3T * x3T + c4T * x4T + c5T * x5T + (c6T + c6LogT * numpy.log(xT)) * x6T + c7T * x7T) / x8T

def spa_length_in_time(**kwds):
    """
    Returns the length in time of the template,
    based on the masses, PN order, and low-frequency
    cut-off.
    """
    m1 = kwds['mass1']
    m2 = kwds['mass2']
    flow = kwds['f_lower']
    porder = int(kwds['phase_order'])

    # For now, we call the swig-wrapped function below in
    # lalinspiral.  Eventually would be nice to replace this
    # with a function using PN coeffs from lalsimulation.
    return findchirp_chirptime(m1, m2, flow, porder)

def spa_amplitude_factor(**kwds):

    _, eta = pycbc.pnutils.mass1_mass2_to_mchirp_eta(m1, m2)

    FTaN = 32.0 * eta*eta / 5.0
    dETaN = 2 * -eta/2.0

    M = m1 + m2

    m_sec = M * lal.MTSUN_SI
    piM = lal.PI * m_sec

    amp0 = 4. * m1 * m2 / (1e6 * lal.PC_SI ) * lal.MRSUN_SI * lal.MTSUN_SI * sqrt(lal.PI/12.0)

    fac = numpy.sqrt( -dETaN / FTaN) * amp0 * (piM ** (-7.0/6.0))
    return -fac

_prec = None
def spa_tmplt_precondition(length, delta_f, kmin=0):
    """Return the amplitude portion of the TaylorF2 approximant, used to precondition
    the strain data. The result is cached, and so should not be modified only read.
    """
    global _prec
    if _prec is None or _prec.delta_f != delta_f or len(_prec) < length:
        v = numpy.arange(0, (kmin+length*2), 1.0) * delta_f
        v = numpy.power(v[1:len(v)], -7.0/6.0)
        _prec = FrequencySeries(v, delta_f=delta_f, dtype=float32)
    return _prec[kmin:kmin + length]

def spa_tmplt_norm(psd, length, delta_f, f_lower):
    amp = spa_tmplt_precondition(length, delta_f)
    k_min = int(f_lower / delta_f)
    sigma = (amp[k_min:length].numpy() ** 2.0 / psd[k_min:length].numpy())
    norm_vec = numpy.zeros(length)
    norm_vec[k_min:length] = sigma.cumsum() * 4 * delta_f
    return norm_vec

def spa_tmplt_end(**kwds):
    return pycbc.pnutils.f_SchwarzISCO(kwds['mass1']+kwds['mass2'])

def spa_distance(psd, mass1, mass2, lower_frequency_cutoff, snr=8):
    """ Return the distance at a given snr (default=8) of the SPA TaylorF2
    template.
    """
    kend = int(spa_tmplt_end(mass1=mass1, mass2=mass2) / psd.delta_f)
    norm1 = spa_tmplt_norm(psd, len(psd), psd.delta_f, lower_frequency_cutoff)
    norm2 = (spa_amplitude_factor(mass1=mass1, mass2=mass2)) ** 2.0

    if kend >= len(psd):
        kend = len(psd) - 2
    return sqrt(norm1[kend] * norm2) / snr

@schemed("pycbc.waveform.spa_tmplt_")
def spa_tmplt_engine(htilde, kmin, phase_order, delta_f, piM, pfaN,
                     pfa2, pfa3, pfa4, pfa5, pfl5,
                     pfa6, pfl6, pfa7, amp_factor):
    """ Calculate the spa tmplt phase
    """

def spa_tmplt(**kwds):
    """ Generate a minimal TaylorF2 approximant with optimizations for the sin/cos
    """
    # Pull out the input arguments
    distance = kwds['distance']
    mass1 = kwds['mass1']
    mass2 = kwds['mass2']
    s1z = kwds['spin1z']
    s2z = kwds['spin2z']
    phase_order = int(kwds['phase_order'])
    #amplitude_order = int(kwds['amplitude_order'])
    spin_order = int(kwds['spin_order'])

    if 'out' in kwds:
        out = kwds['out']
    else:
        out = None

    amp_factor = spa_amplitude_factor(mass1=mass1, mass2=mass2) / distance

    lal_pars = lal.CreateDict()
    if phase_order != -1:
        lalsimulation.SimInspiralWaveformParamsInsertPNPhaseOrder(
            lal_pars, phase_order)

    if spin_order != -1:
        lalsimulation.SimInspiralWaveformParamsInsertPNSpinOrder(
            lal_pars, spin_order)

    #Calculate the PN terms
    phasing = lalsimulation.SimInspiralTaylorF2AlignedPhasing(
                                    float(mass1), float(mass2),
                                    float(s1z), float(s2z),
                                    lal_pars)

    pfaN = phasing.v[0]
    pfa2 = phasing.v[2] / pfaN
    pfa3 = phasing.v[3] / pfaN
    pfa4 = phasing.v[4] / pfaN
    pfa5 = phasing.v[5] / pfaN
    pfa6 = (phasing.v[6] - phasing.vlogv[6] * log(4)) / pfaN
    pfa7 = phasing.v[7] / pfaN

    pfl5 = phasing.vlogv[5] / pfaN
    pfl6 = phasing.vlogv[6] / pfaN

    piM = lal.PI * (mass1 + mass2) * lal.MTSUN_SI

    if 'sample_points' not in kwds:
        f_lower = kwds['f_lower']
        delta_f = kwds['delta_f']
        kmin = int(f_lower / float(delta_f))
        vISCO = 1. / sqrt(6.)
        fISCO = vISCO * vISCO * vISCO / piM

        if 'f_upper' in kwds:
            fISCO = kwds['f_upper']

        kmax = int(fISCO / delta_f)
        f_max = ceilpow2(fISCO)
        n = int(f_max / delta_f) + 1

        if not out:
            htilde = FrequencySeries(zeros(n, dtype=numpy.complex64), delta_f=delta_f, copy=False)
        else:
            if type(out) is not Array:
                raise TypeError("Output must be an instance of Array")
            if len(out) < kmax:
                kmax = len(out)
            if out.dtype != complex64:
                raise TypeError("Output array is the wrong dtype")
            htilde = FrequencySeries(out, delta_f=delta_f, copy=False)

        spa_tmplt_engine(htilde[kmin:kmax], kmin, phase_order,
                         delta_f, piM, pfaN,
                         pfa2, pfa3, pfa4, pfa5, pfl5,
                         pfa6, pfl6, pfa7, amp_factor)
    else:
        from .spa_tmplt_cpu import spa_tmplt_inline_sequence
        htilde = numpy.empty(len(kwds['sample_points']), dtype=numpy.complex64)
        spa_tmplt_inline_sequence(
            piM, pfaN, pfa2, pfa3, pfa4, pfa5, pfl5, pfa6, pfl6, pfa7,
            amp_factor, kwds['sample_points'], htilde)
    return htilde

"""Generate core-collapse supernovae waveform for core bounce and
subsequent postbounce oscillations.
"""

_pc_dict = {}

def get_corecollapse_bounce(**kwargs):
    """ Generates core bounce and postbounce waveform by using principal
    component basis vectors from a .hdf file. The waveform parameters are the
    coefficients of the principal components and the distance. The number of
    principal components used can also be varied.
    """

    try:
        principal_components = _pc_dict['principal_components']
        with h5py.File(kwargs['principal_components_file'], 'r') as pc_file:
            principal_components = numpy.array(pc_file['principal_components'])
            _pc_dict['principal_components'] = principal_components

    if 'coefficients_array' in kwargs:
        coefficients_array = kwargs['coefficients_array']
    else:
        coeffs_keys = [x for x in kwargs if x.startswith('coeff_')]
        coeffs_keys = numpy.sort(numpy.array(coeffs_keys))
        coefficients_array = numpy.array([kwargs[x] for x in coeffs_keys])

    no_of_pcs = int(kwargs['no_of_pcs'])
    coefficients_array = coefficients_array[:no_of_pcs]
    principal_components = principal_components[:no_of_pcs]

    pc_len = len(principal_components)
    assert len(coefficients_array) == pc_len

    distance = kwargs['distance']
    mpc_conversion = 3.08567758128e+22
    distance *= mpc_conversion

    strain = numpy.dot(coefficients_array, principal_components) / distance
    delta_t = kwargs['delta_t']
    outhp = TimeSeries(strain, delta_t=delta_t)
    outhc = TimeSeries(numpy.zeros(len(strain)), delta_t=delta_t)
    return outhp, outhc

supernovae_td_approximants = {'CoreCollapseBounce': get_corecollapse_bounce}

# Copyright (C) 2013  Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
"""This module contains convenience utilities for manipulating waveforms
"""
from pycbc.types import TimeSeries, FrequencySeries, Array, float32, float64, complex_same_precision_as, real_same_precision_as
import lal
from math import frexp
from scipy import signal

def ceilpow2(n):
    """convenience function to determine a power-of-2 upper frequency limit"""
    signif,exponent = frexp(n)
    if (signif < 0):
        return 1;
    if (signif == 0.5):
        exponent -= 1;
    return (1) << exponent;

def coalign_waveforms(h1, h2, psd=None,
                      low_frequency_cutoff=None,
                      high_frequency_cutoff=None,
                      resize=True):
    """ Return two time series which are aligned in time and phase.

    The alignment is only to the nearest sample point and all changes to the
    phase are made to the first input waveform. Waveforms should not be split
    accross the vector boundary. If it is, please use roll or cyclic time shift
    to ensure that the entire signal is contiguous in the time series.

    h1: pycbc.types.TimeSeries
        The first waveform to align.
    h2: pycbc.types.TimeSeries
        The second waveform to align.
    psd: {None, pycbc.types.FrequencySeries}
        A psd to weight the alignment
    low_frequency_cutoff: {None, float}
        The low frequency cutoff to weight the matching in Hz.
    high_frequency_cutoff: {None, float}
        The high frequency cutoff to weight the matching in Hz.
    resize: Optional, {True, boolean}
        If true, the vectors will be resized to match each other. If false,
        they must be the same length and even in length

    Returns
    -------
        The shifted waveform to align with h2
    h2: pycbc.type.TimeSeries
        The resized (if necessary) waveform to align with h1.
    """
    from pycbc.filter import matched_filter
    mlen = ceilpow2(max(len(h1), len(h2)))

    h1 = h1.copy()
    h2 = h2.copy()

    if resize:
        h1.resize(mlen)
        h2.resize(mlen)
    elif len(h1) != len(h2) or len(h2) % 2 != 0:
        raise ValueError("Time series must be the same size and even if you do "
                         "not allow resizing")

    snr = matched_filter(h1, h2, psd=psd,
                         high_frequency_cutoff=high_frequency_cutoff)

    _, l =  snr.abs_max_loc()
    rotation =  snr[l] / abs(snr[l])
    h1 = (h1.to_frequencyseries() * rotation).to_timeseries()
    h1.roll(l)

    h1 = TimeSeries(h1, delta_t=h2.delta_t, epoch=h2.start_time)
    return h1, h2

def phase_from_frequencyseries(htilde, remove_start_phase=True):
    """Returns the phase from the given frequency-domain waveform. This assumes
    that the waveform has been sampled finely enough that the phase cannot
    change by more than pi radians between each step.

        The waveform to get the phase for; must be a complex frequency series.
    remove_start_phase : {True, bool}
        Subtract the initial phase before returning.

    Returns
    -------
        The phase of the waveform as a function of frequency.
    """
    p = numpy.unwrap(numpy.angle(htilde.data)).astype(
    if remove_start_phase:
        p += -p[0]
    return FrequencySeries(p, delta_f=htilde.delta_f, epoch=htilde.epoch,
        copy=False)

def amplitude_from_frequencyseries(htilde):
    """Returns the amplitude of the given frequency-domain waveform as a
    FrequencySeries.

        The waveform to get the amplitude of.

    Returns
    -------
        The amplitude of the waveform as a function of frequency.
    """
    amp = abs(htilde.data).astype(real_same_precision_as(htilde))
    return FrequencySeries(amp, delta_f=htilde.delta_f, epoch=htilde.epoch,

def time_from_frequencyseries(htilde, sample_frequencies=None,
        discont_threshold=0.99*numpy.pi):
    """Computes time as a function of frequency from the given
    frequency-domain waveform. This assumes the stationary phase
    approximation. Any frequencies lower than the first non-zero value in
    htilde are assigned the time at the first non-zero value. Times for any
    frequencies above the next-to-last non-zero value in htilde will be
    assigned the time of the next-to-last non-zero value.

    .. note::
        Some waveform models (e.g., `SEOBNRv2_ROM_DoubleSpin`) can have
        discontinuities in the phase towards the end of the waveform due to
        numerical error. We therefore exclude any points that occur after a
        discontinuity in the phase, as the time estimate becomes untrustworthy
        beyond that point. What determines a discontinuity in the phase is set
        by the `discont_threshold`. To turn this feature off, just set
        `discont_threshold` to a value larger than pi (due to the unwrapping
        of the phase, no two points can have a difference > pi).

        The waveform to get the time evolution of; must be complex.
        The frequencies at which the waveform is sampled. If None, will
        retrieve from ``htilde.sample_frequencies``.
    discont_threshold : {0.99*pi, float}
        If the difference in the phase changes by more than this threshold,
        it is considered to be a discontinuity. Default is 0.99*pi.

    Returns
    -------
        The time evolution of the waveform as a function of frequency.
    """
    phase = phase_from_frequencyseries(htilde).data
    dphi = numpy.diff(phase)
    time = -dphi / (2.*numpy.pi*numpy.diff(sample_frequencies))
    nzidx = numpy.nonzero(abs(htilde.data))[0]
    kmin, kmax = nzidx[0], nzidx[-2]
    # exclude everything after a discontinuity
    discont_idx = numpy.where(abs(dphi[kmin:]) >= discont_threshold)[0]
    if discont_idx.size != 0:
        kmax = min(kmax, kmin + discont_idx[0]-1)
    time[:kmin] = time[kmin]
    time[kmax:] = time[kmax]
    return FrequencySeries(time.astype(real_same_precision_as(htilde)),
        delta_f=htilde.delta_f, epoch=htilde.epoch,

def phase_from_polarizations(h_plus, h_cross, remove_start_phase=True):
    """Return gravitational wave phase

    Return the gravitation-wave phase from the h_plus and h_cross
    polarizations of the waveform. The returned phase is always
    positive and increasing with an initial phase of 0.

    h_plus : TimeSeries
        An PyCBC TmeSeries vector that contains the plus polarization of the
        gravitational waveform.
    h_cross : TimeSeries
        A PyCBC TmeSeries vector that contains the cross polarization of the

    Returns
    -------
    GWPhase : TimeSeries
        A TimeSeries containing the gravitational wave phase.

    Examples
    --------s
    >>> from pycbc.waveform import get_td_waveform, phase_from_polarizations
    >>> hp, hc = get_td_waveform(approximant="TaylorT4", mass1=10, mass2=10,
                         f_lower=30, delta_t=1.0/4096)
    >>> phase = phase_from_polarizations(hp, hc)

    """
    p = numpy.unwrap(numpy.arctan2(h_cross.data, h_plus.data)).astype(
        real_same_precision_as(h_plus))
        p += -p[0]
    return TimeSeries(p, delta_t=h_plus.delta_t, epoch=h_plus.start_time,

def amplitude_from_polarizations(h_plus, h_cross):
    """Return gravitational wave amplitude

    Return the gravitation-wave amplitude from the h_plus and h_cross
    polarizations of the waveform.

    Returns
    -------
    GWAmplitude : TimeSeries
        A TimeSeries containing the gravitational wave amplitude.

    Examples
    --------
    >>> amp = amplitude_from_polarizations(hp, hc)

    """
    amp = (h_plus.squared_norm() + h_cross.squared_norm()) ** (0.5)
    return TimeSeries(amp, delta_t=h_plus.delta_t, epoch=h_plus.start_time)

def frequency_from_polarizations(h_plus, h_cross):
    """Return gravitational wave frequency

    Return the gravitation-wave frequency as a function of time
    from the h_plus and h_cross polarizations of the waveform.
    It is 1 bin shorter than the input vectors and the sample times
    are advanced half a bin.

        A PyCBC TimeSeries vector that contains the plus polarization of the
        A PyCBC TimeSeries vector that contains the cross polarization of the

    Returns
    -------
    GWFrequency : TimeSeries
        A TimeSeries containing the gravitational wave frequency as a function
        of time.

    Examples
    --------
    >>> freq = frequency_from_polarizations(hp, hc)

    """
    phase = phase_from_polarizations(h_plus, h_cross)
    freq = numpy.diff(phase) / ( 2 * lal.PI * phase.delta_t )
    start_time = phase.start_time + phase.delta_t / 2
    return TimeSeries(freq.astype(real_same_precision_as(h_plus)),
        delta_t=phase.delta_t, epoch=start_time)

# map between tapering string in sim_inspiral table or inspiral
# code option and lalsimulation constants
taper_map = {
    'TAPER_NONE'    : None,
    'TAPER_START'   : sim.SIM_INSPIRAL_TAPER_START,
    'start'         : sim.SIM_INSPIRAL_TAPER_START,
    'TAPER_END'     : sim.SIM_INSPIRAL_TAPER_END,
    'end'           : sim.SIM_INSPIRAL_TAPER_END,
    'TAPER_STARTEND': sim.SIM_INSPIRAL_TAPER_STARTEND,
    'startend'      : sim.SIM_INSPIRAL_TAPER_STARTEND
}

taper_func_map = {
    numpy.dtype(float32): sim.SimInspiralREAL4WaveTaper,
    numpy.dtype(float64): sim.SimInspiralREAL8WaveTaper
}

def taper_timeseries(tsdata, tapermethod=None, return_lal=False):
    """
    Taper either or both ends of a time series using wrapped
    LALSimulation functions

    tsdata : TimeSeries
        Series to be tapered, dtype must be either float32 or float64
    tapermethod : string
        Should be one of ('TAPER_NONE', 'TAPER_START', 'TAPER_END',
        'TAPER_STARTEND', 'start', 'end', 'startend') - NB 'TAPER_NONE' will
        not change the series!
    return_lal : Boolean
        If True, return a wrapped LAL time series object, else return a
        PyCBC time series.
    """
    if tapermethod is None:
        raise ValueError("Must specify a tapering method (function was called"
                         "with tapermethod=None)")
    if tapermethod not in taper_map.keys():
        raise ValueError("Unknown tapering method %s, valid methods are %s" % \
                         (tapermethod, ", ".join(taper_map.keys())))
    if tsdata.dtype not in (float32, float64):
        raise TypeError("Strain dtype must be float32 or float64, not "
                    + str(tsdata.dtype))
    taper_func = taper_func_map[tsdata.dtype]
    # make a LAL TimeSeries to pass to the LALSim function
    ts_lal = tsdata.astype(tsdata.dtype).lal()
    if taper_map[tapermethod] is not None:
        taper_func(ts_lal.data, taper_map[tapermethod])
    if return_lal:
        return ts_lal
    else:
        return TimeSeries(ts_lal.data.data[:], delta_t=ts_lal.deltaT,
                          epoch=ts_lal.epoch)

@schemed("pycbc.waveform.utils_")
def apply_fseries_time_shift(htilde, dt, kmin=0, copy=True):
    """Shifts a frequency domain waveform in time. The waveform is assumed to
    be sampled at equal frequency intervals.
    """

def apply_fd_time_shift(htilde, shifttime, kmin=0, fseries=None, copy=True):
    """Shifts a frequency domain waveform in time. The shift applied is
    shiftime - htilde.epoch.

        The waveform frequency series.
    shifttime : float
        The time to shift the frequency series to.
    kmin : {0, int}
        The starting index of htilde to apply the time shift. Default is 0.
    fseries : {None, numpy array}
        The frequencies of each element in htilde. This is only needed if htilde is not
        sampled at equal frequency steps.
    copy : {True, bool}
        Make a copy of htilde before applying the time shift. If False, the time
        shift will be applied to htilde's data.

    Returns
    -------
        A frequency series with the waveform shifted to the new time. If makecopy
        is True, will be a new frequency series; if makecopy is False, will be
        the same as htilde.
    """
    dt = float(shifttime - htilde.epoch)
    if dt == 0.:
        # no shift to apply, just copy if desired
        if copy:
            htilde = 1. * htilde
    elif isinstance(htilde, FrequencySeries):
        # FrequencySeries means equally sampled in frequency, use faster shifting
        htilde = apply_fseries_time_shift(htilde, dt, kmin=kmin, copy=copy)
    else:
        if fseries is None:
            fseries = htilde.sample_frequencies.numpy()
        shift = Array(numpy.exp(-2j*numpy.pi*dt*fseries),
                    dtype=complex_same_precision_as(htilde))
        if copy:
        htilde *= shift

def td_taper(out, start, end, beta=8, side='left'):
    """Applies a taper to the given TimeSeries.

    A half-kaiser window is used for the roll-off.

    out : TimeSeries
        The ``TimeSeries`` to taper.
        The time (in s) to start the taper window.

    end : float
        The time (in s) to end the taper window.
    beta : int, optional
        The beta parameter to use for the Kaiser window. See
        ``scipy.signal.kaiser`` for details. Default is 8.
    side : {'left', 'right'}
        The side to apply the taper to. If ``'left'`` (``'right'``), the taper
        will roll up (down) between ``start`` and ``end``, with all values
        before ``start`` (after ``end``) set to zero. Default is ``'left'``.

    Returns
    -------
    TimeSeries
        The tapered time series.
    """
    out = out.copy()
    width = end - start
    winlen = 2 * int(width / out.delta_t)
    window = Array(signal.get_window(('kaiser', beta), winlen))
    xmin = int((start - out.start_time) / out.delta_t)
    xmax = xmin + winlen//2
        out[xmin:xmax] *= window[:winlen//2]
        if xmin > 0:
            out[:xmin].clear()
        out[xmin:xmax] *= window[winlen//2:]
        if xmax < len(out):
            out[xmax:].clear()
    else:
        raise ValueError("unrecognized side argument {}".format(side))
    return out

def fd_taper(out, start, end, beta=8, side='left'):
    """Applies a taper to the given FrequencySeries.

        The ``FrequencySeries`` to taper.
        The frequency (in Hz) to start the taper window.
    end : float
        The frequency (in Hz) to end the taper window.

    Returns
    -------
        The tapered frequency series.
    """
    winlen = 2 * int(width / out.delta_f)
    kmin = int(start / out.delta_f)
    kmax = kmin + winlen//2
        out[kmin:kmax] *= window[:winlen//2]
        out[:kmin] *= 0.
        out[kmin:kmax] *= window[winlen//2:]
        out[kmax:] *= 0.
    else:
    return out

def fd_to_td(htilde, delta_t=None, left_window=None, right_window=None,
          left_beta=8, right_beta=8):
    """Converts a FD waveform to TD.

    A window can optionally be applied using ``fd_taper`` to the left or right
    side of the waveform before being converted to the time domain.

        The waveform to convert.
    delta_t : float, optional
        Make the returned time series have the given ``delta_t``.
    left_window : tuple of float, optional
        A tuple giving the start and end frequency of the FD taper to apply
        on the left side. If None, no taper will be applied on the left.
    right_window : tuple of float, optional
        on the right side. If None, no taper will be applied on the right.
    left_beta : int, optional
        The beta parameter to use for the left taper. See ``fd_taper`` for
        details. Default is 8.
    right_beta : int, optional
        The beta parameter to use for the right taper. Default is 8.

    Returns
    -------
        The time-series representation of ``htilde``.
    """
    if left_window is not None:
        start, end = left_window
        htilde = fd_taper(htilde, start, end, side='left', beta=left_beta)
    if right_window is not None:
        start, end = right_window
        htilde = fd_taper(htilde, start, end, side='right', beta=right_beta)
    return htilde.to_timeseries(delta_t=delta_t)

#
#
#

#
#
#                                   Preamble
#
#
"""Convenience functions to genenerate gravitational wave templates and
waveforms.
"""

import os
import lal, lalsimulation, numpy, copy
from pycbc.fft import fft
from pycbc.waveform import utils as wfutils
from pycbc.filter import interpolate_complex_frequency, resample_to_delta_t
from .spa_tmplt import spa_tmplt, spa_tmplt_norm, spa_tmplt_end, \
                      spa_tmplt_precondition, spa_amplitude_factor, \
                      spa_length_in_time
from six.moves import range as xrange

class NoWaveformError(Exception):
    """This should be raised if generating a waveform would just result in all
    zeros being returned, e.g., if a requested `f_final` is <= `f_lower`.
    """
    pass

class FailedWaveformError(Exception):
    """This should be raised if a waveform fails to generate.
    """
    pass

# If this is set to True, waveform generation codes will try to regenerate
# waveforms with known failure conditions to try to avoid the failure. For
# example SEOBNRv3 waveforms would be regenerated with double the sample rate.
# If this is set to False waveform failures will always raise exceptions
fail_tolerant_waveform_generation = True

default_args = \
    (parameters.fd_waveform_params.default_dict() +
     parameters.td_waveform_params).default_dict()

default_sgburst_args = {'eccentricity':0, 'polarization':0}
sgburst_required_args = ['q','frequency','hrss']

# td, fd, filter waveforms generated on the CPU
_lalsim_td_approximants = {}
_lalsim_fd_approximants = {}
_lalsim_enum = {}
_lalsim_sgburst_approximants = {}

def _check_lal_pars(p):
    """ Create a laldict object from the dictionary of waveform parameters

    p: dictionary
        The dictionary of lalsimulation paramaters

    Returns
    -------
    laldict: LalDict
        The lal type dictionary to pass to the lalsimulation waveform functions.
    """
    #nonGRparams can be straightforwardly added if needed, however they have to
    # be invoked one by one
    if p['phase_order']!=-1:
        lalsimulation.SimInspiralWaveformParamsInsertPNPhaseOrder(lal_pars,int(p['phase_order']))
    if p['amplitude_order']!=-1:
        lalsimulation.SimInspiralWaveformParamsInsertPNAmplitudeOrder(lal_pars,int(p['amplitude_order']))
    if p['spin_order']!=-1:
        lalsimulation.SimInspiralWaveformParamsInsertPNSpinOrder(lal_pars,int(p['spin_order']))
    if p['tidal_order']!=-1:
        lalsimulation.SimInspiralWaveformParamsInsertPNTidalOrder(lal_pars, p['tidal_order'])
    if p['eccentricity_order']!=-1:
        lalsimulation.SimInspiralWaveformParamsInsertPNEccentricityOrder(lal_pars, p['eccentricity_order'])
    if p['lambda1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalLambda1(lal_pars, p['lambda1'])
    if p['lambda2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalLambda2(lal_pars, p['lambda2'])
    if p['lambda_octu1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalOctupolarLambda1(lal_pars, p['lambda_octu1'])
    if p['lambda_octu2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalOctupolarLambda2(lal_pars, p['lambda_octu2'])
    if p['quadfmode1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalQuadrupolarFMode1(lal_pars, p['quadfmode1'])
    if p['quadfmode2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalQuadrupolarFMode2(lal_pars, p['quadfmode2'])
    if p['octufmode1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalOctupolarFMode1(lal_pars, p['octufmode1'])
    if p['octufmode2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertTidalOctupolarFMode2(lal_pars, p['octufmode2'])
    if p['dquad_mon1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertdQuadMon1(lal_pars, p['dquad_mon1'])
    if p['dquad_mon2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertdQuadMon2(lal_pars, p['dquad_mon2'])
    if p['numrel_data']:
        lalsimulation.SimInspiralWaveformParamsInsertNumRelData(lal_pars, str(p['numrel_data']))
    if p['modes_choice']:
        lalsimulation.SimInspiralWaveformParamsInsertModesChoice(lal_pars, p['modes_choice'])
    if p['frame_axis']:
        lalsimulation.SimInspiralWaveformParamsInsertFrameAxis(lal_pars, p['frame_axis'])
    if p['side_bands']:
        lalsimulation.SimInspiralWaveformParamsInsertSideband(lal_pars, p['side_bands'])
    if p['mode_array'] is not None:
        ma = lalsimulation.SimInspiralCreateModeArray()
        for l,m in p['mode_array']:
            lalsimulation.SimInspiralModeArrayActivateMode(ma, l, m)
        lalsimulation.SimInspiralWaveformParamsInsertModeArray(lal_pars, ma)
    #TestingGR parameters:
    if p['dchi0'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi0(lal_pars,p['dchi0'])
    if p['dchi1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi1(lal_pars,p['dchi1'])
    if p['dchi2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi2(lal_pars,p['dchi2'])
    if p['dchi3'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi3(lal_pars,p['dchi3'])
    if p['dchi4'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi4(lal_pars,p['dchi4'])
    if p['dchi5'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi5(lal_pars,p['dchi5'])
    if p['dchi5l'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi5L(lal_pars,p['dchi5l'])
    if p['dchi6'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi6(lal_pars,p['dchi6'])
    if p['dchi6l'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi6L(lal_pars,p['dchi6l'])
    if p['dchi7'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDChi7(lal_pars,p['dchi7'])
    if p['dalpha1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDAlpha1(lal_pars,p['dalpha1'])
    if p['dalpha2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDAlpha2(lal_pars,p['dalpha2'])
    if p['dalpha3'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDAlpha3(lal_pars,p['dalpha3'])
    if p['dalpha4'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDAlpha4(lal_pars,p['dalpha4'])
    if p['dalpha5'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDAlpha5(lal_pars,p['dalpha5'])
    if p['dbeta1'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDBeta1(lal_pars,p['dbeta1'])
    if p['dbeta2'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDBeta2(lal_pars,p['dbeta2'])
    if p['dbeta3'] is not None:
        lalsimulation.SimInspiralWaveformParamsInsertNonGRDBeta3(lal_pars,p['dbeta3'])
    return lal_pars

def _lalsim_td_waveform(**p):
    lal_pars = _check_lal_pars(p)
    try:
        hp1, hc1 = lalsimulation.SimInspiralChooseTDWaveform(
               float(pnutils.solar_mass_to_kg(p['mass1'])),
               float(pnutils.solar_mass_to_kg(p['mass2'])),
               float(p['spin1x']), float(p['spin1y']), float(p['spin1z']),
               float(p['spin2x']), float(p['spin2y']), float(p['spin2z']),
               pnutils.megaparsecs_to_meters(float(p['distance'])),
               float(p['inclination']), float(p['coa_phase']),
               float(p['long_asc_nodes']), float(p['eccentricity']), float(p['mean_per_ano']),
               float(p['delta_t']), float(p['f_lower']), float(p['f_ref']),
               lal_pars,
               _lalsim_enum[p['approximant']])
    except RuntimeError:
        if not fail_tolerant_waveform_generation:
            raise
        # For some cases failure modes can occur. Here we add waveform-specific
        # instructions to try to work with waveforms that are known to fail.
        if 'SEOBNRv3' in p['approximant']:
            # Try doubling the sample time and redoing.
            # Don't want to get stuck in a loop though!
            if 'delta_t_orig' not in p:
                p['delta_t_orig'] = p['delta_t']
            p['delta_t'] = p['delta_t'] / 2.
            if p['delta_t_orig'] / p['delta_t'] > 9:
                raise
            hp, hc = _lalsim_td_waveform(**p)
            p['delta_t'] = p['delta_t_orig']
            hp = resample_to_delta_t(hp, hp.delta_t*2)
            hc = resample_to_delta_t(hc, hc.delta_t*2)
            return hp, hc
        raise

    #lal.DestroyDict(lal_pars)

    hp = TimeSeries(hp1.data.data[:], delta_t=hp1.deltaT, epoch=hp1.epoch)
    hc = TimeSeries(hc1.data.data[:], delta_t=hc1.deltaT, epoch=hc1.epoch)

_lalsim_td_waveform.required = parameters.cbc_td_required

def _spintaylor_aligned_prec_swapper(**p):
    """
    SpinTaylorF2 is only single spin, it also struggles with anti-aligned spin
    waveforms. This construct chooses between the aligned-twospin TaylorF2 model
    and the precessing singlespin SpinTaylorF2 models. If aligned spins are
    given, use TaylorF2, if nonaligned spins are given use SpinTaylorF2. In
    the case of nonaligned doublespin systems the code will fail at the
    waveform generator level.
    """
    orig_approximant = p['approximant']
    if p['spin2x'] == 0 and p['spin2y'] == 0 and p['spin1x'] == 0 and \
                                                              p['spin1y'] == 0:
        p['approximant'] = 'TaylorF2'
    else:
        p['approximant'] = 'SpinTaylorF2'
    hp, hc = _lalsim_fd_waveform(**p)
    p['approximant'] = orig_approximant

def _lalsim_fd_waveform(**p):
    hp1, hc1 = lalsimulation.SimInspiralChooseFDWaveform(
               p['delta_f'], float(p['f_lower']), float(p['f_final']), float(p['f_ref']),
               lal_pars,

    hp = FrequencySeries(hp1.data.data[:], delta_f=hp1.deltaF,
                            epoch=hp1.epoch)

    hc = FrequencySeries(hc1.data.data[:], delta_f=hc1.deltaF,
                            epoch=hc1.epoch)

_lalsim_fd_waveform.required = parameters.cbc_fd_required

def _lalsim_sgburst_waveform(**p):
    hp, hc = lalsimulation.SimBurstSineGaussian(float(p['q']),
               float(p['frequency']),
               float(p['hrss']),
               float(p['eccentricity']),
               float(p['polarization']),
               float(p['delta_t']))

    hp = TimeSeries(hp.data.data[:], delta_t=hp.deltaT, epoch=hp.epoch)
    hc = TimeSeries(hc.data.data[:], delta_t=hc.deltaT, epoch=hc.epoch)

for approx_enum in xrange(0, lalsimulation.NumApproximants):
    if lalsimulation.SimInspiralImplementedTDApproximants(approx_enum):
        approx_name = lalsimulation.GetStringFromApproximant(approx_enum)
        _lalsim_enum[approx_name] = approx_enum
        _lalsim_td_approximants[approx_name] = _lalsim_td_waveform

    if lalsimulation.SimInspiralImplementedFDApproximants(approx_enum):
        _lalsim_fd_approximants[approx_name] = _lalsim_fd_waveform

# sine-Gaussian burst
        _lalsim_sgburst_approximants[approx_name] = _lalsim_sgburst_waveform

cpu_sgburst = _lalsim_sgburst_approximants

cpu_td = dict(_lalsim_td_approximants.items())
cpu_fd = _lalsim_fd_approximants

# Waveforms written in CUDA
_cuda_td_approximants = {}
_cuda_fd_approximants = {}

if pycbc.HAVE_CUDA:
    from pycbc.waveform.pycbc_phenomC_tmplt import imrphenomc_tmplt
    from pycbc.waveform.SpinTaylorF2 import spintaylorf2 as cuda_spintaylorf2
    _cuda_fd_approximants["IMRPhenomC"] = imrphenomc_tmplt
    _cuda_fd_approximants["SpinTaylorF2"] = cuda_spintaylorf2

cuda_td = dict(list(_lalsim_td_approximants.items()) + list(_cuda_td_approximants.items()))
cuda_fd = dict(list(_lalsim_fd_approximants.items()) + list(_cuda_fd_approximants.items()))

# List the various available approximants ####################################

def print_td_approximants():
    print("LalSimulation Approximants")
    for approx in _lalsim_td_approximants.keys():
        print("  " + approx)
    print("CUDA Approximants")
    for approx in _cuda_td_approximants.keys():

def print_fd_approximants():
    for approx in _lalsim_fd_approximants.keys():
    for approx in _cuda_fd_approximants.keys():

def print_sgburst_approximants():
    for approx in _lalsim_sgburst_approximants.keys():

def td_approximants(scheme=_scheme.mgr.state):
    """Return a list containing the available time domain approximants for
       the given processing scheme.
    """
    return list(td_wav[type(scheme)].keys())

def fd_approximants(scheme=_scheme.mgr.state):
    """Return a list containing the available fourier domain approximants for
    """
    return list(fd_wav[type(scheme)].keys())

def sgburst_approximants(scheme=_scheme.mgr.state):
    """Return a list containing the available time domain sgbursts for
    """
    return list(sgburst_wav[type(scheme)].keys())

def filter_approximants(scheme=_scheme.mgr.state):
    """Return a list of fourier domain approximants including those
       written specifically as templates.
    """
    return list(filter_wav[type(scheme)].keys())

# Input parameter handling ###################################################

def get_obj_attrs(obj):
    """ Return a dictionary built from the attributes of the given object.
    """
    pr = {}
    if obj is not None:
        if isinstance(obj, numpy.core.records.record):
            for name in obj.dtype.names:
                pr[name] = getattr(obj, name)
        elif hasattr(obj, '__dict__') and obj.__dict__:
            pr = obj.__dict__
        elif hasattr(obj, '__slots__'):
            for slot in obj.__slots__:
                if hasattr(obj, slot):
                    pr[slot] = getattr(obj, slot)
        elif isinstance(obj, dict):
            pr = obj.copy()
        else:
            for name in dir(obj):
                try:
                    value = getattr(obj, name)
                    if not name.startswith('__') and not inspect.ismethod(value):
                        pr[name] = value
                except:
                    continue

    return pr

def props(obj, **kwargs):
    """
    pr.update(kwargs)

    input_params = default_args.copy()

def check_args(args, required_args):
    """ check that required args are given """
    missing = []
    for arg in required_args:
        if (arg not in args) or (args[arg] is None):
            missing.append(arg)

    if len(missing) != 0:
        raise ValueError("Please provide {}".format(', '.join(missing)))

# Input parameter handling for bursts ########################################

def props_sgburst(obj, **kwargs):
    pr = {}
        for name in dir(obj):
            try:
                value = getattr(obj, name)
                if not name.startswith('__') and not inspect.ismethod(value):
                    pr[name] = value
            except:
                continue

    input_params = default_sgburst_args.copy()

# Waveform generation ########################################################
fd_sequence = {}

def _lalsim_fd_sequence(**p):
    """ Shim to interface to lalsimulation SimInspiralChooseFDWaveformSequence
    """
    hp, hc = lalsimulation.SimInspiralChooseFDWaveformSequence(
               float(p['coa_phase']),
               float(p['f_ref']),
               float(p['inclination']),
               lal_pars,
               _lalsim_enum[p['approximant']],
               p['sample_points'].lal())
    return Array(hp.data.data), Array(hc.data.data)
_lalsim_fd_sequence.required = parameters.cbc_fd_required

for apx in _lalsim_enum:
    fd_sequence[apx] = _lalsim_fd_sequence

def get_fd_waveform_sequence(template=None, **kwds):
    """Return values of the waveform evaluated at the sequence of frequency
    points.

    template: object
    {params}

    Returns
    -------
    hplustilde: Array
        The plus phase of the waveform in frequency domain evaluated at the
    frequency points.
    hcrosstilde: Array
        The cross phase of the waveform in frequency domain evaluated at the
    """
    input_params = props(template, **kwds)
    input_params['delta_f'] = -1
    input_params['f_lower'] = -1
    if input_params['approximant'] not in fd_sequence:
        raise ValueError("Approximant %s not available" %
                            (input_params['approximant']))
    wav_gen = fd_sequence[input_params['approximant']]
    if hasattr(wav_gen, 'required'):
        required = wav_gen.required
    else:
        required = parameters.fd_required
    check_args(input_params, required)
    return wav_gen(**input_params)

get_fd_waveform_sequence.__doc__ = get_fd_waveform_sequence.__doc__.format(
    params=parameters.fd_waveform_sequence_params.docstr(prefix="    ",
           include_label=False).lstrip(' '))

def get_td_waveform(template=None, **kwargs):
    """Return the plus and cross polarizations of a time domain waveform.

        An object that has attached properties. This can be used to subsitute
    {params}

    Returns
    -------
    hplus: TimeSeries
        The plus polarization of the waveform.
    hcross: TimeSeries
        The cross polarization of the waveform.
    """
    input_params = props(template, **kwargs)
    wav_gen = td_wav[type(_scheme.mgr.state)]
    if input_params['approximant'] not in wav_gen:
    wav_gen = wav_gen[input_params['approximant']]
    else:
        required = parameters.td_required

get_td_waveform.__doc__ = get_td_waveform.__doc__.format(
    params=parameters.td_waveform_params.docstr(prefix="    ",

def get_fd_waveform(template=None, **kwargs):
    """Return a frequency domain gravitational waveform.

    {params}

    Returns
    -------
    hplustilde: FrequencySeries
        The plus phase of the waveform in frequency domain.
    hcrosstilde: FrequencySeries
        The cross phase of the waveform in frequency domain.
    """
    wav_gen = fd_wav[type(_scheme.mgr.state)]
    try:
        ffunc = input_params.pop('f_final_func')
        if ffunc != '':
            # convert the frequency function to a value
            input_params['f_final'] = pnutils.named_frequency_cutoffs[ffunc](
                input_params)
            # if the f_final is < f_lower, raise a NoWaveformError
            if 'f_final' in input_params and \
                    (input_params['f_lower']+input_params['delta_f'] >=
                     input_params['f_final']):
                raise NoWaveformError("cannot generate waveform: f_lower >= "
                                      "f_final")
        pass
    else:

get_fd_waveform.__doc__ = get_fd_waveform.__doc__.format(
    params=parameters.fd_waveform_params.docstr(prefix="    ",

def get_fd_waveform_from_td(**params):
    """ Return time domain version of fourier domain approximant.

    This returns a frequency domain version of a fourier domain approximant,
    with padding and tapering at the start of the waveform.

        The parameters defining the waveform to generator.
        See `get_td_waveform`.

    Returns
    -------
        Plus polarization time series
    hc: pycbc.types.FrequencySeries
        Cross polarization time series
    """

    # determine the duration to use
    full_duration = duration = get_waveform_filter_length_in_time(**params)
    nparams = params.copy()

    while full_duration < duration * 1.5:
        full_duration = get_waveform_filter_length_in_time(**nparams)
        nparams['f_lower'] -= 1

    if 'f_fref' not in nparams:
        nparams['f_ref'] = params['f_lower']

    # We'll try to do the right thing and figure out what the frequency
    # end is. Otherwise, we'll just assume 2048 Hz.
    # (consider removing as we hopefully have better estimates for more
    # approximants
    try:
        f_end = get_waveform_end_frequency(**params)
        delta_t = (0.5 / pnutils.nearest_larger_binary_number(f_end))
    except:
        delta_t = 1.0 / 2048

    nparams['delta_t'] = delta_t
    hp, hc = get_td_waveform(**nparams)

    # Resize to the right duration
    tsamples = int(1.0 / params['delta_f'] / delta_t)

    if tsamples < len(hp):
        raise ValueError("The frequency spacing (df = {}) is too low to "
                         "generate the {} approximant from the time "
                         "domain".format(params['delta_f'], params['approximant']))

    hp.resize(tsamples)
    hc.resize(tsamples)

    # apply the tapering, we will use a safety factor here to allow for
    # somewhat innacurate duration difference estimation.
    window = (full_duration - duration) * 0.8
    hp = wfutils.td_taper(hp, hp.start_time, hp.start_time + window)
    hc = wfutils.td_taper(hc, hc.start_time, hc.start_time + window)

    # avoid wraparound
    hp = hp.to_frequencyseries().cyclic_time_shift(hp.start_time)
    hc = hc.to_frequencyseries().cyclic_time_shift(hc.start_time)

def get_td_waveform_from_fd(rwrap=0.2, **params):

    This returns a time domain version of a fourier domain approximant, with
    padding and tapering at the start of the waveform.

    rwrap: float
        Cyclic time shift parameter in seconds. A fudge factor to ensure
        that the entire time series is contiguous in the array and not
        wrapped around the end.
        See `get_fd_waveform`.

    Returns
    -------
    hp: pycbc.types.TimeSeries
    hc: pycbc.types.TimeSeries
    """

        nparams['f_lower'] *= 0.99

    if 'f_ref' not in nparams:

    # factor to ensure the vectors are all large enough. We don't need to
    # completely trust our duration estimator in this case, at a small
    # increase in computational cost
    fudge_duration = (max(0, full_duration) + .1 + rwrap) * 1.5
    fsamples = int(fudge_duration / params['delta_t'])
    N = pnutils.nearest_larger_binary_number(fsamples)
    fudge_duration = N * params['delta_t']

    nparams['delta_f'] = 1.0 / fudge_duration
    hp, hc = get_fd_waveform(**nparams)

    # Resize to the right sample rate
    tsize = int(1.0 / params['delta_t'] /  nparams['delta_f'])
    fsize = tsize // 2 + 1
    hp.resize(fsize)
    hc.resize(fsize)

    hp = hp.cyclic_time_shift(-rwrap)
    hc = hc.cyclic_time_shift(-rwrap)

    hp = wfutils.fd_to_td(hp, left_window=(nparams['f_lower'],
                                           params['f_lower']))
    hc = wfutils.fd_to_td(hc, left_window=(nparams['f_lower'],

def get_interpolated_fd_waveform(dtype=numpy.complex64, return_hc=True,
                                 **params):
    """ Return a fourier domain waveform approximant, using interpolation
    """

    def rulog2(val):
        return 2.0 ** numpy.ceil(numpy.log2(float(val)))

    orig_approx = params['approximant']
    params['approximant'] = params['approximant'].replace('_INTERP', '')
    df = params['delta_f']

    if 'duration' not in params:
        duration = get_waveform_filter_length_in_time(**params)
    elif params['duration'] > 0:
        duration = params['duration']
    else:
        err_msg = "Waveform duration must be greater than 0."

    #FIXME We should try to get this length directly somehow
    # I think this number should be conservative
    ringdown_padding = 0.5

    df_min = 1.0 / rulog2(duration + ringdown_padding)
    # FIXME: I don't understand this, but waveforms with df_min < 0.5 will chop
    #        off the inspiral when using ringdown_padding - 0.5.
    #        Also, if ringdown_padding is set to a very small
    #        value we can see cases where the ringdown is chopped.
    if df_min > 0.5:
        df_min = 0.5
    params['delta_f'] = df_min
    hp, hc = get_fd_waveform(**params)
    hp = hp.astype(dtype)
    if return_hc:
        hc = hc.astype(dtype)
    else:
        hc = None

    f_end = get_waveform_end_frequency(**params)
    if f_end is None:
        f_end = (len(hp) - 1) * hp.delta_f
    if 'f_final' in params and params['f_final'] > 0:
        f_end_params = params['f_final']
        if f_end is not None:
            f_end = min(f_end_params, f_end)

    n_min = int(rulog2(f_end / df_min)) + 1
    if n_min < len(hp):
        hp = hp[:n_min]
        if hc is not None:
            hc = hc[:n_min]

    offset = int(ringdown_padding * (len(hp)-1)*2 * hp.delta_f)

    hp = interpolate_complex_frequency(hp, df, zeros_offset=offset, side='left')
    if hc is not None:
        hc = interpolate_complex_frequency(hc, df, zeros_offset=offset,
                                           side='left')
    params['approximant'] = orig_approx

def get_sgburst_waveform(template=None, **kwargs):
    """Return the plus and cross polarizations of a time domain
    sine-Gaussian burst waveform.

    approximant : string
        A string that indicates the chosen approximant. See `td_approximants`
        for available options.
    q : float
        The quality factor of a sine-Gaussian burst
    frequency : float
        The centre-frequency of a sine-Gaussian burst
        The time step used to generate the waveform
    hrss : float
        The strain rss
        The strain amplitude

    Returns
    -------
    """
    input_params = props_sgburst(template,**kwargs)

    for arg in sgburst_required_args:
            raise ValueError("Please provide " + str(arg))

    return _lalsim_sgburst_waveform(**input_params)

# Waveform filter routines ###################################################

# Organize Filter Generators
_inspiral_fd_filters = {}
_cuda_fd_filters = {}

_cuda_fd_filters['SPAtmplt'] = spa_tmplt
_inspiral_fd_filters['SPAtmplt'] = spa_tmplt

filter_wav = _scheme.ChooseBySchemeDict()
filter_wav.update( {_scheme.CPUScheme:_inspiral_fd_filters,
                    _scheme.CUDAScheme:_cuda_fd_filters,
                   } )

# Organize functions for function conditioning/precalculated values
_filter_norms = {}
_filter_ends = {}
_filter_preconditions = {}
_template_amplitude_norms = {}
_filter_time_lengths = {}

def seobnrv2_final_frequency(**kwds):
    return pnutils.get_final_freq("SEOBNRv2", kwds['mass1'], kwds['mass2'],
                                  kwds['spin1z'], kwds['spin2z'])

def get_imr_length(approx, **kwds):
    """Call through to pnutils to obtain IMR waveform durations
    """
    m1 = float(kwds['mass1'])
    m2 = float(kwds['mass2'])
    s1z = float(kwds['spin1z'])
    s2z = float(kwds['spin2z'])
    f_low = float(kwds['f_lower'])
    # 10% margin of error is incorporated in the pnutils function
    return pnutils.get_imr_duration(m1, m2, s1z, s2z, f_low, approximant=approx)

def seobnrv2_length_in_time(**kwds):
    """Stub for holding the calculation of SEOBNRv2* waveform duration.
    """
    return get_imr_length("SEOBNRv2", **kwds)

def seobnrv4_length_in_time(**kwds):
    """Stub for holding the calculation of SEOBNRv4* waveform duration.
    """
    return get_imr_length("SEOBNRv4", **kwds)

def imrphenomd_length_in_time(**kwds):
    """Stub for holding the calculation of IMRPhenomD waveform duration.
    """
    return get_imr_length("IMRPhenomD", **kwds)

def imrphenomhm_length_in_time(**kwargs):
    """Estimates the duration of IMRPhenom waveforms that include higher modes.
    """
    # Default maximum node number for IMRPhenomHM is 4
    # The relevant lower order approximant here is IMRPhenomD
    return get_hm_length_in_time("IMRPhenomD", 4, **kwargs)

def seobnrv4hm_length_in_time(**kwargs):
    """ Estimates the duration of SEOBNRv4HM waveforms that include higher modes.
    """
    # Default maximum node number for SEOBNRv4HM is 5
    # The relevant lower order approximant here is SEOBNRv4
    return get_hm_length_in_time('SEOBNRv4', 5, **kwargs)

def get_hm_length_in_time(lor_approx, maxm_default, **kwargs):
    if 'mode_array' in kwargs and kwargs['mode_array'] is not None:
        maxm = max(m for _, m in kwargs['mode_array'])
    else:
        maxm = maxm_default
    try:
        flow = kwargs['f_lower']
        raise ValueError("must provide a f_lower")
    kwargs['f_lower'] = flow * 2./maxm
    return get_imr_length(lor_approx, **kwargs)

_filter_norms["SPAtmplt"] = spa_tmplt_norm
_filter_preconditions["SPAtmplt"] = spa_tmplt_precondition

_filter_ends["SPAtmplt"] = spa_tmplt_end
_filter_ends["TaylorF2"] = spa_tmplt_end
#_filter_ends["SEOBNRv1_ROM_EffectiveSpin"] = seobnrv2_final_frequency
#_filter_ends["SEOBNRv1_ROM_DoubleSpin"] =  seobnrv2_final_frequency
#_filter_ends["SEOBNRv2_ROM_EffectiveSpin"] = seobnrv2_final_frequency
#_filter_ends["SEOBNRv2_ROM_DoubleSpin"] =  seobnrv2_final_frequency
#_filter_ends["SEOBNRv2_ROM_DoubleSpin_HI"] = seobnrv2_final_frequency
# PhenomD returns higher frequencies than this, so commenting this out for now
#_filter_ends["IMRPhenomC"] = seobnrv2_final_frequency
#_filter_ends["IMRPhenomD"] = seobnrv2_final_frequency

_template_amplitude_norms["SPAtmplt"] = spa_amplitude_factor
_filter_time_lengths["SPAtmplt"] = spa_length_in_time
_filter_time_lengths["TaylorF2"] = spa_length_in_time
_filter_time_lengths["SpinTaylorT5"] = spa_length_in_time
_filter_time_lengths["SEOBNRv1_ROM_EffectiveSpin"] = seobnrv2_length_in_time
_filter_time_lengths["SEOBNRv1_ROM_DoubleSpin"] = seobnrv2_length_in_time
_filter_time_lengths["SEOBNRv2_ROM_EffectiveSpin"] = seobnrv2_length_in_time
_filter_time_lengths["SEOBNRv2_ROM_DoubleSpin"] = seobnrv2_length_in_time
_filter_time_lengths["EOBNRv2_ROM"] = seobnrv2_length_in_time
_filter_time_lengths["EOBNRv2HM_ROM"] = seobnrv2_length_in_time
_filter_time_lengths["SEOBNRv2_ROM_DoubleSpin_HI"] = seobnrv2_length_in_time
_filter_time_lengths["SEOBNRv4_ROM"] = seobnrv4_length_in_time
_filter_time_lengths["SEOBNRv4HM_ROM"] = seobnrv4hm_length_in_time
_filter_time_lengths["SEOBNRv4"] = seobnrv4_length_in_time
_filter_time_lengths["SEOBNRv4P"] = seobnrv4_length_in_time
_filter_time_lengths["IMRPhenomC"] = imrphenomd_length_in_time
_filter_time_lengths["IMRPhenomD"] = imrphenomd_length_in_time
_filter_time_lengths["IMRPhenomPv2"] = imrphenomd_length_in_time
_filter_time_lengths["IMRPhenomD_NRTidal"] = imrphenomd_length_in_time
_filter_time_lengths["IMRPhenomPv2_NRTidal"] = imrphenomd_length_in_time
_filter_time_lengths["IMRPhenomHM"] = imrphenomhm_length_in_time
_filter_time_lengths["IMRPhenomPv3HM"] = imrphenomhm_length_in_time
_filter_time_lengths["IMRPhenomXHM"] = imrphenomhm_length_in_time
_filter_time_lengths["IMRPhenomXPHM"] = imrphenomhm_length_in_time
_filter_time_lengths["SpinTaylorF2"] = spa_length_in_time
_filter_time_lengths["TaylorF2NL"] = spa_length_in_time
_filter_time_lengths["PreTaylorF2"] = spa_length_in_time

# Also add generators for switching between approximants
apx_name = "SpinTaylorF2_SWAPPER"
cpu_fd[apx_name] =  _spintaylor_aligned_prec_swapper
_filter_time_lengths[apx_name] = _filter_time_lengths["SpinTaylorF2"]

from . nltides import nonlinear_tidal_spa
cpu_fd["TaylorF2NL"] = nonlinear_tidal_spa

from .premerger import premerger_taylorf2
cpu_fd['PreTaylorF2'] = premerger_taylorf2

from .multiband import multiband_fd_waveform
cpu_fd['multiband'] = multiband_fd_waveform

# Load external waveforms #####################################################
if 'PYCBC_WAVEFORM' in os.environ:
    mods = os.environ['PYCBC_WAVEFORM'].split(':')
    for mod in mods:
        mhandle = __import__(mod, fromlist=[''])
        mhandle.add_me(cpu_fd=cpu_fd,
                       cpu_td=cpu_td,
                       filter_time_lengths=_filter_time_lengths)

for apx in copy.copy(_filter_time_lengths):
    fd_apx = list(cpu_fd.keys())
    td_apx = list(cpu_td.keys())

    if (apx in td_apx) and (apx not in fd_apx):
        # We can make a fd version of td approximants
        cpu_fd[apx] = get_fd_waveform_from_td

    if apx in fd_apx:
        # We can do interpolation for waveforms that have a time length
        apx_int = apx + '_INTERP'
        cpu_fd[apx_int] = get_interpolated_fd_waveform
        _filter_time_lengths[apx_int] = _filter_time_lengths[apx]

        # We can also make a td version of this
        # This will override any existing approximants with the same name
        # (ex. IMRPhenomXX)
        cpu_td[apx] = get_td_waveform_from_fd

td_wav = _scheme.ChooseBySchemeDict()
fd_wav = _scheme.ChooseBySchemeDict()
td_wav.update({_scheme.CPUScheme:cpu_td,_scheme.CUDAScheme:cuda_td})
fd_wav.update({_scheme.CPUScheme:cpu_fd,_scheme.CUDAScheme:cuda_fd})
sgburst_wav = {_scheme.CPUScheme:cpu_sgburst}

def get_waveform_filter(out, template=None, **kwargs):
    """Return a frequency domain waveform filter for the specified approximant
    """
    n = len(out)

    if input_params['approximant'] in filter_approximants(_scheme.mgr.state):
        wav_gen = filter_wav[type(_scheme.mgr.state)]
        htilde = wav_gen[input_params['approximant']](out=out, **input_params)
        htilde.resize(n)
        htilde.chirp_length = get_waveform_filter_length_in_time(**input_params)
        htilde.length_in_time = htilde.chirp_length

    if input_params['approximant'] in fd_approximants(_scheme.mgr.state):
        wav_gen = fd_wav[type(_scheme.mgr.state)]

        duration = get_waveform_filter_length_in_time(**input_params)
        hp, _ = wav_gen[input_params['approximant']](duration=duration,
                                               return_hc=False, **input_params)

        hp.resize(n)
        out[0:len(hp)] = hp[:]
        hp.data = out

        hp.length_in_time = hp.chirp_length = duration
        return hp

    elif input_params['approximant'] in td_approximants(_scheme.mgr.state):
        wav_gen = td_wav[type(_scheme.mgr.state)]
        hp, _ = wav_gen[input_params['approximant']](**input_params)
        # taper the time series hp if required
        if 'taper' in input_params.keys() and \
                input_params['taper'] is not None:
            hp = wfutils.taper_timeseries(hp, input_params['taper'],
                                          return_lal=False)
        return td_waveform_to_fd_waveform(hp, out=out)

    else:

def td_waveform_to_fd_waveform(waveform, out=None, length=None,
                               buffer_length=100):
    """ Convert a time domain into a frequency domain waveform by FFT.
        As a waveform is assumed to "wrap" in the time domain one must be
        careful to ensure the waveform goes to 0 at both "boundaries". To
        ensure this is done correctly the waveform must have the epoch set such
        the merger time is at t=0 and the length of the waveform should be
        shorter than the desired length of the FrequencySeries (times 2 - 1)
        so that zeroes can be suitably pre- and post-pended before FFTing.
        If given, out is a memory array to be used as the output of the FFT.
        If not given memory is allocated internally.
        If present the length of the returned FrequencySeries is determined
        from the length out. If out is not given the length can be provided
        expicitly, or it will be chosen as the nearest power of 2. If choosing
        length explicitly the waveform length + buffer_length is used when
        choosing the nearest binary number so that some zero padding is always
        added.
    """
    # Figure out lengths and set out if needed
        if length is None:
            N = pnutils.nearest_larger_binary_number(len(waveform) + \
                                                     buffer_length)
            n = int(N//2) + 1
        else:
            n = length
            N = (n-1)*2
        out = zeros(n, dtype=complex_same_precision_as(waveform))
    else:
        n = len(out)
        N = (n-1)*2
    delta_f =  1. / (N * waveform.delta_t)

    # total duration of the waveform
    tmplt_length = len(waveform) * waveform.delta_t
    if len(waveform) > N:
        err_msg = "The time domain template is longer than the intended "
        err_msg += "duration in the frequency domain. This situation is "
        err_msg += "not supported in this function. Please shorten the "
        err_msg += "waveform appropriately before calling this function or "
        err_msg += "increase the allowed waveform length. "
        err_msg += "Waveform length (in samples): {}".format(len(waveform))
        err_msg += ". Intended length: {}.".format(N)
    # for IMR templates the zero of time is at max amplitude (merger)
    # thus the start time is minus the duration of the template from
    # lower frequency cutoff to merger, i.e. minus the 'chirp time'
    tChirp = - float( waveform.start_time )  # conversion from LIGOTimeGPS
    waveform.resize(N)
    k_zero = int(waveform.start_time / waveform.delta_t)
    waveform.roll(k_zero)
    htilde = FrequencySeries(out, delta_f=delta_f, copy=False)
    fft(waveform.astype(real_same_precision_as(htilde)), htilde)
    htilde.length_in_time = tmplt_length
    htilde.chirp_length = tChirp

def get_two_pol_waveform_filter(outplus, outcross, template, **kwargs):
    """Return a frequency domain waveform filter for the specified approximant.
    Unlike get_waveform_filter this function returns both h_plus and h_cross
    components of the waveform, which are needed for searches where h_plus
    and h_cross are not related by a simple phase shift.
    """
    n = len(outplus)

    # If we don't have an inclination column alpha3 might be used
    if not hasattr(template, 'inclination') and 'inclination' not in kwargs:
        if hasattr(template, 'alpha3'):
            kwargs['inclination'] = template.alpha3

        hp, hc = wav_gen[input_params['approximant']](**input_params)
        hc.resize(n)
        outplus[0:len(hp)] = hp[:]
        hp = FrequencySeries(outplus, delta_f=hp.delta_f, copy=False)
        outcross[0:len(hc)] = hc[:]
        hc = FrequencySeries(outcross, delta_f=hc.delta_f, copy=False)
        hp.chirp_length = get_waveform_filter_length_in_time(**input_params)
        hp.length_in_time = hp.chirp_length
        hc.chirp_length = hp.chirp_length
        hc.length_in_time = hp.length_in_time
        # N: number of time samples required
        N = (n-1)*2
        delta_f = 1.0 / (N * input_params['delta_t'])
            hc = wfutils.taper_timeseries(hc, input_params['taper'],
        # total duration of the waveform
        tmplt_length = len(hp) * hp.delta_t
        # for IMR templates the zero of time is at max amplitude (merger)
        # thus the start time is minus the duration of the template from
        # lower frequency cutoff to merger, i.e. minus the 'chirp time'
        tChirp = - float( hp.start_time )  # conversion from LIGOTimeGPS
        hp.resize(N)
        hc.resize(N)
        k_zero = int(hp.start_time / hp.delta_t)
        hp.roll(k_zero)
        hc.roll(k_zero)
        hp_tilde = FrequencySeries(outplus, delta_f=delta_f, copy=False)
        hc_tilde = FrequencySeries(outcross, delta_f=delta_f, copy=False)
        fft(hp.astype(real_same_precision_as(hp_tilde)), hp_tilde)
        fft(hc.astype(real_same_precision_as(hc_tilde)), hc_tilde)
        hp_tilde.length_in_time = tmplt_length
        hp_tilde.chirp_length = tChirp
        hc_tilde.length_in_time = tmplt_length
        hc_tilde.chirp_length = tChirp
        return hp_tilde, hc_tilde
    else:

def waveform_norm_exists(approximant):
    if approximant in _filter_norms:
    else:

def get_template_amplitude_norm(template=None, **kwargs):
    """ Return additional constant template normalization. This only affects
        the effective distance calculation. Returns None for all templates with a
        physically meaningful amplitude.
    """
    input_params = props(template,**kwargs)
    approximant = kwargs['approximant']

    if approximant in _template_amplitude_norms:
        return _template_amplitude_norms[approximant](**input_params)
    else:

def get_waveform_filter_precondition(approximant, length, delta_f):
    """Return the data preconditioning factor for this approximant.
    """
    if approximant in _filter_preconditions:
        return _filter_preconditions[approximant](length, delta_f)
    else:

def get_waveform_filter_norm(approximant, psd, length, delta_f, f_lower):
    """ Return the normalization vector for the approximant
    """
        return _filter_norms[approximant](psd, length, delta_f, f_lower)
    else:

def get_waveform_end_frequency(template=None, **kwargs):
    """Return the stop frequency of a template
    """

    if approximant in _filter_ends:
        return _filter_ends[approximant](**input_params)
    else:

def get_waveform_filter_length_in_time(approximant, template=None, **kwargs):
    """For filter templates, return the length in time of the template.
    """
    kwargs = props(template, **kwargs)

        return _filter_time_lengths[approximant](**kwargs)
    else:

__all__ = ["get_td_waveform", "get_fd_waveform", "get_fd_waveform_sequence",
           "get_fd_waveform_from_td",
           "print_td_approximants", "print_fd_approximants",
           "td_approximants", "fd_approximants",
           "get_waveform_filter", "filter_approximants",
           "get_waveform_filter_norm", "get_waveform_end_frequency",
           "waveform_norm_exists", "get_template_amplitude_norm",
           "get_waveform_filter_length_in_time", "get_sgburst_waveform",
           "print_sgburst_approximants", "sgburst_approximants",
           "td_waveform_to_fd_waveform", "get_two_pol_waveform_filter",
           "NoWaveformError", "FailedWaveformError", "get_td_waveform_from_fd",
           'cpu_fd', 'cpu_td', 'fd_sequence', '_filter_time_lengths']

# Copyright (C) 2020  Collin Capano, Alex Nitz
#
#

"""Provides functions and utilities for generating waveforms mode-by-mode.
"""

from string import Formatter
import lal

from pycbc.types import (TimeSeries, FrequencySeries)
from .waveform import (props, _check_lal_pars, check_args)
from . import parameters

def _formatdocstr(docstr):
    """Utility for formatting docstrings with parameter information.
    """
    return docstr.format(
        **{_p[1]: getattr(parameters, _p[1]).docstr(
            prefix="    ", include_label=False).lstrip(' ')
           for _p in Formatter().parse(docstr) if _p[1] is not None
           })

def _formatdocstrlist(docstr, paramlist, skip_params=None):
    """
    if skip_params is None:
        skip_params = []
    pl = '\n'.join([_p.docstr(prefix="    ", include_label=False)
                    for _p in paramlist if _p not in skip_params]).lstrip(' ')
    return docstr.format(params=pl)

def sum_modes(hlms, inclination, phi):
    """Applies spherical harmonics and sums modes to produce a plus and cross
    polarization.

    hlms : dict
        Dictionary of ``(l, m)`` -> complex ``hlm``. The ``hlm`` may be a
        complex number or array, or complex ``TimeSeries``. All modes in the
        dictionary will be summed.
        The inclination to use.
    phi : float
        The phase to use.

    Returns
    -------
    complex float or array
        The plus and cross polarization as a complex number. The real part
        gives the plus, the negative imaginary part the cross.
    """
    out = None
    for mode in hlms:
        l, m = mode
        hlm = hlms[l, m]
        ylm = lal.SpinWeightedSphericalHarmonic(inclination, phi, -2, l, m)
        if out is None:
            out = ylm * hlm
        else:
            out += ylm * hlm
    return out

def default_modes(approximant):
    """Returns the default modes for the given approximant.
    """
    # FIXME: this should be replaced to a call to a lalsimulation function,
    # whenever that's added
    if approximant in ['IMRPhenomXPHM', 'IMRPhenomXHM']:
        # according to arXiv:2004.06503
        ma = [(2, 2), (2, 1), (3, 3), (3, 2), (4, 4)]
        # add the -m modes
        ma += [(l, -m) for l, m in ma]
    elif approximant in ['IMRPhenomPv3HM', 'IMRPhenomHM']:
        # according to arXiv:1911.06050
        ma = [(2, 2), (2, 1), (3, 3), (3, 2), (4, 4), (4, 3)]
    elif approximant.startswith('NRSur7dq4'):
        # according to arXiv:1905.09300
        ma = [(l, m) for l in [2, 3, 4] for m in range(-l, l+1)]
    else:
        raise ValueError("I don't know what the default modes are for "
                         "approximant {}, sorry!".format(approximant))
    return ma

def get_glm(l, m, theta):
    r"""The maginitude of the :math:`{}_{-2}Y_{\ell m}`.

    The spin-weighted spherical harmonics can be written as
    :math:`{}_{-2}Y_{\ell m}(\theta, \phi) = g_{\ell m}(\theta)e^{i m \phi}`.
    This returns the `g_{\ell m}(\theta)` part. Note that this is real.

    l : int
        The :math:`\ell` index of the spherical harmonic.
    m : int
        The :math:`m` index of the spherical harmonic.
    theta : float
        The polar angle (in radians).

    Returns
    -------
    float :
        The amplitude of the harmonic at the given polar angle.
    """
    return lal.SpinWeightedSphericalHarmonic(theta, 0., -2, l, m).real

def get_nrsur_modes(**params):
    """Generates NRSurrogate waveform mode-by-mode.

    All waveform parameters should be provided as keyword arguments.
    Recognized parameters are listed below. Unrecognized arguments are ignored.

        The approximant to generate. Must be one of the ``NRSur*`` models.
    {delta_t}
    {mass1}
    {mass2}
    {spin1x}
    {spin1y}
    {spin1z}
    {spin2x}
    {spin2y}
    {spin2z}
    {f_lower}
    {f_ref}
    {distance}
    {mode_array}

    Returns
    -------
    dict :
        Dictionary of ``(l, m)`` -> ``(h_+, -h_x)`` ``TimeSeries``.
    """
    laldict = _check_lal_pars(params)
    ret = lalsimulation.SimInspiralPrecessingNRSurModes(
        params['delta_t'],
        params['mass1']*lal.MSUN_SI,
        params['mass2']*lal.MSUN_SI,
        params['spin1x'], params['spin1y'], params['spin1z'],
        params['spin2x'], params['spin2y'], params['spin2z'],
        params['f_lower'], params['f_ref'],
        params['distance']*1e6*lal.PC_SI, laldict,
        getattr(lalsimulation, params['approximant'])
    )
    hlms = {}
    while ret:
        hlm = TimeSeries(ret.mode.data.data, delta_t=ret.mode.deltaT,
                         epoch=ret.mode.epoch)
        hlms[ret.l, ret.m] = (hlm.real(), hlm.imag())
        ret = ret.next
    return hlms

get_nrsur_modes.__doc__ = _formatdocstr(get_nrsur_modes.__doc__)

def _get_imrphenomx_modes(return_posneg=False, **params):
    """Generates ``IMRPhenomX[P]HM`` waveforms mode-by-mode.

    Currently does not work; just raises a ``NotImplementedError``.
    """
    # FIXME: raising not implemented error because this currently does not
    # work. The issue is the OneMode function adds the +/- m modes together
    # automatically. Remove once this is fixed in lalsimulation, and/or I
    # figure out a reliable way to separate the +/-m modes.
    raise NotImplementedError("Currently not implemented")
    approx = params['approximant']
    if not approx.startswith('IMRPhenomX'):
        raise ValueError("unsupported approximant")
    mode_array = params.pop('mode_array', None)
    if mode_array is None:
        mode_array = default_modes(approx)
    if 'f_final' not in params:
        # setting to 0 will default to ringdown frequency
        params['f_final'] = 0.
    hlms = {}
    for (l, m) in mode_array:
        params['mode_array'] = [(l, m)]
        laldict = _check_lal_pars(params)
        hpos, hneg = lalsimulation.SimIMRPhenomXPHMOneMode(
            l, m,
            params['mass1']*lal.MSUN_SI,
            params['mass2']*lal.MSUN_SI,
            params['spin1x'], params['spin1y'], params['spin1z'],
            params['spin2x'], params['spin2y'], params['spin2z'],
            params['distance']*1e6*lal.PC_SI, params['coa_phase'],
            params['delta_f'], params['f_lower'], params['f_final'],
            params['f_ref'],
            laldict)
        hpos = FrequencySeries(hpos.data.data, delta_f=hpos.deltaF,
                               epoch=hpos.epoch)
        hneg = FrequencySeries(hneg.data.data, delta_f=hneg.deltaF,
                               epoch=hneg.epoch)
        if return_posneg:
            hlms[l, m] = (hpos, hneg)
        else:
            # convert to ulm, vlm
            ulm = 0.5 * (hpos + hneg.conj())
            vlm = 0.5j * (hneg.conj() - hpos)
            hlms[l, m] = (ulm, vlm)

def l0frame_to_jframe(mass1, mass2, f_ref, phiref=0., inclination=0.,
                      spin1x=0., spin1y=0., spin1z=0.,
                      spin2x=0., spin2y=0., spin2z=0.):
    """Converts L0-frame parameters to J-frame.

    {mass1}
    {mass2}
    {f_ref}
    phiref : float
        The orbital phase at ``f_ref``.
    {inclination}
    {spin1x}
    {spin1y}
    {spin1z}
    {spin2x}
    {spin2y}
    {spin2z}

    Returns
    -------
    dict :
        Dictionary of:
        * thetajn : float
            Angle between the line of sight and the total angular momentume J.
        * phijl : float
            Azimuthal angle of L on its cone about J.
        * {spin1_a}
        * {spin2_a}
        * spin1_polar : float
            Angle between L and the spin magnitude of the larger object.
        * spin2_polar : float
            Angle betwen L and the spin magnitude of the smaller object.
        * spin12_deltaphi : float
            Difference between the azimuthal angles of the spin of the larger
            object (S1) and the spin of the smaller object (S2).
    """
    # Note: unlike other LALSimulation functions, this one takes masses in
    # solar masses
    thetajn, phijl, s1pol, s2pol, s12_deltaphi, spin1_a, spin2_a = \
        lalsimulation.SimInspiralTransformPrecessingWvf2PE(
            inclination, spin1x, spin1y, spin1z, spin2x, spin2y, spin2z,
            mass1, mass2, f_ref, phiref)
    out = {'thetajn': thetajn,
           'phijl': phijl,
           'spin1_polar': s1pol,
           'spin2_polar': s2pol,
           'spin12_deltaphi': s12_deltaphi,
           'spin1_a': spin1_a,
           'spin2_a': spin2_a}
    return out

l0frame_to_jframe.__doc__ = _formatdocstr(l0frame_to_jframe.__doc__)

def jframe_to_l0frame(mass1, mass2, f_ref, phiref=0., thetajn=0., phijl=0.,
                      spin1_a=0., spin2_a=0.,
                      spin1_polar=0., spin2_polar=0.,
                      spin12_deltaphi=0.):
    """Converts J-frame parameters into L0 frame.

    {mass1}
    {mass2}
    {f_ref}
    thetajn : float
        Angle between the line of sight and the total angular momentume J.
    phijl : float
        Azimuthal angle of L on its cone about J.
    {spin1_a}
    {spin2_a}
    spin1_polar : float
        Angle between L and the spin magnitude of the larger object.
    spin2_polar : float
        Angle betwen L and the spin magnitude of the smaller object.
    spin12_deltaphi : float
        Difference between the azimuthal angles of the spin of the larger
        object (S1) and the spin of the smaller object (S2).

    Returns
    -------
    dict :
        * {inclination}
        * {spin1x}
        * {spin1y}
        * {spin1z}
        * {spin2x}
        * {spin2y}
        * {spin2z}
    """
    inclination, spin1x, spin1y, spin1z, spin2x, spin2y, spin2z = \
        lalsimulation.SimInspiralTransformPrecessingNewInitialConditions(
            thetajn, phijl, spin1_polar, spin2_polar, spin12_deltaphi,
            spin1_a, spin2_a, mass1*lal.MSUN_SI, mass2*lal.MSUN_SI, f_ref,
            phiref)
    out = {'inclination': inclination,
           'spin1x': spin1x,
           'spin1y': spin1y,
           'spin1z': spin1z,
           'spin2x': spin2x,
           'spin2y': spin2y,
           'spin2z': spin2z}
    return out

jframe_to_l0frame.__doc__ = _formatdocstr(jframe_to_l0frame.__doc__)

_mode_waveform_td = {'NRSur7dq4': get_nrsur_modes,
                     }

# Remove commented out once IMRPhenomX one mode is fixed
_mode_waveform_fd = {#'IMRPhenomXHM': get_imrphenomhm_modes,
                     #'IMRPhenomXPHM' : get_imrphenomhm_modes,
                    }

def fd_waveform_mode_approximants():
    """Frequency domain approximants that will return separate modes."""
    return sorted(_mode_waveform_fd.keys())

def td_waveform_mode_approximants():
    """Time domain approximants that will return separate modes."""
    return sorted(_mode_waveform_td.keys())

def get_fd_waveform_modes(template=None, **kwargs):
    r"""Generates frequency domain waveforms, but does not sum over the modes.

    The returned values are the frequency-domain equivalents of the real and
    imaginary parts of the complex :math:`\mathfrak{{h}}_{{\ell m}}(t)` time
    series. In other words, the returned values are equivalent to the Fourier
    Transform of the two time series returned by
    :py:func:`get_td_waveform_modes`; see that function for more details.

        for keyword arguments.
    {params}

    Returns
    -------
    ulm : dict
        Dictionary of mode tuples -> fourier transform of the real part of the
        hlm time series, as a :py:class:`pycbc.types.FrequencySeries`.
    vlm : dict
        Dictionary of mode tuples -> fourier transform of the imaginary part of
        the hlm time series, as a :py:class:`pycbc.types.FrequencySeries`.
    """
    params = props(template, **kwargs)
    required = parameters.fd_required
    check_args(params, required)
    apprx = params['approximant']
    if apprx not in _mode_waveform_fd:
        raise ValueError("I don't support approximant {}, sorry"
                         .format(apprx))
    return _mode_waveform_fd[apprx](**params)

get_fd_waveform_modes.__doc__ = _formatdocstrlist(
    get_fd_waveform_modes.__doc__, parameters.fd_waveform_params,
    skip_params=['inclination', 'coa_phase'])

def get_td_waveform_modes(template=None, **kwargs):
    r"""Generates time domain waveforms, but does not sum over the modes.

    The returned values are the real and imaginary parts of the complex
    :math:`\mathfrak{{h}}_{{\ell m}}(t)`. These are defined such that the plus
    and cross polarizations :math:`h_{{+,\times}}` are:

    .. math::

       h_{{+,\times}}(\theta, \phi; t) = (\Re, -\Im) \sum_{{\ell m}}
        {{}}_{{-2}}Y_{{\ell m}}(\theta, \phi) \mathfrak{{h}}_{{\ell m}}(t).

    {params}

    Returns
    -------
    ulm : dict
        Dictionary of mode tuples -> real part of the hlm, as a
        :py:class:`pycbc.types.TimeSeries`.
    vlm : dict
        Dictionary of mode tuples -> imaginary part of the hlm, as a
    """
    if apprx not in _mode_waveform_td:
    return _mode_waveform_td[apprx](**params)

get_td_waveform_modes.__doc__ = _formatdocstrlist(
    get_td_waveform_modes.__doc__, parameters.td_waveform_params,

# Copyright (C) 2013  Ian Harry
#
#
#

#
#
#                                   Preamble
#
#
"""
This module is responsible for setting up the coincidence stage of pycbc
workflows. For details about this module and its capabilities see here:
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/coincidence.html
"""

import os
from pycbc.workflow.core import FileList, make_analysis_dir, Executable, Node, File

class PyCBCBank2HDFExecutable(Executable):
    """Converts xml tmpltbank to hdf format"""

    current_retention_level = Executable.MERGED_TRIGGERS
    def create_node(self, bank_file):
        node = Node(self)
        node.add_input_opt('--bank-file', bank_file)
        node.new_output_file_opt(bank_file.segment, '.hdf', '--output-file')
        return node

class PyCBCTrig2HDFExecutable(Executable):
    """Converts xml triggers to hdf format, grouped by template hash"""

    def create_node(self, trig_files, bank_file):
        node.add_input_list_opt('--trigger-files', trig_files)
        node.new_output_file_opt(trig_files[0].segment, '.hdf',
                                 '--output-file', use_tmp_subdirs=True)

class PyCBCFitByTemplateExecutable(Executable):
    """Calculates values that describe the background distribution template by template"""

    def create_node(self, trig_file, bank_file, veto_file, veto_name):
        # Executable objects are initialized with ifo information
        node.add_opt('--ifo', self.ifo_string)
        node.add_input_opt('--trigger-file', trig_file)
        node.add_input_opt('--veto-file', veto_file)
        node.add_opt('--veto-segment-name', veto_name)
        node.new_output_file_opt(trig_file.segment, '.hdf', '--output')

class PyCBCFitOverParamExecutable(Executable):
    """Smooths the background distribution parameters over a continuous parameter"""

    def create_node(self, raw_fit_file, bank_file):
        node.add_input_opt('--template-fit-file', raw_fit_file)
        node.new_output_file_opt(raw_fit_file.segment, '.hdf', '--output')

class PyCBCFindCoincExecutable(Executable):
    """Find coinc triggers using a folded interval method"""

    current_retention_level = Executable.ALL_TRIGGERS
    file_input_options = ['--statistic-files']
    def create_node(self, trig_files, bank_file, stat_files, veto_file,
                    veto_name, template_str, pivot_ifo, fixed_ifo, tags=None):
        if tags is None:
            tags = []
        segs = trig_files.get_times_covered_by_files()
        seg = segments.segment(segs[0][0], segs[-1][1])
        node.add_input_opt('--template-bank', bank_file)
        if len(stat_files) > 0:
            node.add_input_list_opt('--statistic-files', stat_files)
        if veto_file is not None:
            node.add_input_opt('--veto-files', veto_file)
            node.add_opt('--segment-name', veto_name)
        node.add_opt('--pivot-ifo', pivot_ifo)
        node.add_opt('--fixed-ifo', fixed_ifo)
        node.add_opt('--template-fraction-range', template_str)
        node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags)

class PyCBCStatMapExecutable(Executable):
    """Calculate FAP, IFAR, etc"""

    def create_node(self, coinc_files, ifos, tags=None):
            tags = []
        segs = coinc_files.get_times_covered_by_files()

        node.add_input_list_opt('--coinc-files', coinc_files)
        node.add_opt('--ifos', ifos)

class PyCBCStatMapInjExecutable(Executable):

    def create_node(self, zerolag, full_data,
                    injfull, fullinj, ifos, tags=None):
            tags = []
        segs = zerolag.get_times_covered_by_files()

        node.add_input_list_opt('--zero-lag-coincs', zerolag)

        if isinstance(full_data, list):
            node.add_input_list_opt('--full-data-background', full_data)
        else:
            node.add_input_opt('--full-data-background', full_data)

        node.add_input_list_opt('--mixed-coincs-inj-full', injfull)
        node.add_input_list_opt('--mixed-coincs-full-inj', fullinj)

class PyCBCHDFInjFindExecutable(Executable):
    """Find injections in the hdf files output"""

    def create_node(self, inj_coinc_file, inj_xml_file, veto_file, veto_name, tags=None):
            tags = []
        node.add_input_list_opt('--trigger-file', inj_coinc_file)
        node.add_input_list_opt('--injection-file', inj_xml_file)
        if veto_name is not None:
            node.add_input_opt('--veto-file', veto_file)
        node.add_opt('--segment-name', veto_name)
        node.new_output_file_opt(inj_xml_file[0].segment, '.hdf',
                                 '--output-file', tags=tags)

class PyCBCDistributeBackgroundBins(Executable):
    """Distribute coinc files among different background bins"""

    def create_node(self, coinc_files, bank_file, background_bins, tags=None):
            tags = []
        node.add_opt('--background-bins', ' '.join(background_bins))

        names = [b.split(':')[0] for b in background_bins]

        output_files = [File(coinc_files[0].ifo_list,
                             self.name,
                             coinc_files[0].segment,
                             directory=self.out_dir,
                             tags = tags + ['mbin-%s' % i],
                             extension='.hdf') for i in range(len(background_bins))]
        node.add_output_list_opt('--output-files', output_files)
        node.names = names

class PyCBCCombineStatmap(Executable):
    """Combine coincs over different bins and apply trials factor"""

    def create_node(self, statmap_files, tags=None):
            tags = []
        node.add_input_list_opt('--statmap-files', statmap_files)
        node.new_output_file_opt(statmap_files[0].segment, '.hdf',

class PyCBCAddStatmap(PyCBCCombineStatmap):
    """Combine statmap files and add FARs over different coinc types"""

    def create_node(self, statmap_files, background_files, tags=None):
            tags = []
        node = super(PyCBCAddStatmap, self).create_node(statmap_files,
                                                            tags=tags)
        # Enforce upper case
        ctags = [t.upper() for t in (tags + self.tags)]
        if 'INJECTIONS' in ctags:
            node.add_input_list_opt('--background-files', background_files)

class PyCBCExcludeZerolag(Executable):
    """ Remove times of zerolag coincidences of all types from exclusive
        background """
    def create_node(self, statmap_file, other_statmap_files, tags=None):
            tags = []
        node.add_input_opt('--statmap-file', statmap_file)
        node.add_input_list_opt('--other-statmap-files',
                                other_statmap_files)
        node.new_output_file_opt(statmap_file.segment, '.hdf',
                                 '--output-file', tags=None)

class MergeExecutable(Executable):

class CensorForeground(Executable):

def make_foreground_censored_veto(workflow, bg_file, veto_file, veto_name,
                                  censored_name, out_dir, tags=None):
    tags = [] if tags is None else tags
    node = CensorForeground(workflow.cp, 'foreground_censor', ifos=workflow.ifos,
                            out_dir=out_dir, tags=tags).create_node()
    node.add_input_opt('--foreground-triggers', bg_file)
    node.add_input_opt('--veto-file', veto_file)
    node.add_opt('--segment-name', veto_name)
    node.add_opt('--output-segment-name', censored_name)
    node.new_output_file_opt(workflow.analysis_time, '.xml', '--output-file')
    workflow += node
    return node.output_files[0]

def merge_single_detector_hdf_files(workflow, bank_file, trigger_files, out_dir, tags=None):
    if tags is None:
        tags = []
    make_analysis_dir(out_dir)
    out = FileList()
    for ifo in workflow.ifos:
        node = MergeExecutable(workflow.cp, 'hdf_trigger_merge',
                        ifos=ifo, out_dir=out_dir, tags=tags).create_node()
        node.add_input_list_opt('--trigger-files', trigger_files.find_output_with_ifo(ifo))
        node.new_output_file_opt(workflow.analysis_time, '.hdf', '--output-file')
        workflow += node
        out += node.output_files
    return out

def setup_trigger_fitting(workflow, insps, hdfbank, veto_file, veto_name,
                          output_dir=None, tags=None):
    if not workflow.cp.has_option('workflow-coincidence', 'do-trigger-fitting'):
        return FileList()
    else:
        smoothed_fit_files = FileList()
        for i in workflow.ifos:
            ifo_insp = [insp for insp in insps if (insp.ifo == i)]
            assert len(ifo_insp)==1
            ifo_insp = ifo_insp[0]
            raw_exe = PyCBCFitByTemplateExecutable(workflow.cp,
                                                   'fit_by_template', ifos=i,
                                                   out_dir=output_dir,
                                                   tags=tags)
            raw_node = raw_exe.create_node(ifo_insp, hdfbank,
                                           veto_file, veto_name)
            workflow += raw_node
            smooth_exe = PyCBCFitOverParamExecutable(workflow.cp,
                                                     'fit_over_param', ifos=i,
                                                     out_dir=output_dir,
                                                     tags=tags)
            smooth_node = smooth_exe.create_node(raw_node.output_file,
                                                 hdfbank)
            workflow += smooth_node
            smoothed_fit_files += smooth_node.output_files
        return smoothed_fit_files

def find_injections_in_hdf_coinc(workflow, inj_coinc_file, inj_xml_file,
                                 veto_file, veto_name, out_dir, tags=None):
        tags = []
    exe = PyCBCHDFInjFindExecutable(workflow.cp, 'hdfinjfind',
                                    ifos=workflow.ifos,
                                    out_dir=out_dir, tags=tags)
    node = exe.create_node(inj_coinc_file, inj_xml_file, veto_file, veto_name)

def convert_bank_to_hdf(workflow, xmlbank, out_dir, tags=None):
    """Return the template bank in hdf format"""
        tags = []
    #FIXME, make me not needed
    if len(xmlbank) > 1:
        raise ValueError('Can only convert a single template bank')

    logging.info('convert template bank to HDF')
    bank2hdf_exe = PyCBCBank2HDFExecutable(workflow.cp, 'bank2hdf',
                                            ifos=workflow.ifos,
                                            out_dir=out_dir, tags=tags)
    bank2hdf_node = bank2hdf_exe.create_node(xmlbank[0])
    workflow.add_node(bank2hdf_node)
    return bank2hdf_node.output_files

def convert_trig_to_hdf(workflow, hdfbank, xml_trigger_files, out_dir, tags=None):
    """Return the list of hdf5 trigger files outputs"""
        tags = []
    logging.info('convert single inspiral trigger files to hdf5')

    trig_files = FileList()
    for ifo, insp_group in zip(*xml_trigger_files.categorize_by_attr('ifo')):
        trig2hdf_exe = PyCBCTrig2HDFExecutable(workflow.cp, 'trig2hdf',
                                       ifos=ifo, out_dir=out_dir, tags=tags)
        _, insp_bundles = insp_group.categorize_by_attr('segment')
        for insps in insp_bundles:
            trig2hdf_node =  trig2hdf_exe.create_node(insps, hdfbank[0])
            workflow.add_node(trig2hdf_node)
            trig_files += trig2hdf_node.output_files
    return trig_files

def setup_statmap(workflow, ifos, coinc_files, out_dir, tags=None):

    statmap_exe = PyCBCStatMapExecutable(workflow.cp, 'statmap',
                                         ifos=ifos,
                                         tags=tags, out_dir=out_dir)

    ifolist = ' '.join(ifos)
    stat_node = statmap_exe.create_node(coinc_files, ifolist)
    workflow.add_node(stat_node)
    return stat_node.output_file

def setup_statmap_inj(workflow, ifos, coinc_files, background_file,
                      out_dir, tags=None):

    statmap_exe = PyCBCStatMapInjExecutable(workflow.cp,
                                            'statmap_inj',
                                            ifos=ifos,
                                            tags=tags, out_dir=out_dir)

    stat_node = statmap_exe.create_node(FileList(coinc_files['injinj']),
                                        background_file,
                                        FileList(coinc_files['injfull']),
                                        FileList(coinc_files['fullinj']),
                                        ifolist)
    return stat_node.output_files[0]

def setup_interval_coinc_inj(workflow, hdfbank, full_data_trig_files,
                             inj_trig_files, stat_files,
                             background_file, veto_file, veto_name,
                             out_dir, pivot_ifo, fixed_ifo, tags=None):
    """
    This function sets up exact match coincidence for injections
    """
        tags = []
    logging.info('Setting up coincidence for injections')

    # Wall time knob and memory knob
    factor = int(workflow.cp.get_opt_tags('workflow-coincidence',
                                          'parallelization-factor', tags))

    ffiles = {}
    ifiles = {}
    for ifo, ffi in zip(*full_data_trig_files.categorize_by_attr('ifo')):
        ffiles[ifo] = ffi[0]
    for ifo, ifi in zip(*inj_trig_files.categorize_by_attr('ifo')):
        ifiles[ifo] = ifi[0]

    injinj_files = FileList()
    injfull_files = FileList()
    fullinj_files = FileList()
    # For the injfull and fullinj separation we take the pivot_ifo on one side,
    # and the rest that are attached to the fixed_ifo on the other side
    for ifo in ifiles:  # ifiles is keyed on ifo
        if ifo == pivot_ifo:
            injinj_files.append(ifiles[ifo])
            injfull_files.append(ifiles[ifo])
            fullinj_files.append(ffiles[ifo])
        else:
            injfull_files.append(ffiles[ifo])
            fullinj_files.append(ifiles[ifo])

    combo = [(injinj_files, "injinj"),
             (injfull_files, "injfull"),
             (fullinj_files, "fullinj"),
            ]
    bg_files = {'injinj':[], 'injfull':[], 'fullinj':[]}

    for trig_files, ctag in combo:
        findcoinc_exe = PyCBCFindCoincExecutable(workflow.cp,
                                                 'coinc',
                                                 ifos=ifiles.keys(),
                                                 tags=tags + [ctag],
                                                 out_dir=out_dir)
        for i in range(factor):
            group_str = '%s/%s' % (i, factor)
            coinc_node = findcoinc_exe.create_node(trig_files, hdfbank,
                                                   stat_files,
                                                   veto_file, veto_name,
                                                   group_str,
                                                   pivot_ifo,
                                                   fixed_ifo,
                                                   tags=['JOB'+str(i)])

            bg_files[ctag] += coinc_node.output_files
            workflow.add_node(coinc_node)

    logging.info('...leaving coincidence for injections')

    return setup_statmap_inj(workflow, ifiles.keys(), bg_files,
                             background_file, out_dir,
                             tags=tags + [veto_name])

def setup_interval_coinc(workflow, hdfbank, trig_files, stat_files,
                         veto_file, veto_name, out_dir, pivot_ifo,
                         fixed_ifo, tags=None):
    """
    This function sets up exact match coincidence
    """
        tags = []
    logging.info('Setting up coincidence')

    ifos, _ = trig_files.categorize_by_attr('ifo')
    findcoinc_exe = PyCBCFindCoincExecutable(workflow.cp, 'coinc',
                                             ifos=ifos,
                                             tags=tags, out_dir=out_dir)

                                          'parallelization-factor',
                                          [findcoinc_exe.ifo_string] + tags))

    statmap_files = []
    bg_files = FileList()
    for i in range(factor):
        group_str = '%s/%s' % (i, factor)
        coinc_node = findcoinc_exe.create_node(trig_files, hdfbank,
                                               stat_files,
                                               veto_file, veto_name,
                                               group_str,
                                               pivot_ifo,
                                               fixed_ifo,
                                               tags=['JOB'+str(i)])
        bg_files += coinc_node.output_files
        workflow.add_node(coinc_node)

    statmap_files = setup_statmap(workflow, ifos, bg_files,
                                  out_dir, tags=tags)

    logging.info('...leaving coincidence ')
    return statmap_files

def select_files_by_ifo_combination(ifocomb, insps):
    """
    This function selects single-detector files ('insps') for a given ifo combination
    """
    inspcomb = FileList()
    for ifo, ifile in zip(*insps.categorize_by_attr('ifo')):
        if ifo in ifocomb:
            inspcomb += ifile

    return inspcomb

def get_ordered_ifo_list(ifocomb, ifo_ids):
    """
    This function sorts the combination of ifos (ifocomb) based on the given
    precedence list (ifo_ids dictionary) and returns the first ifo as pivot
    the second ifo as fixed, and the ordered list joined as a string.
    """
    # combination_prec stores precedence info for the detectors in the combination
    combination_prec = {ifo: ifo_ids[ifo] for ifo in ifocomb}
    ordered_ifo_list = sorted(combination_prec, key = combination_prec.get)
    pivot_ifo = ordered_ifo_list[0]
    fixed_ifo = ordered_ifo_list[1]

    return pivot_ifo, fixed_ifo, ''.join(ordered_ifo_list)

def setup_combine_statmap(workflow, final_bg_file_list, bg_file_list,
                          out_dir, tags=None):
    """
    Combine the statmap files into one background file
    """
        tags = []
    logging.info('Setting up combine statmap')

    cstat_exe_name = os.path.basename(workflow.cp.get("executables",
                                                      "combine_statmap"))
    if cstat_exe_name == 'pycbc_combine_statmap':
        cstat_class = PyCBCCombineStatmap
    elif cstat_exe_name == 'pycbc_add_statmap':
        cstat_class = PyCBCAddStatmap
    else:
        raise NotImplementedError('executable should be '
            'pycbc_combine_statmap or pycbc_add_statmap')

    cstat_exe = cstat_class(workflow.cp, 'combine_statmap', ifos=workflow.ifos,
                            tags=tags, out_dir=out_dir)

        combine_statmap_node = cstat_exe.create_node(final_bg_file_list)
        combine_statmap_node = cstat_exe.create_node(final_bg_file_list,
                                                     bg_file_list)

    workflow.add_node(combine_statmap_node)
    return combine_statmap_node.output_file

def setup_exclude_zerolag(workflow, statmap_file, other_statmap_files,
                          out_dir, ifos, tags=None):
    """
    Exclude single triggers close to zerolag triggers from forming any
    background events
    """
        tags = []
    logging.info('Setting up exclude zerolag')

    exc_zerolag_exe = PyCBCExcludeZerolag(workflow.cp, 'exclude_zerolag',
                                          ifos=ifos, tags=tags,
                                          out_dir=out_dir)
    exc_zerolag_node = exc_zerolag_exe.create_node(statmap_file,
                                                   other_statmap_files,
                                                   tags=None)
    workflow.add_node(exc_zerolag_node)
    return exc_zerolag_node.output_file

def rerank_coinc_followup(workflow, statmap_file, bank_file, out_dir,
                          tags=None,
                          injection_file=None,
                          ranking_file=None):
        tags = []

    if not workflow.cp.has_section("workflow-rerank"):
        logging.info("No reranking done in this workflow")
        return statmap_file
    else:
        logging.info("Setting up reranking of candidates")

    # Generate reduced data files (maybe this could also be used elsewhere?)
    stores = FileList([])
        make_analysis_dir('strain_files')
        node = Executable(workflow.cp, 'strain_data_reduce', ifos=[ifo],
                          out_dir='strain_files', tags=tags).create_node()
        node.add_opt('--gps-start-time', workflow.analysis_time[0])
        node.add_opt('--gps-end-time', workflow.analysis_time[1])
        if injection_file:
            node.add_input_opt('--injection-file', injection_file)

        fil = node.new_output_file_opt(workflow.analysis_time, '.hdf',
                                       '--output-file')
        stores.append(fil)

    # Generate trigger input file
    node = Executable(workflow.cp, 'rerank_trigger_input', ifos=workflow.ifos,
                      out_dir=out_dir, tags=tags).create_node()
    node.add_input_opt('--statmap-file', statmap_file)
    node.add_input_opt('--bank-file', bank_file)
    trigfil = node.new_output_file_opt(workflow.analysis_time, '.hdf',
                                   '--output-file')

    # Parallelize coinc trigger followup
    factor = int(workflow.cp.get_opt_tags("workflow-rerank",
                                          "parallelization-factor", tags))
    exe = Executable(workflow.cp, 'coinc_followup', ifos=workflow.ifos,
                     out_dir=out_dir, tags=tags)

    stat_files = FileList([])
        node = exe.create_node()
        node.new_output_file_opt(workflow.analysis_time, '.hdf',
                                 '--output-file', tags=[str(i)])
        node.add_multiifo_input_list_opt('--hdf-store', stores)
        node.add_input_opt('--input-file', trigfil)
        node.add_opt('--start-index', str(i))
        node.add_opt('--stride', factor)
        stat_files += node.output_files

    exe = Executable(workflow.cp, 'rerank_coincs', ifos=workflow.ifos,
    node = exe.create_node()
    node.add_input_list_opt('--stat-files', stat_files)
    node.add_input_opt('--followup-file', trigfil)

    if ranking_file:
        node.add_input_opt('--ranking-file', ranking_file)

    node.new_output_file_opt(workflow.analysis_time, '.hdf',
                             '--output-file')
    return node.output_file

import re
try:
    import configparser as ConfigParser
    import ConfigParser

def parse_workflow_ini_file(cpFile,parsed_filepath=None):
    """Read a .ini file in, parse it as described in the documentation linked
    to above, and return the parsed ini file.

    cpFile : The path to a .ini file to be read in
    parsed_filepath: Boolean, optional
        If provided, the .ini file, after parsing, will be written to this
        location

    Returns
    -------
    cp: The parsed ConfigParser class containing the read in .ini file
    """
    # First read the .ini file
    cp = read_ini_file(cpFile)
    print(cp.sections())

    # Check for any substitutions that can be made
    # FIXME: The python 3 version of ConfigParser can do this automatically
    # move over to that if it can be backported to python2.X.
    # We use the same formatting as the new configparser module when doing
    # ExtendedInterpolation
    # This is described at http://docs.python.org/3.4/library/configparser.html
    #cp = perform_extended_interpolation(cp)

    # Split sections like [inspiral&tmplt] into [inspiral] and [tmplt]
    cp = split_multi_sections(cp)

    # Check for duplicate options in sub-sections
    sanity_check_subsections(cp)

    # Dump parsed .ini file if needed
    if parsed_filepath:
        fp = open(parsed_filepath,'w')
        cp.write(fp)

    return cp

def read_ini_file(cpFile):
    """Read a .ini file and return it as a ConfigParser class.
    This function does none of the parsing/combining of sections. It simply
    reads the file and returns it unedited

    Returns
    -------
    cp: The ConfigParser class containing the read in .ini file
    """

    # Initialise ConfigParser class
    cp = ConfigParser.ConfigParser(\
        interpolation=ConfigParser.ExtendedInterpolation())
    # Read the file
    fp = open(cpFile,'r')
    cp.read_file(fp)
    return cp

def perform_extended_interpolation(cp,preserve_orig_file=False):
    """Filter through an ini file and replace all examples of
    ExtendedInterpolation formatting with the exact value. For values like
    ${example} this is replaced with the value that corresponds to the option
    called example ***in the same section***

    For values like ${common|example} this is replaced with the value that
    corresponds to the option example in the section [common]. Note that
    in the python3 config parser this is ${common:example} but python2.7
    interprets the : the same as a = and this breaks things

    Nested interpolation is not supported here.

    cp: ConfigParser object
    preserve_orig_file: Boolean, optional
        By default the input ConfigParser object will be modified in place. If
        this is set deepcopy will be used and the input will be preserved.
        Default = False

    Returns
    -------
    cp: parsed ConfigParser object
    """
    # Deepcopy the cp object if needed
    if preserve_orig_file:
        cp = copy.deepcopy(cp)

    # Do not allow any interpolation of the section names
    for section in cp.sections():
         for option,value in cp.items(section):
             # Check the option name
             newStr = interpolate_string(option,cp,section)
             if newStr != option:
                 cp.set(section,newStr,value)
                 cp.remove_option(section,option)
             # Check the value
             newStr = interpolate_string(value,cp,section)
             if newStr != value:
                 cp.set(section,option,newStr)

    return cp

def interpolate_string(testString,cp,section):
    """Take a string and replace all example of ExtendedInterpolation formatting
    within the string with the exact value.

    For values like ${example} this is replaced with the value that corresponds
    to the option called example ***in the same section***

    testString: String
        The string to parse and interpolate
        The ConfigParser object to look for the interpolation strings within
    section: String
        The current section of the ConfigParser object

    Returns
        Interpolated string
    """

    # First check if any interpolation is needed and abort if not
    reObj = re.search(r"\$\{.*?\}", testString)
    while reObj:
        # Not really sure how this works, but this will obtain the first
        # instance of a string contained within ${....}
        repString = (reObj).group(0)[2:-1]
        # Need to test which of the two formats we have
        splitString = repString.split('|')
        if len(splitString) == 1:
            testString = testString.replace('${'+repString+'}',\
                                    cp.get(section,splitString[0]))
        if len(splitString) == 2:
                                    cp.get(splitString[0],splitString[1]))
        reObj = re.search(r"\$\{.*?\}", testString)

    return testString

def split_multi_sections(cp,preserve_orig_file=False):
    """Parse through a supplied ConfigParser object and splits any sections
    labelled with an "&" sign (for e.g. [inspiral&tmpltbank]) into [inspiral]
    and [tmpltbank] sections. If these individual sections already exist they
    will be appended to. If an option exists in both the [inspiral] and
    [inspiral&tmpltbank] sections an error will be thrown

    cp: The ConfigParser class

    Returns
    """

    # Begin by looping over all sections
        # Only continue if section needs splitting
        if '&' not in section:
            continue
        # Get list of section names to add these options to
        splitSections = section.split('&')
        for newSec in splitSections:
            # Add sections if they don't already exist
            if not cp.has_section(newSec):
                cp.add_section(newSec)
            add_options_to_section(cp,newSec,cp.items(section))
        cp.remove_section(section)
    return cp

def sanity_check_subsections(cp):
    """This function goes through the ConfigParset and checks that any options
    given in the [SECTION_NAME] section are not also given in any
    [SECTION_NAME-SUBSECTION] sections.

    Returns
    None
    """
    # Loop over the sections in the ini file
        # Loop over the sections again
        for section2 in cp.sections():
            # Check if any are subsections of section
            if section2.startswith(section + '-'):
                # Check for duplicate options whenever this exists
                check_duplicate_options(cp,section,section2,raise_error=True)

def add_options_to_section(cp,section,items,preserve_orig_file=False,\
                           overwrite_options=False):
    """Add a set of options and values to a section of a ConfigParser object.
    Will throw an error if any of the options being added already exist, this
    behaviour can be overridden if desired

    section: string
        The name of the section to add options+values to
    items: list of tuples
        Each tuple contains (at [0]) the option and (at [1]) the value to add
        to the section of the ini file
    overwrite_options: Boolean, optional
        By default this function will throw a ValueError if an option exists in
        both the original section in the ConfigParser *and* in the provided
        items. This will override so that the options+values given in items
        will replace the original values if the value is set to True.
        Default = True
    Returns
    """
    # Sanity checking
    if not cp.has_section(section):
        raise ValueError('Section %s not present in ConfigParser.' %(section,))

    # Check for duplicate options first
    for option,value in items:
        if not overwrite_options:
            if option in cp.options(section):
                raise ValueError('Option %s exists in both original' + \
                                 'ConfigParser and input list' %(option,))
        cp.set(section,option,value)

    return cp

def check_duplicate_options(cp,section1,section2,raise_error=False):
    """Check for duplicate options in two sections, section1 and section2.
    Will return True if there are duplicate options and False if not

    section1: string
        The name of the first section to compare
    section2: string
        The name of the second section to compare
    raise_error: Boolean, optional
        If True, raise an error if duplicates are present.

    Returns
    duplicate: List
        List of duplicate options
    """
    if not cp.has_section(section1):
        raise ValueError('Section %s not present in ConfigParser.'%(section1,))
    if not cp.has_section(section2):
        raise ValueError('Section %s not present in ConfigParser.'%(section2,))

    items1 = cp.items(section1)
    items2 = cp.items(section2)

    # The list comprehension here creates a list of all duplicate items
    duplicates = [x for x in items1 if x in items2]

    if duplicates and raise_error:
        raise ValueError('The following options appear in both section ' +\
                         '%s and %s: %s' \
                         %(section1,section2,duplicates.join(' ')))

    return duplicates

# Copyright (C) 2013,2017 Ian Harry, Duncan Brown
#
#
#

#
#
#                                   Preamble
#
#
"""
This module provides a wrapper to the ConfigParser utilities for pycbc
workflow construction. This module is described in the page here:
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/ahope/initialization_inifile.html
"""

import os
import re
import stat
import requests
import distutils.spawn
import six
from six.moves import http_cookiejar as cookielib
from six.moves.http_cookiejar import (
    _warn_unhandled_exception,
    LoadError,
    Cookie,
)
from bs4 import BeautifulSoup

def _really_load(self, f, filename, ignore_discard, ignore_expires):
    """
    This function is required to monkey patch MozillaCookieJar's _really_load
    function which does not understand the curl format cookie file created
    by ecp-cookie-init. It patches the code so that #HttpOnly_ get loaded.

    https://bugs.python.org/issue2190
    https://bugs.python.org/file37625/httponly.patch
    """
    now = time.time()

    magic = f.readline()
    if not re.search(self.magic_re, magic):
        f.close()
        raise LoadError(
            "%r does not look like a Netscape format cookies file" % filename
        )

    try:
        while 1:
            line = f.readline()
            if line == "":
                break

            # last field may be absent, so keep any trailing tab
            if line.endswith("\n"):
                line = line[:-1]

            sline = line.strip()
            # support HttpOnly cookies (as stored by curl or old Firefox).
            if sline.startswith("#HttpOnly_"):
                line = sline[10:]
            # skip comments and blank lines ... what is $ for?
            elif sline.startswith(("#", "$")) or sline == "":
                continue

            (
                domain,
                domain_specified,
                path,
                secure,
                expires,
                name,
                value,
            ) = line.split("\t")
            secure = secure == "TRUE"
            domain_specified = domain_specified == "TRUE"
            if name == "":
                # cookies.txt regards 'Set-Cookie: foo' as a cookie
                # with no name, whereas cookielib regards it as a
                # cookie with no value.
                name = value
                value = None

            initial_dot = domain.startswith(".")
            assert domain_specified == initial_dot

            discard = False
            if expires == "":
                expires = None
                discard = True

            # assume path_specified is false
            c = Cookie(
                0,
                name,
                value,
                None,
                False,
                domain,
                initial_dot,
                path,
                False,
                secure,
                expires,
                discard,
                None,
                None,
                {},
            )
            if not ignore_discard and c.discard:
                continue
            if not ignore_expires and c.is_expired(now):
                continue
            self.set_cookie(c)

        raise
    except Exception:
        _warn_unhandled_exception()
            "invalid Netscape format cookies file %r: %r" % (filename, line)
        )

# Now monkey patch the code
cookielib.MozillaCookieJar._really_load = _really_load  # noqa

ecp_cookie_error = """The attempt to download the file at

{}

was redirected to the git.ligo.org sign-in page. This means that you likely
forgot to initialize your ECP cookie or that your LIGO.ORG credentials are
otherwise invalid. Create a valid ECP cookie for git.ligo.org by running

ecp-cookie-init LIGO.ORG https://git.ligo.org/users/auth/shibboleth/callback albert.einstein

before attempting to download files from git.ligo.org.
"""

def istext(s, text_characters=None, threshold=0.3):
    """
    Determines if the string is a set of binary data or a text file.
    This is done by checking if a large proportion of characters are > 0X7E
    (0x7F is <DEL> and unprintable) or low bit control codes. In other words
    things that you wouldn't see (often) in a text file. (ASCII past 0x7F
    might appear, but rarely).

    Code modified from
    https://www.safaribooksonline.com/library/view/python-cookbook-2nd/0596007973/ch01s12.html
    """
    # if s contains any null, it's not text:
    if six.PY2 and "\0" in s:
    # an "empty" string is "text" (arbitrary but reasonable choice):
    if not s:

    text_characters = "".join(map(chr, range(32, 127))) + "\n\r\t\b"
        _null_trans = string.maketrans("", "")
        # Get the substring of s made up of non-text characters
        t = s.translate(_null_trans, text_characters)
    else:
        # Not yet sure how to deal with this in python3. Will need example.

        # trans = str.maketrans('', '', text_characters)
        # t = s.translate(trans)

    # s is 'text' if less than 30% of its characters are non-text ones:
    return len(t) / float(len(s)) <= threshold

def resolve_url(url, directory=None, permissions=None, copy_to_cwd=True):
    """Resolves a URL to a local file, and returns the path to that file.

    If a URL is given, the file will be copied to the current working
    directory. If a local file path is given, the file will only be copied
    to the current working directory if ``copy_to_cwd`` is ``True``
    (the default).
    """

    u = urlparse(url)

    # determine whether the file exists locally
    islocal = u.scheme == "" or u.scheme == "file"

    if not islocal or copy_to_cwd:
        # create the name of the destination file
        if directory is None:
            directory = os.getcwd()
        filename = os.path.join(directory, os.path.basename(u.path))
    else:
        filename = u.path

    if islocal:
        # check that the file exists
        if not os.path.isfile(u.path):
            errmsg = "Cannot open file %s from URL %s" % (u.path, url)
            raise ValueError(errmsg)
        # for regular files, make a direct copy if requested
        elif copy_to_cwd:
            if os.path.isfile(filename):
                # check to see if src and dest are the same file
                src_inode = os.stat(u.path)[stat.ST_INO]
                dst_inode = os.stat(filename)[stat.ST_INO]
                if src_inode != dst_inode:
                    shutil.copy(u.path, filename)
            else:
                shutil.copy(u.path, filename)

    elif u.scheme == "http" or u.scheme == "https":
        s = requests.Session()
        s.mount(
            str(u.scheme) + "://", requests.adapters.HTTPAdapter(max_retries=5)
        )

        # look for an ecp cookie file and load the cookies
        cookie_dict = {}
        ecp_file = "/tmp/ecpcookie.u%d" % os.getuid()
        if os.path.isfile(ecp_file):
            cj = cookielib.MozillaCookieJar()
            cj.load(ecp_file, ignore_discard=True, ignore_expires=True)
        else:
            cj = []

        for c in cj:
            if c.domain == u.netloc:
                # load cookies for this server
                cookie_dict[c.name] = c.value
            elif (
                u.netloc == "code.pycbc.phy.syr.edu"
                and c.domain == "git.ligo.org"
            ):
                # handle the redirect for code.pycbc to git.ligo.org

        r = s.get(url, cookies=cookie_dict, allow_redirects=True)
        if r.status_code != 200:
            errmsg = "Unable to download %s\nError code = %d" % (
                url,
                r.status_code,
            )

        # if we are downloading from git.ligo.org, check that we
        # did not get redirected to the sign-in page
        if u.netloc == "git.ligo.org" or u.netloc == "code.pycbc.phy.syr.edu":
            # Check if we have downloaded a binary file.
            if istext(r.content):
                soup = BeautifulSoup(r.content, "html.parser")
                desc = soup.findAll(attrs={"property": "og:url"})
                if (
                    len(desc)
                    and desc[0]["content"]
                    == "https://git.ligo.org/users/sign_in"
                ):
                    raise ValueError(ecp_cookie_error.format(url))

        output_fp = open(filename, "wb")
        output_fp.write(r.content)
        output_fp.close()

    else:
        # TODO: We could support other schemes such as gsiftp by
        # calling out to globus-url-copy
        errmsg = "Unknown URL scheme: %s\n" % (u.scheme)
        errmsg += "Currently supported are: file, http, and https."
        raise ValueError(errmsg)

    if not os.path.isfile(filename):
        errmsg = "Error trying to create file %s from %s" % (filename, url)

    if permissions:
        if os.access(filename, os.W_OK):
            os.chmod(filename, permissions)
        else:
            # check that the file has at least the permissions requested
            s = os.stat(filename)[stat.ST_MODE]
            if (s & permissions) != permissions:
                errmsg = "Could not change permissions on %s (read-only)" % url

    return filename

def add_workflow_command_line_group(parser):
    """
    The standard way of initializing a ConfigParser object in workflow will be
    to do it from the command line. This is done by giving a

    --local-config-files filea.ini fileb.ini filec.ini

    command. You can also set config file override commands on the command
    line. This will be most useful when setting (for example) start and
    end times, or active ifos. This is done by

    --config-overrides section1:option1:value1 section2:option2:value2 ...

    This can also be given as

    --config-overrides section1:option1

    where the value will be left as ''.

    To remove a configuration option, use the command line argument

    --config-delete section1:option1

    which will delete option1 from [section1] or

    --config-delete section1

    to delete all of the options in [section1]

    Deletes are implemented before overrides.

    This function returns an argparse OptionGroup to ensure these options are
    parsed correctly and can then be sent directly to initialize an
    WorkflowConfigParser.

    parser : argparse.ArgumentParser instance
        The initialized argparse instance to add the workflow option group to.
    """
    workflowArgs = parser.add_argument_group(
        "Configuration", "Options needed for parsing " "config file(s)."
    )
    workflowArgs.add_argument(
        "--config-files",
        nargs="+",
        action="store",
        metavar="CONFIGFILE",
        help="List of config files to be used in " "analysis.",
    )
        "--config-overrides",
        nargs="*",
        metavar="SECTION:OPTION:VALUE",
        help="List of section,option,value combinations to "
        "add into the configuration file. Normally the gps "
        "start and end times might be provided this way, "
        "and user specific locations (ie. output directories). "
        "This can also be provided as SECTION:OPTION or "
        "SECTION:OPTION: both of which indicate that the "
        "corresponding value is left blank.",
    )
        "--config-delete",
        metavar="SECTION:OPTION",
        help="List of section,option combinations to delete "
        "from the configuration file. This can also be "
        "provided as SECTION which deletes the enture section"
        " from the configuration file or SECTION:OPTION "
        "which deletes a specific option from a given "
        "section.",
    )

class WorkflowConfigParser(InterpolatingConfigParser):
    """
    This is a sub-class of InterpolatingConfigParser, which lets
    """

        self,
        copy_to_cwd=False,
    ):
        """
        Initialize an WorkflowConfigParser. This reads the input configuration
        files, overrides values if necessary and performs the interpolation.
        See https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/ahope/initialization_inifile.html

        configFiles : Path to .ini file, or list of paths
            The file(s) to be read in and parsed.
        overrideTuples : List of (section, option, value) tuples
            Add the (section, option, value) triplets provided
            in this list to the provided .ini file(s). If the section, option
            pair is already present, it will be overwritten.
        parsedFilePath : Path, optional (default=None)
            If given, write the parsed .ini file back to disk at this location.
        deleteTuples : List of (section, option) tuples
            Delete the (section, option) pairs provided
            in this list from provided .ini file(s). If the section only
            is provided, the entire section will be deleted.
        copy_to_cwd : bool, optional
            Copy the configuration files to the current working directory if
            they are not already there, even if they already exist locally.
            If False, files will only be copied to the current working
            directory if they are remote. Default is False.

        Returns
        --------
        WorkflowConfigParser
            Initialized WorkflowConfigParser instance.
        """
        if configFiles is not None:
            configFiles = [
                resolve_url(cFile, copy_to_cwd=copy_to_cwd)
                for cFile in configFiles
            ]

        InterpolatingConfigParser.__init__(
            self,
            configFiles,
            overrideTuples,
            parsedFilePath,
            deleteTuples,
            skip_extended=True,
        )
        # expand executable which statements
        self.perform_exe_expansion()

        self.perform_extended_interpolation()

        # Resolve any URLs needing resolving
        self.curr_resolved_files = {}
        self.resolve_urls()

    def perform_exe_expansion(self):
        """
        This function will look through the executables section of the
        ConfigParser object and replace any values using macros with full paths.

        For any values that look like

        ${which:lalapps_tmpltbank}

        will be replaced with the equivalent of which(lalapps_tmpltbank)

        Otherwise values will be unchanged.
        """
        # Only works on executables section
        if self.has_section("executables"):
            for option, value in self.items("executables"):
                newStr = self.interpolate_exe(value)
                if newStr != value:
                    self.set("executables", option, newStr)

    def interpolate_exe(self, testString):
        """
        Replace testString with a path to an executable based on the format.

        If this looks like

        it will return the equivalent of which(lalapps_tmpltbank)

        Otherwise it will return an unchanged string.

        testString : string
            The input string

        Returns
        --------
        newString : string
            The output string.
        """
        testString = testString.strip()
        if not (testString.startswith("${") and testString.endswith("}")):
            return testString

        # This may not be an exe interpolation, so even if it has ${ ... } form
        # I may not have to do anything
        newString = testString

        # Strip the ${ and }
        testString = testString[2:-1]

        testList = testString.split(":")

        # Maybe we can add a few different possibilities for substitution
        if len(testList) == 2:
            if testList[0] == "which":
                newString = distutils.spawn.find_executable(testList[1])
                if not newString:
                    errmsg = "Cannot find exe %s in your path " % (testList[1])
                    errmsg += "and you specified ${which:%s}." % (testList[1])
                    raise ValueError(errmsg)

        return newString

    def section_to_cli(self, section, skip_opts=None):
        """Converts a section into a command-line string.

        For example:

        .. code::

            [section_name]
            foo =
            bar = 10

        yields: `'--foo --bar 10'`.

            The name of the section to convert.
            List of options to skip. Default (None) results in all options
            in the section being converted.

        Returns
        -------
        str :
            The options as a command-line string.
        """
        read_opts = [
            opt for opt in self.options(section) if opt not in skip_opts
        ]
        opts = []
        for opt in read_opts:
            opts.append("--{}".format(opt))
            val = self.get(section, opt)
            if val != "":
                opts.append(val)
        return " ".join(opts)

    def get_cli_option(self, section, option_name, **kwds):
        """Return option using CLI action parsing

        section: str
            Section to find option to parse
        option_name: str
            Name of the option to parse from the config file
        kwds: keywords
            Additional keywords are passed directly to the argument parser.

        Returns
        -------
        value:
            The parsed value for this option
        """
        import argparse

        optstr = self.section_to_cli(section)
        parser = argparse.ArgumentParser()
        name = "--" + option_name.replace("_", "-")
        parser.add_argument(name, **kwds)
        args, _ = parser.parse_known_args(optstr.split())
        return getattr(args, option_name)

    def resolve_urls(self):
        """
        This function will look through all sections of the
        ConfigParser object and replace any URLs that are given the resolve
        magic flag with a path on the local drive.

        Specifically for any values that look like

        ${resolve:https://git.ligo.org/detchar/SOME_GATING_FILE.txt}

        the file will be replaced with the output of resolve_url(URL)

        """
                new_str = self.resolve_file_url(value)
                if new_str is not None and new_str != value:

    def resolve_file_url(self, test_string):
        """
        Replace test_string with a path to an executable based on the format.

        test_string : string

        Returns
        --------
        new_string : string
        """
        test_string = test_string.strip()
        if not (test_string.startswith("${") and test_string.endswith("}")):
            return test_string

        # This may not be a "resolve" interpolation, so even if it has
        # ${ ... } form I may not have to do anything

        test_string = test_string[2:-1]

        test_list = test_string.split(":", 1)

        if len(test_list) == 2:
            if test_list[0] == "resolve":
                curr_lfn = os.path.basename(test_list[1])
                if curr_lfn in self.curr_resolved_files:
                    return self.curr_resolved_files[curr_lfn]
                local_url = resolve_url(test_list[1])
                self.curr_resolved_files[curr_lfn] = local_url
                return local_url

# Copyright (C) 2013, 2017  Ian Harry, Alex Nitz, Duncan Brown
#
#
#

#
#
#                                   Preamble
#
#
"""
This module provides the worker functions and classes that are used when
creating a workflow. For details about the workflow module see here:
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/ahope.html
"""
import sys, os, stat, subprocess, logging, math, string
from six.moves import urllib
from six.moves.urllib.request import pathname2url
from six.moves.urllib.parse import urljoin
import numpy, random
from itertools import combinations, groupby, permutations
from operator import attrgetter
import lal
import lal.utils
import Pegasus.DAX3
from glue import lal as gluelal
from glue.ligolw import table, lsctables, ligolw
from glue.ligolw.utils import segments as ligolw_segments
from pycbc import makedir
from pycbc.workflow.configuration import WorkflowConfigParser, resolve_url
from pycbc.workflow import pegasus_workflow

class ContentHandler(ligolw.LIGOLWContentHandler):
    pass

lsctables.use_in(ContentHandler)

#REMOVE THESE FUNCTIONS  FOR PYTHON >= 2.7 ####################################
def check_output_error_and_retcode(*popenargs, **kwargs):
    """
    This function is used to obtain the stdout of a command. It is only used
    internally, recommend using the make_external_call command if you want
    to call external executables.
    """
    if 'stdout' in kwargs:
        raise ValueError('stdout argument not allowed, it will be overridden.')
    process = subprocess.Popen(stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               *popenargs, **kwargs)
    output, error = process.communicate()
    retcode = process.poll()
    return output, error, retcode

def check_output(*popenargs, **kwargs):
    output, _, _ = check_output_error_and_retcode(*popenargs, **kwargs)

###############################################################################

def make_analysis_dir(path):
    """
    Make the analysis directory path, any parent directories that don't already
    exist, and the 'logs' subdirectory of path.
    """
        makedir(os.path.join(path, 'logs'))

def is_condor_exec(exe_path):
    """
    Determine if an executable is condor-compiled

    exe_path : str
          The executable path

    Returns
    -------
    truth_value  : boolean
        Return True if the exe is condor compiled, False otherwise.
    """
    if str(check_output(['nm', '-a', exe_path])).find('condor') != -1:
    else:

file_input_from_config_dict = {}

class Executable(pegasus_workflow.Executable):
    # These are the file retention levels
    INTERMEDIATE_PRODUCT = 1
    ALL_TRIGGERS = 2
    MERGED_TRIGGERS = 3
    FINAL_RESULT = 4

    # Set this parameter to indicate that this option is used to specify a
    # file and is *not* handled explicitly in the create_node or __init__
    # methods of the sub-class. Usually that is to say that this option is a
    # file and is normally specified in an file, e.g. a PSD file. As files
    # need to be identified as such to pegasus, this attempts to catch this
    # case.
    # file_input_options = ['--psd-file, '--bank-file'] (as an example)
    file_input_options = []

    # Set this parameter to indicate that this option should take different
    # values based on the time. E.g. something like
    # --option1 value1[0:1000],value2[1000:2000]
    # would be replaced with --option1 value1 if the time is within 0,1000 and
    # value2 if in 1000,2000. A failure will be replaced if the job time is
    # not fully contained in one of these windows, or if fully contained in
    # multiple of these windows. This is resolved when creating the Job from
    # the Executable
    time_dependent_options = []

    # This is the default value. It will give a warning if a class is
    # used where the retention level is not set. The file will still be stored
    KEEP_BUT_RAISE_WARNING = 5
    _warned_classes_list = ['Executable']

    # Sub classes, or instances, should override this. If not overriden the
    # file will be retained, but a warning given
    current_retention_level = KEEP_BUT_RAISE_WARNING
    def __init__(self, cp, name,
                 universe=None, ifos=None, out_dir=None, tags=None,
                 reuse_executable=True):
        """
        Initialize the Executable class.

        cp : ConfigParser object
            The ConfigParser object holding the workflow configuration settings
        exec_name : string
            Executable name
        universe : string, optional
            Condor universe to run the job in
        ifos : string or list, optional
            The ifo(s) that the Job is valid for. If the job is
            independently valid for multiple ifos it can be provided as a list.
            Ie. ['H1',L1','V1'], if the job is only valid for the combination
            of ifos (for e.g. ligolw_thinca) then this can be supplied
            as, for e.g. "H1L1V1".
        out_dir: path, optional
            The folder to store output files of this job.
            A list of strings that is used to identify this job.
        """
        if isinstance(ifos, string_types):
            self.ifo_list = [ifos]
        else:
            self.ifo_list = ifos
        if self.ifo_list is not None:
            self.ifo_list = sorted(self.ifo_list)
            self.ifo_string = ''.join(self.ifo_list)
        else:
            self.ifo_string = None
        self.cp = cp
        self.universe=universe
        self.container_cls = None
        self.container_type = None

        try:
            self.installed = cp.getboolean('pegasus_profile-%s' % name, 'pycbc|installed')
        except:
            self.installed = True

        self.name=name

        self.update_current_tags(tags)

        self.update_output_directory(out_dir=out_dir)

        # Determine the level at which output files should be kept
        self.update_current_retention_level(self.current_retention_level)

        # Should I reuse this executable?
        if reuse_executable:
            self.pegasus_name = self.name
        else:
            self.pegasus_name = self.tagged_name

        # Determine if this executables should be run in a container
        try:
            self.container_type = cp.get('pegasus_profile-%s' % name,
                                         'container|type')
        except:
            pass

        if self.container_type is not None:
            self.container_img = cp.get('pegasus_profile-%s' % name,
                                        'container|image')
            try:
                self.container_site = cp.get('pegasus_profile-%s' % name,
                                             'container|image_site')
            except:
                self.container_site = 'local'

            try:
                self.container_mount = cp.get('pegasus_profile-%s' % name,
                                             'container|mount').split(',')
            except:
                self.container_mount = None

            self.container_cls = Pegasus.DAX3.Container("{}-container".format(
                                                    name),
                                                    self.container_type,
                                                    self.container_img,
                                                    imagesite=self.container_site,
                                                    mount=self.container_mount)

            super(Executable, self).__init__(self.pegasus_name,
                                             installed=self.installed,
                                             container=self.container_cls)

        else:
                                             installed=self.installed)

        self._set_pegasus_profile_options()

        # Check that the executable actually exists locally or
        # looks like a URL, in which case trust Pegasus to be
        # able to fetch it.
        exe_path = cp.get('executables', name)
        self.needs_fetching = False

        exe_url = urllib.parse.urlparse(exe_path)

        # See if the user specified a list of sites for the executable
        try:
            exe_site_list = cp.get('pegasus_profile-%s' % name, 'pycbc|site')
        except:
            exe_site_list = 'local'

        for s in exe_site_list.split(','):
            exe_site = s.strip()

            if exe_url.scheme in ['', 'file']:
                if exe_site is 'local':
                    # Check that executables at file urls
                    #  on the local site exist
                    if os.path.isfile(exe_url.path) is False:
                        raise TypeError("Failed to find %s executable "
                                        "at %s on site %s" % (name, exe_path,
                                        exe_site))
            else:
                # Could be http, gsiftp, etc. so it needs fetching if run now
                self.needs_fetching = True

            self.add_pfn(exe_path, site=exe_site)
            logging.info("Using %s executable "
                         "at %s on site %s" % (name, exe_url.path, exe_site))

        # Determine the condor universe if we aren't given one
        if self.universe is None:
            if is_condor_exec(exe_path):
                self.universe = 'standard'
            else:
                self.universe = 'vanilla'

        if not self.universe == 'vanilla':
            logging.info("%s executable will run as %s universe"
                         % (name, self.universe))

        self.set_universe(self.universe)

        if hasattr(self, "group_jobs"):
            self.add_profile('pegasus', 'clusters.size', self.group_jobs)

    @property
    def ifo(self):
        """Return the ifo.

        If only one ifo in the ifo list this will be that ifo. Otherwise an
        error is raised.
        """
        if self.ifo_list and len(self.ifo_list) == 1:
            return self.ifo_list[0]
        else:
            errMsg = "self.ifoList must contain only one ifo to access the "
            errMsg += "ifo property. %s." %(str(self.ifo_list),)
            raise TypeError(errMsg)

    def add_ini_profile(self, cp, sec):
        """Add profile from configuration file.

        sec : string
            The section containing options for this job.
        """
            namespace = opt.split('|')[0]
            if namespace == 'pycbc' or namespace == 'container':
                continue

            value = cp.get(sec, opt).strip()
            key = opt.split('|')[1]
            self.add_profile(namespace, key, value, force=True)

            # Remove if Pegasus can apply this hint in the TC
            if namespace == 'hints' and key == 'execution.site':
                self.execution_site = value

    def add_ini_opts(self, cp, sec):
        """Add job-specific options from configuration file.

        """
            opt = '--%s' %(opt,)
            if opt in self.file_input_options:
                # This now expects the option to be a file
                # Check is we have a list of files
                values = [path for path in value.split(' ') if path]

                self.common_raw_options.append(opt)
                self.common_raw_options.append(' ')

                # Get LFN and PFN
                for path in values:
                    # Here I decide if the path is URL or
                    # IFO:/path/to/file or IFO:url://path/to/file
                    # That's somewhat tricksy as we used : as delimiter
                    split_path = path.split(':', 1)
                    if len(split_path) == 1:
                        ifo = None
                        path = path
                    else:
                        # Have I split a URL or not?
                        if split_path[1].startswith('//'):
                            # URL
                            ifo = None
                            path = path
                        else:
                            #IFO:path or IFO:URL
                            ifo = split_path[0]
                            path = split_path[1]

                    # If the file exists make sure to use the
                    # fill path as a file:// URL
                    if os.path.isfile(path):
                        curr_pfn = urljoin('file:',
                                           pathname2url(os.path.abspath(path)))
                    else:
                        curr_pfn = path

                    curr_file = resolve_url_to_file(curr_pfn)
                    self.common_input_files.append(curr_file)
                    if ifo:
                        self.common_raw_options.append(ifo + ':')
                        self.common_raw_options.append(curr_file.dax_repr)
                    else:
                    self.common_raw_options.append(' ')
            elif opt in self.time_dependent_options:
                # There is a possibility of time-dependent, file options.
                # For now we will avoid supporting that complication unless
                # it is needed. This would require resolving the file first
                # in this function, and then dealing with the time-dependent
                # stuff later.
                self.unresolved_td_options[opt] = value
            else:
                self.common_options += [opt, value]

    def add_opt(self, opt, value=None):
        """Add option to job.

        opt : string
            Name of option (e.g. --output-file-format)
        value : string, (default=None)
            The value for the option (no value if set to None).
        """
        if value is None:
            self.common_options += [opt]
        else:
            self.common_options += [opt, value]

    def get_opt(self, opt):
        """Get value of option from configuration file

            Name of option (e.g. output-file-format)

        Returns
        --------
        value : string
            The value for the option. Returns None if option not present.
        """
        for sec in self.sections:
            try:
                key = self.cp.get(sec, opt)
                if key:
                    return key
            except ConfigParser.NoOptionError:
                pass

    def has_opt(self, opt):
        """Check if option is present in configuration file

        """
            val = self.cp.has_option(sec, opt)
            if val:
                return val

    def create_node(self, **kwargs):
        """Default node constructor.

        This is usually overridden by subclasses of Executable.
        """
        return Node(self, **kwargs)

    def update_current_retention_level(self, value):
        """Set a new value for the current retention level.

        This updates the value of self.retain_files for an updated value of the
        retention level.

        value : int
            The new value to use for the retention level.
        """
        self.current_retention_level = value
        try:
            global_retention_level = \
                self.cp.get_opt_tags("workflow", "file-retention-level",
                                   self.tags+[self.name])
            msg="Cannot find file-retention-level in [workflow] section "
            msg+="of the configuration file. Setting a default value of "
            msg+="retain all files."
            logging.warn(msg)
            self.retain_files = True
            self.global_retention_threshold = 1
            self.cp.set("workflow", "file-retention-level", "all_files")
        else:
            # FIXME: Are these names suitably descriptive?
            retention_choices = {
                                 'all_files' : 1,
                                 'all_triggers' : 2,
                                 'merged_triggers' : 3,
                                 'results' : 4
                                }
            try:
                self.global_retention_threshold = \
                      retention_choices[global_retention_level]
                err_msg = "Cannot recognize the file-retention-level in the "
                err_msg += "[workflow] section of the ini file. "
                err_msg += "Got : {0}.".format(global_retention_level)
                err_msg += "Valid options are: 'all_files', 'all_triggers',"
                err_msg += "'merged_triggers' or 'results' "
            if self.current_retention_level == 5:
                self.retain_files = True
                if type(self).__name__ in Executable._warned_classes_list:
                    pass
                else:
                    warn_msg = "Attribute current_retention_level has not "
                    warn_msg += "been set in class {0}. ".format(type(self))
                    warn_msg += "This value should be set explicitly. "
                    warn_msg += "All output from this class will be stored."
                    logging.warn(warn_msg)
                    Executable._warned_classes_list.append(type(self).__name__)
            elif self.global_retention_threshold > self.current_retention_level:
                self.retain_files = False
            else:

    def update_current_tags(self, tags):
        """Set a new set of tags for this executable.

        Update the set of tags that this job will use. This updated default
        file naming and shared options. It will *not* update the pegasus
        profile, which belong to the executable and cannot be different for
        different nodes.

        tags : list
            The new list of tags to consider.
        """
            tags = []
        if '' in tags:
            logging.warn('DO NOT GIVE ME EMPTY TAGS')
            tags.remove('')
        tags = [tag.upper() for tag in tags]
        self.tags = tags

        if len(tags) > 6:
            warn_msg = "This job has way too many tags. "
            warn_msg += "Current tags are {}. ".format(' '.join(tags))
            warn_msg += "Current executable {}.".format(self.name)
            logging.info(warn_msg)

        if len(tags) != 0:
            self.tagged_name = "{0}-{1}".format(self.name, '_'.join(tags))
        else:
            self.tagged_name = self.name
        if self.ifo_string is not None:
            self.tagged_name = "{0}-{1}".format(self.tagged_name,
                                                self.ifo_string)

        # Determine the sections from the ini file that will configure
        # this executable
        sections = [self.name]
            if len(self.ifo_list) > 1:
                sec_tags = tags + self.ifo_list + [self.ifo_string]
            else:
                sec_tags = tags + self.ifo_list
        else:
            sec_tags = tags
        for sec_len in range(1, len(sec_tags)+1):
            for tag_permutation in permutations(sec_tags, sec_len):
                joined_name = '-'.join(tag_permutation)
                section = '{0}-{1}'.format(self.name, joined_name.lower())
                if self.cp.has_section(section):
                    sections.append(section)

        self.sections = sections

        # Do some basic sanity checking on the options
        for sec1, sec2 in combinations(sections, 2):
            self.cp.check_duplicate_options(sec1, sec2, raise_error=True)

        # collect the options and profile information
        # from the ini file section(s)
        self.common_options = []
        self.common_raw_options = []
        self.unresolved_td_options = {}
        self.common_input_files = []
        for sec in sections:
            if self.cp.has_section(sec):
                self.add_ini_opts(self.cp, sec)
            else:
                warn_string = "warning: config file is missing section "
                warn_string += "[{0}]".format(sec)
                logging.warn(warn_string)

    def update_output_directory(self, out_dir=None):
        """Update the default output directory for output files.

        out_dir : string (optional, default=None)
            If provided use this as the output directory. Else choose this
            automatically from the tags.
        """
        # Determine the output directory
        if out_dir is not None:
            self.out_dir = out_dir
        elif len(self.tags) == 0:
            self.out_dir = self.name
        else:
            self.out_dir = self.tagged_name
        if not os.path.isabs(self.out_dir):
            self.out_dir = os.path.join(os.getcwd(), self.out_dir)

    def _set_pegasus_profile_options(self):
        """Set the pegasus-profile settings for this Executable.

        These are a property of the Executable and not of nodes that it will
        spawn. Therefore it *cannot* be updated without also changing values
        for nodes that might already have been created. Therefore this is
        only called once in __init__. Second calls to this will fail.
        """
        # Add executable non-specific profile information
        if self.cp.has_section('pegasus_profile'):
            self.add_ini_profile(self.cp, 'pegasus_profile')

        # Executable- and tag-specific profile information
            if self.cp.has_section('pegasus_profile-{0}'.format(sec)):
                self.add_ini_profile(self.cp,
                                     'pegasus_profile-{0}'.format(sec))

class Workflow(pegasus_workflow.Workflow):
    """
    This class manages a pycbc workflow. It provides convenience
    functions for finding input files using time and keywords. It can also
    generate cache files from the inputs.
    """
    def __init__(self, args, name):
        """
        Create a pycbc workflow

        args : argparse.ArgumentParser
            The command line options to initialize a CBC workflow.
        """
        super(Workflow, self).__init__(name)

        # Parse ini file
        self.cp = WorkflowConfigParser.from_cli(args)

        # Set global values
        start_time = end_time = 0
        if self.cp.has_option('workflow', 'start-time'):
            start_time = int(self.cp.get("workflow", "start-time"))

        if self.cp.has_option('workflow', 'end-time'):
            end_time = int(self.cp.get("workflow", "end-time"))
        self.analysis_time = segments.segment([start_time, end_time])

        # Set the ifos to analyse
        ifos = []
        if self.cp.has_section('workflow-ifos'):
            for ifo in self.cp.options('workflow-ifos'):
                ifos.append(ifo.upper())

        self.ifos.sort(key=str.lower)
        self.ifo_string = ''.join(self.ifos)

        # Set up input and output file lists for workflow
        self._inputs = FileList([])
        self._outputs = FileList([])

    @property
    def output_map(self):
        if self.in_workflow is not False:
            name = self.name + '.map'
        else:
            name = 'output.map'
        path =  os.path.join(os.getcwd(), name)

    @property
    def transformation_catalog(self):
            name = self.name + '.tc.txt'
        else:
            name = 'tc.txt'

    @property
    def staging_site(self):
            workflow_section = 'workflow-%s' % self.name
        else:
            workflow_section = 'workflow'
        try:
            staging_site = self.cp.get(workflow_section,'staging-site')
        except:
            staging_site = None
        return staging_site

    def execute_node(self, node, verbatim_exe = False):
        """ Execute this node immediately on the local machine
        """
        node.executed = True

        # Check that the PFN is for a file or path
        if node.executable.needs_fetching:
            try:
                # The pfn may have been marked local...
                pfn = node.executable.get_pfn()
            except:
                # or it may have been marked nonlocal.  That's
                # fine, we'll resolve the URL and make a local
                # entry.
                pfn = node.executable.get_pfn('nonlocal')

            resolved = resolve_url(pfn, permissions=stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
            node.executable.clear_pfns()
            node.executable.add_pfn(urljoin('file:', pathname2url(resolved)),
                                    site='local')

        cmd_list = node.get_command_line()

        # Must execute in output directory.
        curr_dir = os.getcwd()
        out_dir = node.executable.out_dir
        os.chdir(out_dir)

        # Make call
        make_external_call(cmd_list, out_dir=os.path.join(out_dir, 'logs'),
                                     out_basename=node.executable.name)
        # Change back
        os.chdir(curr_dir)

        for fil in node._outputs:
            fil.node = None
            fil.PFN(urljoin('file:', pathname2url(fil.storage_path)),
                    site='local')

    def set_job_properties(job, output_map_file, transformation_catalog_file,
                           staging_site=None):

        job.addArguments('-Dpegasus.dir.storage.mapper.replica.file=%s' %
                         os.path.basename(output_map_file.name))
        job.uses(output_map_file, link=Pegasus.DAX3.Link.INPUT)
        job.addArguments('-Dpegasus.dir.storage.mapper.replica=File')

        # FIXME this is an ugly hack to connect the right transformation
        # catalog to the right DAX beacuse Pegasus 4.9 does not support
        # the full transformation catalog syntax in the DAX. This will go
        # away in Pegasus 5.x when this code is re-written.

        job.addArguments('-Dpegasus.catalog.transformation.file=%s' %
                         os.path.basename(transformation_catalog_file.name))
        job.uses(transformation_catalog_file, link=Pegasus.DAX3.Link.INPUT)

        job.addArguments('--output-site local')
        job.addArguments('--cleanup inplace')
        job.addArguments('--cluster label,horizontal')
        job.addArguments('-vvv')

        # FIXME _reuse_cache needs to be fixed to use PFNs properly. This will
        # work as pegasus-plan is currently invoked on the local site so has
        # access to a file in os.getcwd() but this code is fragile.
        job.addArguments('--cache %s' % os.path.join(os.getcwd(), '_reuse.cache'))

        if staging_site:
            job.addArguments('--staging-site %s' % staging_site)

    def save(self, filename=None, output_map_path=None,
             transformation_catalog_path=None, staging_site=None):

        if output_map_path is None:
            output_map_path = self.output_map
        output_map_file = Pegasus.DAX3.File(os.path.basename(output_map_path))
        output_map_file.addPFN(Pegasus.DAX3.PFN(output_map_path, 'local'))
            self.in_workflow._adag.addFile(output_map_file)

        if transformation_catalog_path is None:
            transformation_catalog_path = self.transformation_catalog
        transformation_catalog_file = Pegasus.DAX3.File(os.path.basename(
                                                        transformation_catalog_path))
        transformation_catalog_file.addPFN(Pegasus.DAX3.PFN(
            transformation_catalog_path, 'local'))
            self.in_workflow._adag.addFile(transformation_catalog_file)

        if staging_site is None:
            staging_site = self.staging_site

        Workflow.set_job_properties(self.as_job, output_map_file,
                                    transformation_catalog_file,
                                    staging_site)

        # add executable pfns for local site to dax
        for exe in self._executables:
            exe.insert_into_dax(self._adag)

        # add workflow input files pfns for local site to dax
        for fil in self._inputs:
            fil.insert_into_dax(self._adag)

        # save the configuration file
        ini_file = os.path.abspath(self.name + '.ini')

        # This shouldn't already exist, but just in case
        if os.path.isfile(ini_file):
            err_msg = "Refusing to overwrite configuration file that "
            err_msg += "shouldn't be there: "
            err_msg += os.path.join(os.getcwd(), ini_file)

        fp = open(ini_file, 'w')
        self.cp.write(fp)

        # save the dax file
        super(Workflow, self).save(filename=filename,
                                   tc=transformation_catalog_path)

        # add workflow storage locations to the output mapper
        f = open(output_map_path, 'w')
        for out in self._outputs:
            try:
                f.write(out.output_map_str() + '\n')
                # There was no storage path
                pass

    def save_config(self, fname, output_dir, cp=None):
        """ Writes configuration file to disk and returns a pycbc.workflow.File
        instance for the configuration file.

        fname : string
            The filename of the configuration file written to disk.
        output_dir : string
            The directory where the file is written to disk.
            The ConfigParser object to write. If None then uses self.cp.

        Returns
        -------
        FileList
            The FileList object with the configuration file.
        """
        cp = self.cp if cp is None else cp
        ini_file_path = os.path.abspath(os.path.join(output_dir, fname))
        with open(ini_file_path, "w") as fp:
            cp.write(fp)
        ini_file = File(self.ifos, "", self.analysis_time,
                        file_url="file://" + ini_file_path)
        # set the physical file name
        ini_file.PFN(ini_file_path, "local")
        # set the storage path to be the same
        ini_file.storage_path = ini_file_path
        return FileList([ini_file])

class Node(pegasus_workflow.Node):
    def __init__(self, executable, valid_seg=None):
        super(Node, self).__init__(executable)
        self.executed = False
        self.set_category(executable.name)
        self.valid_seg = valid_seg

        if executable.universe == 'vanilla' and executable.installed:
            self.add_profile('condor', 'getenv', 'True')

        if hasattr(executable, 'execution_site'):
            self.add_profile('hints', 'execution.site', executable.execution_site)

        self._options += self.executable.common_options
        self._raw_options += self.executable.common_raw_options
        for inp in self.executable.common_input_files:
            self._add_input(inp)

        if len(self.executable.time_dependent_options):
            # Resolving these options requires the concept of a valid time.
            # To keep backwards compatibility we will allow this to work if
            # valid_seg is not supplied and no option actually needs resolving.
            # It would be good to get this from the workflow's valid_seg if
            # not overriden. But the Node is not connected to the Workflow
            # until the dax starts to be written.
            self.resolve_td_options(self.executable.unresolved_td_options)

    def get_command_line(self):
        self._finalize()
        arglist = self._dax_node.arguments

        tmpargs = []
        for a in arglist:
            if not isinstance(a, File):
                tmpargs += a.split(' ')
            else:
                tmpargs.append(a)
        arglist = tmpargs

        arglist = [a for a in arglist if a != '']

        arglist = [a.storage_path if isinstance(a, File) else a for a in arglist]

        # This allows the pfn to be an http(s) URL, which will be
        # downloaded by resolve_url
        exe_path = urllib.parse.urlsplit(self.executable.get_pfn()).path

        return [exe_path] + arglist

    def new_output_file_opt(self, valid_seg, extension, option_name, tags=None,
                            store_file=None, use_tmp_subdirs=False):
        """
        This function will create a workflow.File object corresponding to the given
        information and then add that file as output of this node.

        valid_seg : ligo.segments.segment
            The time span over which the job is valid for.
        extension : string
            The extension to be used at the end of the filename.
            E.g. '.xml' or '.sqlite'.
        option_name : string
            The option that is used when setting this job as output. For e.g.
            'output-name' or 'output-file', whatever is appropriate for the
            current executable.
        tags : list of strings, (optional, default=[])
            These tags will be added to the list of tags already associated with
            the job. They can be used to uniquely identify this output file.
        store_file : Boolean, (optional, default=True)
            This file is to be added to the output mapper and will be stored
            in the specified output location if True. If false file will be
            removed when no longer needed in the workflow.
        """
            tags = []

        # Changing this from set(tags) to enforce order. It might make sense
        # for all jobs to have file names with tags in the same order.
        all_tags = copy.deepcopy(self.executable.tags)
        for tag in tags:
            if tag not in all_tags:
                all_tags.append(tag)

        store_file = store_file if store_file is not None else self.executable.retain_files

        fil = File(self.executable.ifo_list, self.executable.name,
                   valid_seg, extension=extension, store_file=store_file,
                   directory=self.executable.out_dir, tags=all_tags,
                   use_tmp_subdirs=use_tmp_subdirs)
        self.add_output_opt(option_name, fil)
        return fil

    def add_multiifo_input_list_opt(self, opt, inputs):
        """ Add an option that determines a list of inputs from multiple
            detectors. Files will be supplied as --opt ifo1:input1 ifo2:input2
            .....
        """
        # NOTE: Here we have to use the raw arguments functionality as the
        #       file and ifo are not space separated.
        self.add_raw_arg(opt)
        self.add_raw_arg(' ')
        for infile in inputs:
            self.add_raw_arg(infile.ifo)
            self.add_raw_arg(':')
            self.add_raw_arg(infile.name)
            self.add_raw_arg(' ')
            self._add_input(infile)

    def add_multiifo_output_list_opt(self, opt, outputs):
        """ Add an option that determines a list of outputs from multiple
            .....
        """
        for outfile in outputs:
            self.add_raw_arg(outfile.ifo)
            self.add_raw_arg(outfile.name)
            self._add_output(outfile)

    def new_multiifo_output_list_opt(self, opt, ifos, analysis_time, extension,
                                     tags=None, store_file=None,
                                     use_tmp_subdirs=False):
            .....
            File names are created internally from the provided extension and
            analysis time.
        """
            tags = []

        output_files = FileList([])
        store_file = store_file if store_file is not None \
                                              else self.executable.retain_files

            curr_file = File(ifo, self.executable.name, analysis_time,
                             extension=extension, store_file=store_file,
                             directory=self.executable.out_dir, tags=all_tags,
                             use_tmp_subdirs=use_tmp_subdirs)
            output_files.append(curr_file)
        self.add_multiifo_output_list_opt(opt, output_files)

    def resolve_td_options(self, td_options):
        for opt in td_options:
            new_opt = resolve_td_option(td_options[opt], self.valid_seg)
            self._options += [opt, new_opt]

    @property
    def output_files(self):
        return FileList(self._outputs)

    @property
    def output_file(self):
        """
        If only one output file return it. Otherwise raise an exception.
        """
        out_files = self.output_files
        if len(out_files) != 1:
            err_msg = "output_file property is only valid if there is a single"
            err_msg += " output file. Here there are "
            err_msg += "%d output files." %(len(out_files))
        return out_files[0]

class File(pegasus_workflow.File):
    '''
    This class holds the details of an individual output file
    This file(s) may be pre-supplied, generated from within the workflow
    command line script, or generated within the workflow. The important stuff
    is:

    * The ifo that the File is valid for
    * The time span that the OutFile is valid for
    * A short description of what the file is
    * The extension that the file should have
    * The url where the file should be located

    An example of initiating this class:

    >> c = File("H1", "INSPIRAL_S6LOWMASS", segments.segment(815901601, 815902001), file_url="file://localhost/home/spxiwh/H1-INSPIRAL_S6LOWMASS-815901601-400.xml.gz" )

    another where the file url is generated from the inputs:

    >> c = File("H1", "INSPIRAL_S6LOWMASS", segments.segment(815901601, 815902001), directory="/home/spxiwh", extension="xml.gz" )

    '''
    def __init__(self, ifos, exe_name, segs, file_url=None,
                 extension=None, directory=None, tags=None,
                 store_file=True, use_tmp_subdirs=False):
        """
        Create a File instance

        ifos : string or list
            The ifo(s) that the File is valid for. If the file is
            Ie. ['H1',L1','V1'], if the file is only valid for the combination
            of ifos (for e.g. ligolw_thinca output) then this can be supplied
        exe_name: string
            A short description of the executable description, tagging
            only the program that ran this job.
        segs : glue.segment or glue.segmentlist
            The time span that the OutFile is valid for. Note that this is
            *not* the same as the data that the job that made the file reads in.
            Lalapps_inspiral jobs do not analyse the first an last 72s of the
            data that is read, and are therefore not valid at those times. If
            the time is not continuous a segmentlist can be supplied.
        file_url : url (optional, default=None)
            If this is *not* supplied, extension and directory must be given.
            If specified this explicitly points to the url of the file, or the
            url where the file will be generated when made in the workflow.
        extension : string (optional, default=None)
            Either supply this *and* directory *or* supply only file_url.
            If given this gives the extension at the end of the file name. The
            full file name will be inferred from the other arguments
            following the workflow standard.
        directory : string (optional, default=None)
            Either supply this *and* extension *or* supply only file_url.
            If given this gives the directory in which the file exists, or will
            exists. The file name will be inferred from the other arguments
        tags : list of strings (optional, default=None)
            This is a list of descriptors describing what this file is. For
            e.g. this might be ["BNSINJECTIONS" ,"LOWMASS","CAT_2_VETO"].
            These are used in file naming.
        """

        # Set the science metadata on the file
        else:
        self.ifo_string = ''.join(self.ifo_list)
        self.description = exe_name

        if isinstance(segs, segments.segment):
            self.segment_list = segments.segmentlist([segs])
        elif isinstance(segs, (segments.segmentlist)):
            self.segment_list = segs
        else:
            err = "segs input must be either ligo.segments.segment or "
            err += "segments.segmentlist. Got %s." %(str(type(segs)),)
            raise ValueError(err)
            tags = []

        if len(self.tags):
            self.tag_str = '_'.join(tags)
            tagged_description = '_'.join([self.description] + tags)
        else:
            tagged_description = self.description

        # Follow the capitals-for-naming convention
        self.ifo_string = self.ifo_string.upper()
        self.tagged_description = tagged_description.upper()

        if not file_url:
            if not extension:
                raise TypeError("a file extension required if a file_url "
                                "is not provided")
            if not directory:
                raise TypeError("a directory is required if a file_url is "
                                "not provided")

            filename = self._filename(self.ifo_string, self.tagged_description,
                                      extension, self.segment_list.extent())
            path = os.path.join(directory, filename)
            if not os.path.isabs(path):
                path = os.path.join(os.getcwd(), path)
            file_url = urllib.parse.urlunparse(['file', 'localhost', path,
                                                None, None, None])

        # Let's do a test here
        if use_tmp_subdirs and len(self.segment_list):
            pegasus_lfn = str(int(self.segment_list.extent()[0]))[:-4]
            pegasus_lfn = pegasus_lfn + '/' + os.path.basename(file_url)
        else:
            pegasus_lfn = os.path.basename(file_url)
        super(File, self).__init__(pegasus_lfn)

        if store_file:
            self.storage_path = urllib.parse.urlsplit(file_url).path
        else:
            self.storage_path = None

    def __getstate__(self):
        """ Allow the ahope file to be picklable. This disables the usage of
        the internal cache entry.
        """
        for i, seg in enumerate(self.segment_list):
            self.segment_list[i] = segments.segment(float(seg[0]), float(seg[1]))
        self.cache_entry = None
        safe_dict = copy.copy(self.__dict__)
        safe_dict['cache_entry'] = None
        return safe_dict

    def add_metadata(self, key, value):
        """ Add arbitrary metadata to this file """
        self.metadata[key] = value

    @property
        """
        If only one ifo in the ifo_list this will be that ifo. Otherwise an
        """
        if len(self.ifo_list) == 1:
        else:
            err = "self.ifo_list must contain only one ifo to access the "
            err += "ifo property. %s." %(str(self.ifo_list),)
            raise TypeError(err)

    @property
    def segment(self):
        """
        If only one segment in the segmentlist this will be that segment.
        Otherwise an error is raised.
        """
        if len(self.segment_list) == 1:
            return self.segment_list[0]
        else:
            err = "self.segment_list must only contain one segment to access"
            err += " the segment property. %s." %(str(self.segment_list),)

    @property
    def cache_entry(self):
        """
        Returns a CacheEntry instance for File.
        """
        if self.storage_path is None:
            raise ValueError('This file is temporary and so a lal '
                             'cache entry cannot be made')

        file_url = urllib.parse.urlunparse(['file', 'localhost',
                                            self.storage_path, None,
                                            None, None])
        cache_entry = lal.utils.CacheEntry(self.ifo_string,
                   self.tagged_description, self.segment_list.extent(), file_url)
        cache_entry.workflow_file = self
        return cache_entry

    def _filename(self, ifo, description, extension, segment):
        """
        Construct the standard output filename. Should only be used internally
        of the File class.
        """
        if extension.startswith('.'):
            extension = extension[1:]

        # Follow the frame convention of using integer filenames,
        # but stretching to cover partially covered seconds.
        start = int(segment[0])
        end = int(math.ceil(segment[1]))
        duration = str(end-start)
        start = str(start)

        return "%s-%s-%s-%s.%s" % (ifo, description.upper(), start,
                                   duration, extension)

class FileList(list):
    '''
    This class holds a list of File objects. It inherits from the
    built-in list class, but also allows a number of features. ONLY
    pycbc.workflow.File instances should be within a FileList instance.
    '''
    entry_class = File

    def categorize_by_attr(self, attribute):
        '''
        Function to categorize a FileList by a File object
        attribute (eg. 'segment', 'ifo', 'description').

        attribute : string
           File object attribute to categorize FileList

        Returns
        --------
        keys : list
           A list of values for an attribute
        groups : list
           A list of FileLists
        '''

        # need to sort FileList otherwise using groupby without sorting does
        # 'AAABBBCCDDAABB' -> ['AAA','BBB','CC','DD','AA','BB']
        # and using groupby with sorting does
        # 'AAABBBCCDDAABB' -> ['AAAAA','BBBBB','CC','DD']
        flist = sorted(self, key=attrgetter(attribute), reverse=True)

        # use groupby to create lists
        groups = []
        keys = []
        for k, g in groupby(flist, attrgetter(attribute)):
            groups.append(FileList(g))
            keys.append(k)

        return keys, groups

    def find_output(self, ifo, time):
        '''Returns one File most appropriate at the given time/time range.

        Return one File that covers the given time, or is most
        appropriate for the supplied time range.

           Name of the ifo (or ifos) that the file should be valid for.
        time : int/float/LIGOGPStime or tuple containing two values
           If int/float/LIGOGPStime (or similar may of specifying one time) is
           given, return the File corresponding to the time. This calls
           self.find_output_at_time(ifo,time).
           If a tuple of two values is given, return the File that is
           **most appropriate** for the time range given. This calls
           self.find_output_in_range

        Returns
        --------
        pycbc_file : pycbc.workflow.File instance
           The File that corresponds to the time or time range
        '''
        # Determine whether I have a specific time, or a range of times
        try:
            lenTime = len(time)
            # This is if I have a single time
            outFile = self.find_output_at_time(ifo,time)
        else:
            # This is if I have a range of times
            if lenTime == 2:
                outFile = self.find_output_in_range(ifo,time[0],time[1])
            # This is if I got a list that had more (or less) than 2 entries
            if len(time) != 2:
                raise TypeError("I do not understand the input variable time")
        return outFile

    def find_output_at_time(self, ifo, time):
        '''
        Return File that covers the given time.

           Name of the ifo (or ifos) that the File should correspond to
        time : int/float/LIGOGPStime
           Return the Files that covers the supplied time. If no
           File covers the time this will return None.

        Returns
        --------
        list of File classes
           The Files that corresponds to the time.
         '''
        # Get list of Files that overlap time, for given ifo
        outFiles = [i for i in self if ifo in i.ifo_list and time in i.segment_list]
        if len(outFiles) == 0:
            # No OutFile at this time
        elif len(outFiles) == 1:
            # 1 OutFile at this time (good!)
            return outFiles
        else:
            # Multiple output files. Currently this is valid, but we may want
            # to demand exclusivity later, or in certain cases. Hence the
            # separation.

    def find_outputs_in_range(self, ifo, current_segment, useSplitLists=False):
        """
        Return the list of Files that is most appropriate for the supplied
        time range. That is, the Files whose coverage time has the
        largest overlap with the supplied time range.

        current_segment : glue.segment.segment
           The segment of time that files must intersect.

        Returns
        --------
        FileList class
           The list of Files that are most appropriate for the time range
        """
        currsegment_list = segments.segmentlist([current_segment])

        # Get all files overlapping the window
        overlap_files = self.find_all_output_in_range(ifo, current_segment,
                                                    useSplitLists=useSplitLists)

        # By how much do they overlap?
        overlap_windows = [abs(i.segment_list & currsegment_list) for i in overlap_files]

        if not overlap_windows:
            return []

        # Return the File with the biggest overlap
        # Note if two File have identical overlap, the first is used
        # to define the valid segment
        overlap_windows = numpy.array(overlap_windows, dtype = int)
        segmentLst = overlap_files[overlap_windows.argmax()].segment_list

        # Get all output files with the exact same segment definition
        output_files = [f for f in overlap_files if f.segment_list==segmentLst]
        return output_files

    def find_output_in_range(self, ifo, start, end):
        '''
        Return the File that is most appropriate for the supplied
        time range. That is, the File whose coverage time has the
        largest overlap with the supplied time range. If no Files
        overlap the supplied time window, will return None.

        start : int/float/LIGOGPStime
           The start of the time range of interest.
        end : int/float/LIGOGPStime
           The end of the time range of interest

        Returns
        --------
        File class
           The File that is most appropriate for the time range
        '''
        currsegment_list = segments.segmentlist([segments.segment(start, end)])

        # First filter Files corresponding to ifo
        outFiles = [i for i in self if ifo in i.ifo_list]

            # No OutFiles correspond to that ifo
        # Filter OutFiles to those overlapping the given window
        currSeg = segments.segment([start,end])
        outFiles = [i for i in outFiles \
                                  if i.segment_list.intersects_segment(currSeg)]

            # No OutFile overlap that time period
            # One OutFile overlaps that period
            return outFiles[0]
        else:
            overlap_windows = [abs(i.segment_list & currsegment_list) \
                                                        for i in outFiles]
            # Return the File with the biggest overlap
            # Note if two File have identical overlap, this will return
            # the first File in the list
            overlap_windows = numpy.array(overlap_windows, dtype = int)
            return outFiles[overlap_windows.argmax()]

    def find_all_output_in_range(self, ifo, currSeg, useSplitLists=False):
        """
        Return all files that overlap the specified segment.
        """
        if not useSplitLists:
            # Slower, but simpler method
            outFiles = [i for i in self if ifo in i.ifo_list]
            outFiles = [i for i in outFiles \
                                      if i.segment_list.intersects_segment(currSeg)]
        else:
            # Faster, but more complicated
            # Basically only check if a subset of files intersects_segment by
            # using a presorted list. Sorting only happens once.
            if not self._check_split_list_validity():
                # FIXME: DO NOT hard code this.
                self._temporal_split_list(100)
            startIdx = int( (currSeg[0] - self._splitListsStart) / \
                                                          self._splitListsStep )
            # Add some small rounding here
            endIdx = (currSeg[1] - self._splitListsStart) / self._splitListsStep
            endIdx = int(endIdx - 0.000001)

            outFiles = []
            for idx in range(startIdx, endIdx + 1):
                if idx < 0 or idx >= self._splitListsNum:
                    continue
                outFilesTemp = [i for i in self._splitLists[idx] \
                                                            if ifo in i.ifo_list]
                outFiles.extend([i for i in outFilesTemp \
                                      if i.segment_list.intersects_segment(currSeg)])
                # Remove duplicates
                outFiles = list(set(outFiles))

        return self.__class__(outFiles)

    def find_output_with_tag(self, tag):
        """
        Find all files who have tag in self.tags
        """
        tag = tag.upper()
        return FileList([i for i in self if tag in i.tags])

    def find_output_without_tag(self, tag):
        """
        Find all files who do not have tag in self.tags
        """
        return FileList([i for i in self if tag not in i.tags])

    def find_output_with_ifo(self, ifo):
        """
        Find all files who have ifo = ifo
        """
        ifo = ifo.upper()
        return FileList([i for i in self if ifo in i.ifo_list])

    def get_times_covered_by_files(self):
        """
        Find the coalesced intersection of the segments of all files in the
        list.
        """
        times = segments.segmentlist([])
        for entry in self:
            times.extend(entry.segment_list)
        times.coalesce()
        return times

    def convert_to_lal_cache(self):
        """
        Return all files in this object as a glue.lal.Cache object
        """
        lal_cache = gluelal.Cache([])
            try:
                lal_cache.append(entry.cache_entry)
                pass
        return lal_cache

    def _temporal_split_list(self,numSubLists):
        """
        This internal function is used to speed the code up in cases where a
        number of operations are being made to determine if files overlap a
        specific time. Normally such operations are done on *all* entries with
        *every* call. However, if we predetermine which files are at which
        times, we can avoid testing *every* file every time.

        We therefore create numSubLists distinct and equal length time windows
        equally spaced from the first time entry in the list until the last.
        A list is made for each window and files are added to lists which they
        overlap.

        If the list changes it should be captured and these split lists become
        invalid. Currently the testing for this is pretty basic
        """
        # Assume segment lists are coalesced!
        startTime = float( min([i.segment_list[0][0] for i in self]))
        endTime = float( max([i.segment_list[-1][-1] for i in self]))
        step = (endTime - startTime) / float(numSubLists)

        # Set up storage
        self._splitLists = []
        for idx in range(numSubLists):
            self._splitLists.append(FileList([]))

        # Sort the files

        for currFile in self:
            segExtent = currFile.segment_list.extent()
            startIdx = (segExtent[0] - startTime) / step
            endIdx = (segExtent[1] - startTime) / step
            startIdx = int(startIdx - 0.001)
            endIdx = int(endIdx + 0.001)

            if startIdx < 0:
                startIdx = 0
            if endIdx >= numSubLists:
                endIdx = numSubLists - 1

                self._splitLists[idx].append(currFile)

        # Set information needed to detect changes and to be used elsewhere
        self._splitListsLength = len(self)
        self._splitListsNum = numSubLists
        self._splitListsStart = startTime
        self._splitListsEnd = endTime
        self._splitListsStep = step
        self._splitListsSet = True

    def _check_split_list_validity(self):
        """
        See _temporal_split_list above. This function checks if the current
        split lists are still valid.
        """
        # FIXME: Currently very primitive, but needs to be fast
        if not (hasattr(self,"_splitListsSet") and (self._splitListsSet)):
        elif len(self) != self._splitListsLength:
        else:

    def load(cls, filename):
        """
        Load an AhopeFileList from a pickle file
        """
        f = open(filename, 'r')
        return cPickle.load(f)

    def dump(self, filename):
        """
        Output this AhopeFileList to a pickle file
        """
        f = open(filename, 'w')
        cPickle.dump(self, f)

    def to_file_object(self, name, out_dir):
        """Dump to a pickle file and return an File object reference of this list

        name : str
            An identifier of this file. Needs to be unique.
        out_dir : path
            path to place this file

        Returns
        -------
        file : AhopeFile
        """
        make_analysis_dir(out_dir)

        file_ref = File('ALL', name, self.get_times_covered_by_files(),
                             extension='.pkl', directory=out_dir)
        self.dump(file_ref.storage_path)
        return file_ref

class SegFile(File):
    '''
    This class inherits from the File class, and is designed to store
    workflow output files containing a segment dict. This is identical in
    usage to File except for an additional kwarg for holding the
    segment dictionary, if it is known at workflow run time.
    '''
    def __init__(self, ifo_list, description, valid_segment,
                 segment_dict=None, seg_summ_dict=None, **kwargs):
        """
        See File.__init__ for a full set of documentation for how to
        call this class. The only thing unique and added to this class is
        the optional segment_dict. NOTE that while segment_dict is a
        ligo.segments.segmentlistdict rather than the usual dict[ifo]
        we key by dict[ifo:name].

        ------------
        ifo_list : string or list (required)
            See File.__init__
        description : string (required)
        segment : ligo.segments.segment or ligo.segments.segmentlist
        segment_dict : ligo.segments.segmentlistdict (optional, default=None)
            A ligo.segments.segmentlistdict covering the times covered by the
            segmentlistdict associated with this file.
            Can be added by setting self.segment_dict after initializing an
            instance of the class.

        """
        super(SegFile, self).__init__(ifo_list, description, valid_segment,
                                      **kwargs)
        # To avoid confusion with the segment_list property of the parent class
        # we refer to this as valid_segments here
        self.valid_segments = self.segment_list
        self.segment_dict = segment_dict
        self.seg_summ_dict = seg_summ_dict

    def from_segment_list(cls, description, segmentlist, name, ifo,
                          seg_summ_list=None, **kwargs):
        """ Initialize a SegFile object from a segmentlist.

        segmentlist : ligo.segments.segmentslist
            The segment list that will be stored in this file.
        name : str
            The name of the segment lists to be stored in the file.
        ifo : str
            The ifo of the segment lists to be stored in this file.
        seg_summ_list : ligo.segments.segmentslist (OPTIONAL)
            Specify the segment_summary segmentlist that goes along with the
            segmentlist. Default=None, in this case segment_summary is taken
            from the valid_segment of the SegFile class.
        """
        seglistdict = segments.segmentlistdict()
        seglistdict[ifo + ':' + name] = segmentlist
        if seg_summ_list is not None:
            seg_summ_dict = segments.segmentlistdict()
            seg_summ_dict[ifo + ':' + name] = seg_summ_list
        else:
            seg_summ_dict = None
        return cls.from_segment_list_dict(description, seglistdict,
                                          seg_summ_dict=None, **kwargs)

    def from_multi_segment_list(cls, description, segmentlists, names, ifos,
                                seg_summ_lists=None, **kwargs):
        """ Initialize a SegFile object from a list of segmentlists.

        segmentlists : List of ligo.segments.segmentslist
            List of segment lists that will be stored in this file.
        names : List of str
            List of names of the segment lists to be stored in the file.
        ifos : str
            List of ifos of the segment lists to be stored in this file.
        seg_summ_lists : ligo.segments.segmentslist (OPTIONAL)
            Specify the segment_summary segmentlists that go along with the
            segmentlists. Default=None, in this case segment_summary is taken
        """
        for name, ifo, segmentlist in zip(names, ifos, segmentlists):
            seglistdict[ifo + ':' + name] = segmentlist
        if seg_summ_lists is not None:
            for name, ifo, seg_summ_list in zip(names, ifos, seg_summ_lists):
                seg_summ_dict[ifo + ':' + name] = seg_summ_list
        else:

                                         seg_summ_dict=seg_summ_dict, **kwargs)

    def from_segment_list_dict(cls, description, segmentlistdict,
                               ifo_list=None, valid_segment=None,
                               file_exists=False, seg_summ_dict=None,
                               **kwargs):
        """ Initialize a SegFile object from a segmentlistdict.

        segmentlistdict : ligo.segments.segmentslistdict
            See SegFile.__init__
        ifo_list : string or list (optional)
            See File.__init__, if not given a list of all ifos in the
            segmentlistdict object will be used
        valid_segment : ligo.segments.segment or ligo.segments.segmentlist
            See File.__init__, if not given the extent of all segments in the
            segmentlistdict is used.
        file_exists : boolean (default = False)
            If provided and set to True it is assumed that this file already
            exists on disk and so there is no need to write again.
        seg_summ_dict : ligo.segments.segmentslistdict
            Optional. See SegFile.__init__.
        """
        if ifo_list is None:
            ifo_set = set([i.split(':')[0] for i in segmentlistdict.keys()])
            ifo_list = list(ifo_set)
            ifo_list.sort()
        if valid_segment is None:
            if seg_summ_dict and \
                    numpy.any([len(v) for _, v in seg_summ_dict.items()]):
                # Only come here if seg_summ_dict is supplied and it is
                # not empty.
                valid_segment = seg_summ_dict.extent_all()
            else:
                try:
                    valid_segment = segmentlistdict.extent_all()
                except:
                    # Numpty probably didn't supply a glue.segmentlistdict
                    segmentlistdict=segments.segmentlistdict(segmentlistdict)
                    try:
                        valid_segment = segmentlistdict.extent_all()
                        # No segment_summary and segment list is empty
                        # Setting valid segment now is hard!
                        warn_msg = "No information with which to set valid "
                        warn_msg += "segment."
                        logging.warn(warn_msg)
                        valid_segment = segments.segment([0,1])
        instnc = cls(ifo_list, description, valid_segment,
                     segment_dict=segmentlistdict, seg_summ_dict=seg_summ_dict,
                     **kwargs)
        if not file_exists:
            instnc.to_segment_xml()
        else:
            instnc.PFN(urljoin('file:', pathname2url(instnc.storage_path)),
                       site='local')
        return instnc

    def from_segment_xml(cls, xml_file, **kwargs):
        """
        Read a ligo.segments.segmentlist from the file object file containing an
        xml segment table.

        xml_file : file object
            file object for segment xml file
        """
        # load xmldocument and SegmentDefTable and SegmentTables
        fp = open(xml_file, 'rb')
        xmldoc, _ = ligolw_utils.load_fileobj(fp,
                                              gz=xml_file.endswith(".gz"),
                                              contenthandler=ContentHandler)

        seg_def_table = table.get_table(xmldoc,
                                        lsctables.SegmentDefTable.tableName)
        seg_table = table.get_table(xmldoc, lsctables.SegmentTable.tableName)
        seg_sum_table = table.get_table(xmldoc,
                                        lsctables.SegmentSumTable.tableName)

        segs = segments.segmentlistdict()
        seg_summ = segments.segmentlistdict()

        seg_id = {}
        for seg_def in seg_def_table:
            # Here we want to encode ifo and segment name
            seg_id[int(seg_def.segment_def_id)] = full_channel_name
            segs[full_channel_name] = segments.segmentlist()
            seg_summ[full_channel_name] = segments.segmentlist()

        for seg in seg_table:
            seg_obj = segments.segment(
                    lal.LIGOTimeGPS(seg.start_time, seg.start_time_ns),
                    lal.LIGOTimeGPS(seg.end_time, seg.end_time_ns))
            segs[seg_id[int(seg.segment_def_id)]].append(seg_obj)

        for seg in seg_sum_table:
            seg_summ[seg_id[int(seg.segment_def_id)]].append(seg_obj)

        for seg_name in seg_id.values():
            segs[seg_name] = segs[seg_name].coalesce()

        xmldoc.unlink()
        curr_url = urllib.parse.urlunparse(['file', 'localhost', xml_file,
                                            None, None, None])

        return cls.from_segment_list_dict('SEGMENTS', segs, file_url=curr_url,
                                          file_exists=True,
                                          seg_summ_dict=seg_summ, **kwargs)

    def remove_short_sci_segs(self, minSegLength):
        """
        Function to remove all science segments
        shorter than a specific length. Also updates the file on disk to remove
        these segments.

        minSegLength : int
            Maximum length of science segments. Segments shorter than this will
            be removed.
        """
        newsegment_list = segments.segmentlist()
        for key, seglist in self.segment_dict.items():
            newsegment_list = segments.segmentlist()
            for seg in seglist:
                if abs(seg) > minSegLength:
                    newsegment_list.append(seg)
            newsegment_list.coalesce()
            self.segment_dict[key] = newsegment_list
        self.to_segment_xml(override_file_if_exists=True)

    def return_union_seglist(self):
        return self.segment_dict.union(self.segment_dict.keys())

    def parse_segdict_key(self, key):
        """
        Return ifo and name from the segdict key.
        """
        splt = key.split(':')
        if len(splt) == 2:
            return splt[0], splt[1]
        else:
            err_msg = "Key should be of the format 'ifo:name', got %s." %(key,)

    def to_segment_xml(self, override_file_if_exists=False):
        """
        Write the segment list in self.segmentList to self.storage_path.
        """
        # create XML doc and add process table
        process = ligolw_process.register_to_xmldoc(outdoc, sys.argv[0], {})

            ifo, name = self.parse_segdict_key(key)
            # Ensure we have LIGOTimeGPS
            fsegs = [(lal.LIGOTimeGPS(seg[0]),
                      lal.LIGOTimeGPS(seg[1])) for seg in seglist]

            if self.seg_summ_dict is None:
                vsegs = [(lal.LIGOTimeGPS(seg[0]),
                          lal.LIGOTimeGPS(seg[1])) \
                         for seg in self.valid_segments]
            else:
                         for seg in self.seg_summ_dict[key]]

            # Add using glue library to set all segment tables
            with ligolw_segments.LigolwSegments(outdoc, process) as x:
                x.add(ligolw_segments.LigolwSegmentList(active=fsegs,
                                    instruments=set([ifo]), name=name,
                                    version=1, valid=vsegs))

        # write file
        url = urljoin('file:', pathname2url(self.storage_path))
        if not override_file_if_exists or not self.has_pfn(url, site='local'):
            self.PFN(url, site='local')
        ligolw_utils.write_filename(outdoc, self.storage_path)

def make_external_call(cmdList, out_dir=None, out_basename='external_call',
                       shell=False, fail_on_error=True):
    """
    Use this to make an external call using the python subprocess module.
    See the subprocess documentation for more details of how this works.
    http://docs.python.org/2/library/subprocess.html

    cmdList : list of strings
        This list of strings contains the command to be run. See the subprocess
        documentation for more details.
    out_dir : string
        If given the stdout and stderr will be redirected to
        os.path.join(out_dir,out_basename+[".err",".out])
        If not given the stdout and stderr will not be recorded
    out_basename : string
        The value of out_basename used to construct the file names used to
        store stderr and stdout. See out_dir for more information.
    shell : boolean, default=False
        This value will be given as the shell kwarg to the subprocess call.
        **WARNING** See the subprocess documentation for details on this
        Kwarg including a warning about a serious security exploit. Do not
        use this unless you are sure it is necessary **and** safe.
    fail_on_error : boolean, default=True
        If set to true an exception will be raised if the external command does
        not return a code of 0. If set to false such failures will be ignored.
        Stderr and Stdout can be stored in either case using the out_dir
        and out_basename options.

    Returns
    --------
    exitCode : int
        The code returned by the process.
    """
    if out_dir:
        outBase = os.path.join(out_dir,out_basename)
        errFile = outBase + '.err'
        errFP = open(errFile, 'w')
        outFile = outBase + '.out'
        outFP = open(outFile, 'w')
        cmdFile = outBase + '.sh'
        cmdFP = open(cmdFile, 'w')
        cmdFP.write(' '.join(cmdList))
        cmdFP.close()
    else:
        errFile = None
        outFile = None
        cmdFile = None
        errFP = None
        outFP = None

    msg = "Making external call %s" %(' '.join(cmdList))
    logging.info(msg)
    errCode = subprocess.call(cmdList, stderr=errFP, stdout=outFP,\
                              shell=shell)
    if errFP:
        errFP.close()
    if outFP:
        outFP.close()

    if errCode and fail_on_error:
        raise CalledProcessErrorMod(errCode, ' '.join(cmdList),
                errFile=errFile, outFile=outFile, cmdFile=cmdFile)
    logging.info("Call successful, or error checking disabled.")

class CalledProcessErrorMod(Exception):
    """
    This exception is raised when subprocess.call returns a non-zero exit code
    and checking has been requested. This should not be accessed by the user
    it is used only within make_external_call.
    """
    def __init__(self, returncode, cmd, errFile=None, outFile=None,
                 cmdFile=None):
        self.returncode = returncode
        self.cmd = cmd
        self.errFile = errFile
        self.outFile = outFile
        self.cmdFile = cmdFile
        msg = "Command '%s' returned non-zero exit status %d.\n" \
              %(self.cmd, self.returncode)
        if self.errFile:
            msg += "Stderr can be found in %s .\n" %(self.errFile)
        if self.outFile:
            msg += "Stdout can be found in %s .\n" %(self.outFile)
        if self.cmdFile:
            msg += "The failed command has been printed in %s ." %(self.cmdFile)
        return msg

def resolve_url_to_file(curr_pfn, attrs=None):
    """
    Resolves a PFN into a workflow.File object.

    This function will resolve a PFN to a workflow.File object. If a File
    object already exists for that PFN that will be returned, otherwise a new
    object is returned. We will implement default site schemes here as needed,
    for example cvfms paths will be added to the osg and nonfsio sites in
    addition to local. If the LFN is a duplicate of an existing one, but with a
    different PFN an AssertionError is raised. The attrs keyword-argument can
    be used to specify attributes of a file. All files have 4 possible
    attributes. A list of ifos, an identifying string - usually used to give
    the name of the executable that created the file, a segmentlist over which
    the file is valid and tags specifying particular details about those files.
    If attrs['ifos'] is set it will be used as the ifos, otherwise this will
    default to ['H1', 'K1', 'L1', 'V1']. If attrs['exe_name'] is given this
    will replace the "exe_name" sent to File.__init__ otherwise 'INPUT' will
    be given. segs will default to [[1,2000000000]] unless overridden with
    attrs['segs']. tags will default to an empty list unless overriden
    with attrs['tag']. If attrs is None it will be ignored and all defaults
    will be used. It is emphasized that these attributes are for the most part
    not important with input files. Exceptions include things like input
    template banks, where ifos and valid times will be checked in the workflow
    and used in the naming of child job output files.
    """
    cvmfsstr1 = 'file:///cvmfs/'
    cvmfsstr2 = 'file://localhost/cvmfs/'
    cvmfsstrs = (cvmfsstr1, cvmfsstr2)

    # Get LFN
    urlp = urllib.parse.urlparse(curr_pfn)
    curr_lfn = os.path.basename(urlp.path)

    # Does this already exist as a File?
    if curr_lfn in file_input_from_config_dict.keys():
        file_pfn = file_input_from_config_dict[curr_lfn][2]
        # If the PFNs are different, but LFNs are the same then fail.
        assert(file_pfn == curr_pfn)
        curr_file = file_input_from_config_dict[curr_lfn][1]
    else:
        # Use resolve_url to download file/symlink as appropriate
        local_file_path = resolve_url(curr_pfn)
        # Create File object with default local path
        # To do this we first need to check the attributes
        if attrs and 'ifos' in attrs:
            ifos = attrs['ifos']
        else:
            ifos = ['H1', 'K1', 'L1', 'V1']
        if attrs and 'exe_name' in attrs:
            exe_name = attrs['exe_name']
        else:
            exe_name = 'INPUT'
        if attrs and 'segs' in attrs:
            segs = attrs['segs']
        else:
            segs = segments.segment([1, 2000000000])
        if attrs and 'tags' in attrs:
            tags = attrs['tags']
        else:
            tags = []

        curr_file = File(ifos, exe_name, segs, local_file_path, tags=tags)
        pfn_local = urljoin('file:', pathname2url(local_file_path))
        curr_file.PFN(pfn_local, 'local')
        # Add other PFNs for nonlocal sites as needed.
        # This block could be extended as needed
        if curr_pfn.startswith(cvmfsstrs):
            curr_file.PFN(curr_pfn, site='osg')
            curr_file.PFN(curr_pfn, site='nonfsio')
            # Also register the CVMFS PFN with the local site. We want to
            # prefer this, and symlink from here, when possible.
            # However, I think we need a little more to avoid it symlinking
            # to this through an NFS mount.
            curr_file.PFN(curr_pfn, site='local')
        # Store the file to avoid later duplication
        tuple_val = (local_file_path, curr_file, curr_pfn)
        file_input_from_config_dict[curr_lfn] = tuple_val
    return curr_file

def get_full_analysis_chunk(science_segs):
    """
    Function to find the first and last time point contained in the science segments
    and return a single segment spanning that full time.

    science_segs : ifo-keyed dictionary of ligo.segments.segmentlist instances
        The list of times that are being analysed in this workflow.

    Returns
    --------
    fullSegment : ligo.segments.segment
        The segment spanning the first and last time point contained in science_segs.
    """
    extents = [science_segs[ifo].extent() for ifo in science_segs.keys()]
    min, max = extents[0]
    for lo, hi in extents:
        if min > lo:
            min = lo
        if max < hi:
            max = hi
    fullSegment = segments.segment(min, max)
    return fullSegment

def get_random_label():
    """
    Get a random label string to use when clustering jobs.
    """
    return ''.join(random.choice(string.ascii_uppercase + string.digits) \
                   for _ in range(15))

def resolve_td_option(val_str, valid_seg):
    """
    Take an option which might be time-dependent and resolve it

    Some options might take different values depending on the GPS time. For
    example if you want opt_1 to take value_a if the time is between 10 and
    100, value_b if between 100 and 250, and value_c if between 250 and 500 you
    can supply:

    value_a[10:100],value_b[100:250],value_c[250:500].

    This function will parse that string (as opt) and return the value fully
    contained in valid_seg. If valid_seg is not full contained in one, and only
    one, of these options. The code will fail. If given a simple option like:

    value_a

    The function will just return value_a.
    """
    # Track if we've already found a matching option
    output = ''
    # Strip any whitespace, and split on comma
    curr_vals = val_str.replace(' ', '').strip().split(',')

    # Resolving the simple case is trivial and can be done immediately.
    if len(curr_vals) == 1 and '[' not in curr_vals[0]:
        return curr_vals[0]

    # Loop over all possible values
    for cval in curr_vals:
        start = int(valid_seg[0])
        end = int(valid_seg[1])
        # Extract limits for each case, and check overlap with valid_seg
        if '[' in cval:
            bopt = cval.split('[')[1].split(']')[0]
            cval = cval.replace('[' + bopt + ']', '')
        curr_seg = segments.segment(int(start), int(end))
        # The segments module is a bit weird so we need to check if the two
        # overlap using the following code. If valid_seg is fully within
        # curr_seg this will be true.
        if curr_seg.intersects(valid_seg) and \
                (curr_seg & valid_seg == valid_seg):
            if output:
                err_msg = "Time-dependent options must be disjoint."
            output = cval
    if not output:
        err_msg = "Could not resolve option {}".format(val_str)

def add_workflow_settings_cli(parser, include_subdax_opts=False):
    """Adds workflow options to an argument parser.

        Argument parser to add the options to.
    include_subdax_opts : bool, optional
        If True, will add output-map, transformation-catalog, and dax-file
        options to the parser. These can be used for workflows that are
        generated as a subdax of another workflow. Default is False.
    """
    wfgrp = parser.add_argument_group("Options for setting workflow files")
    wfgrp.add_argument("--workflow-name", required=True,
                       help="Name of the workflow.")
    wfgrp.add_argument("--tags", nargs="+", default=[],
                       help="Append the given tags to file names.")
    wfgrp.add_argument("--output-dir", default=None,
                       help="Path to directory where the workflow will be "
                            "written. Default is to use "
                            "{workflow-name}_output.")
    if include_subdax_opts:
        wfgrp.add_argument("--output-map", default="output.map",
                           help="Path to an output map file. Default is "
                                "output.map.")
        wfgrp.add_argument("--transformation-catalog", default=None,
                           help="Path to transformation catalog file.")
        wfgrp.add_argument("--dax-file", default=None,
                           help="Path to DAX file. Default is to write to the "
                                "output directory with name "
                                "{workflow-name}.dax.")

#
#
#

#
#
#                                   Preamble
#
#
"""
This module is responsible for querying a datafind server to determine the
availability of the data that the code is attempting to run on. It also
performs a number of tests and can act on these as described below. Full
documentation for this function can be found here:
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/ahope/datafind.html
"""

import os, copy
from glue import lal
from glue.ligolw import utils, table, lsctables, ligolw
from pycbc.workflow.core import SegFile, File, FileList, make_analysis_dir
from pycbc.frame import datafind_connection

    pass

def setup_datafind_workflow(workflow, scienceSegs, outputDir, seg_file=None,
                            tags=None):
    """
    Setup datafind section of the workflow. This section is responsible for
    generating, or setting up the workflow to generate, a list of files that
    record the location of the frame files needed to perform the analysis.
    There could be multiple options here, the datafind jobs could be done at
    run time or could be put into a dag. The subsequent jobs will know
    what was done here from the OutFileList containing the datafind jobs
    (and the Dagman nodes if appropriate.
    For now the only implemented option is to generate the datafind files at
    runtime. This module can also check if the frameFiles actually exist, check
    whether the obtained segments line up with the original ones and update the
    science segments to reflect missing data files.

    workflow: pycbc.workflow.core.Workflow
        The workflow class that stores the jobs that will be run.
    scienceSegs : Dictionary of ifo keyed glue.segment.segmentlist instances
        This contains the times that the workflow is expected to analyse.
    outputDir : path
        All output files written by datafind processes will be written to this
        directory.
    seg_file : SegFile, optional (default=None)
        The file returned by get_science_segments containing the science
        segments and the associated segment_summary. This will
        be used for the segment_summary test and is required if, and only if,
        performing that test.
    tags : list of string, optional (default=None)
        Use this to specify tags. This can be used if this module is being
        called more than once to give call specific configuration (by setting
        options in [workflow-datafind-${TAG}] rather than [workflow-datafind]).
        This is also used to tag the Files returned by the class to uniqueify
        the Files and uniqueify the actual filename.
        FIXME: Filenames may not be unique with current codes!

    Returns
    --------
    datafindOuts : OutGroupList
        List of all the datafind output files for use later in the pipeline.
    sci_avlble_file : SegFile
        SegFile containing the analysable time after checks in the datafind
        module are applied to the input segment list. For production runs this
        is expected to be equal to the input segment list.
        This contains the times that the workflow is expected to analyse. If
        the updateSegmentTimes kwarg is given this will be updated to reflect
        any instances of missing data.
    sci_avlble_name : string
        The name with which the analysable time is stored in the
        sci_avlble_file.
    """
        tags = []
    logging.info("Entering datafind module")
    make_analysis_dir(outputDir)
    cp = workflow.cp

    # Parse for options in ini file
    datafind_method = cp.get_opt_tags("workflow-datafind",
                                     "datafind-method", tags)

    if cp.has_option_tags("workflow-datafind",
                          "datafind-check-segment-gaps", tags):
        checkSegmentGaps = cp.get_opt_tags("workflow-datafind",
                                          "datafind-check-segment-gaps", tags)
    else:
        checkSegmentGaps = "no_test"
                          "datafind-check-frames-exist", tags):
        checkFramesExist = cp.get_opt_tags("workflow-datafind",
                                          "datafind-check-frames-exist", tags)
    else:
        checkFramesExist = "no_test"
                          "datafind-check-segment-summary", tags):
        checkSegmentSummary = cp.get_opt_tags("workflow-datafind",
                                       "datafind-check-segment-summary", tags)
    else:
        checkSegmentSummary = "no_test"

    logging.info("Starting datafind with setup_datafind_runtime_generated")
    if datafind_method == "AT_RUNTIME_MULTIPLE_CACHES":
        datafindcaches, datafindouts = \
            setup_datafind_runtime_cache_multi_calls_perifo(cp, scienceSegs,
                                                          outputDir, tags=tags)
    elif datafind_method == "AT_RUNTIME_SINGLE_CACHES":
            setup_datafind_runtime_cache_single_call_perifo(cp, scienceSegs,
    elif datafind_method == "AT_RUNTIME_MULTIPLE_FRAMES":
            setup_datafind_runtime_frames_multi_calls_perifo(cp, scienceSegs,
    elif datafind_method == "AT_RUNTIME_SINGLE_FRAMES":
            setup_datafind_runtime_frames_single_call_perifo(cp, scienceSegs,
    elif datafind_method == "AT_RUNTIME_FAKE_DATA":
        pass
    elif datafind_method == "FROM_PREGENERATED_LCF_FILES":
        ifos = scienceSegs.keys()
            setup_datafind_from_pregenerated_lcf_files(cp, ifos,
                                                       outputDir, tags=tags)
    else:
        msg = """Entry datafind-method in [workflow-datafind] does not have "
              expected value. Valid values are 
              AT_RUNTIME_MULTIPLE_FRAMES, AT_RUNTIME_SINGLE_FRAMES 
              AT_RUNTIME_MULTIPLE_CACHES, AT_RUNTIME_SINGLE_CACHES,
              FROM_PREGENERATED_LCF_FILES, or AT_RUNTIME_FAKE_DATA.
              Consult the documentation for more info."""
        raise ValueError(msg)

    using_backup_server = False
    if datafind_method == "AT_RUNTIME_MULTIPLE_FRAMES" or \
                                  datafind_method == "AT_RUNTIME_SINGLE_FRAMES":
        if cp.has_option_tags("workflow-datafind",
                          "datafind-backup-datafind-server", tags):
            using_backup_server = True
            backup_server = cp.get_opt_tags("workflow-datafind",
                                      "datafind-backup-datafind-server", tags)
            cp_new = copy.deepcopy(cp)
            cp_new.set("workflow-datafind",
                                "datafind-ligo-datafind-server", backup_server)
            cp_new.set('datafind', 'urltype', 'gsiftp')
            backup_datafindcaches, backup_datafindouts =\
                setup_datafind_runtime_frames_single_call_perifo(cp_new,
                                             scienceSegs, outputDir, tags=tags)
            backup_datafindouts = datafind_keep_unique_backups(\
                                             backup_datafindouts, datafindouts)
            datafindcaches.extend(backup_datafindcaches)
            datafindouts.extend(backup_datafindouts)

    logging.info("setup_datafind_runtime_generated completed")
    # If we don't have frame files covering all times we can update the science
    # segments.
    if checkSegmentGaps in ['warn','update_times','raise_error']:
        logging.info("Checking science segments against datafind output....")
        newScienceSegs = get_science_segs_from_datafind_outs(datafindcaches)
        logging.info("New segments calculated from data find output.....")
        missingData = False
        for ifo in scienceSegs.keys():
            # If no science segments in input then do nothing
            if not scienceSegs[ifo]:
                msg = "No science segments are present for ifo %s, " %(ifo)
                msg += "the segment metadata indicates there is no analyzable"
                msg += " strain data between the selected GPS start and end "
                msg += "times."
                logging.warning(msg)
                continue
            if ifo not in newScienceSegs:
                msg = "No data frames were found corresponding to the science "
                msg += "segments for ifo %s" %(ifo)
                logging.error(msg)
                missingData = True
                if checkSegmentGaps == 'update_times':
                    scienceSegs[ifo] = segments.segmentlist()
                continue
            missing = scienceSegs[ifo] - newScienceSegs[ifo]
            if abs(missing):
                msg = "From ifo %s we are missing frames covering:" %(ifo)
                msg += "\n%s" % "\n".join(map(str, missing))
                    # Remove missing time, so that we can carry on if desired
                    logging.info("Updating science segments for ifo %s."
                                 %(ifo))
                    scienceSegs[ifo] = scienceSegs[ifo] - missing

        if checkSegmentGaps == 'raise_error' and missingData:
            raise ValueError("Workflow cannot find needed data, exiting.")
        logging.info("Done checking, any discrepancies are reported above.")
    elif checkSegmentGaps == 'no_test':
        pass
    else:
        errMsg = "checkSegmentGaps kwarg must take a value from 'no_test', "
        errMsg += "'warn', 'update_times' or 'raise_error'."

    # Do all of the frame files that were returned actually exist?
    if checkFramesExist in ['warn','update_times','raise_error']:
        logging.info("Verifying that all frames exist on disk.")
        missingFrSegs, missingFrames = \
                          get_missing_segs_from_frame_file_cache(datafindcaches)
        missingFlag = False
        for ifo in missingFrames.keys():
            # If no data in the input then do nothing
                continue
            # If using a backup server, does the frame exist remotely?
            if using_backup_server:
                # WARNING: This will be slow, but hopefully it will not occur
                #          for too many frames. This could be optimized if
                #          it becomes necessary.
                new_list = []
                for frame in missingFrames[ifo]:
                    for dfout in datafindouts:
                        dfout_pfns = list(dfout.pfns)
                        dfout_urls = [a.url for a in dfout_pfns]
                        if frame.url in dfout_urls:
                            pfn = dfout_pfns[dfout_urls.index(frame.url)]
                            dfout.removePFN(pfn)
                            if len(dfout.pfns) == 0:
                                new_list.append(frame)
                            else:
                                msg = "Frame %s not found locally. "\
                                                                  %(frame.url,)
                                msg += "Replacing with remote url(s) %s." \
                                           %(str([a.url for a in dfout.pfns]),)
                                logging.info(msg)
                            break
                    else:
                        new_list.append(frame)
                missingFrames[ifo] = new_list
            if missingFrames[ifo]:
                msg = "From ifo %s we are missing the following frames:" %(ifo)
                msg +='\n'.join([a.url for a in missingFrames[ifo]])
                missingFlag = True
            if checkFramesExist == 'update_times':
                # Remove missing times, so that we can carry on if desired
                logging.info("Updating science times for ifo %s." %(ifo))
                scienceSegs[ifo] = scienceSegs[ifo] - missingFrSegs[ifo]

        if checkFramesExist == 'raise_error' and missingFlag:
            raise ValueError("Workflow cannot find all frames, exiting.")
        logging.info("Finished checking frames.")
    elif checkFramesExist == 'no_test':
        pass
    else:
        errMsg = "checkFramesExist kwarg must take a value from 'no_test', "

    # Check if there are cases where frames exist, but no entry in the segment
    # summary table are present.
    if checkSegmentSummary in ['warn', 'raise_error']:
        logging.info("Checking the segment summary table against frames.")
        dfScienceSegs = get_science_segs_from_datafind_outs(datafindcaches)
        # NOTE: Should this be overrideable in the config file?
        sci_seg_name = "SCIENCE"
        if seg_file is None:
            err_msg = "You must provide the science segments SegFile object "
            err_msg += "if using the datafind-check-segment-summary option."
        if seg_file.seg_summ_dict is None:
            err_msg = "The provided science segments SegFile object must "
            err_msg += "contain a valid segment_summary table if using the "
            err_msg += "datafind-check-segment-summary option."
        seg_summary_times = seg_file.seg_summ_dict
        for ifo in dfScienceSegs.keys():
            curr_seg_summ_times = seg_summary_times[ifo + ":" + sci_seg_name]
            missing = (dfScienceSegs[ifo] & seg_file.valid_segments)
            missing.coalesce()
            missing = missing - curr_seg_summ_times
            scienceButNotFrame = scienceSegs[ifo] - dfScienceSegs[ifo]
            scienceButNotFrame.coalesce()
            missing2 = scienceSegs[ifo] - scienceButNotFrame
            missing2.coalesce()
            missing2 = missing2 - curr_seg_summ_times
                msg = "From ifo %s the following times have frames, " %(ifo)
                msg += "but are not covered in the segment summary table."
            if abs(missing2):
                msg += "are science, and are not covered in the segment "
                msg += "summary table."
                msg += "\n%s" % "\n".join(map(str, missing2))
        if checkSegmentSummary == 'raise_error' and missingFlag:
            errMsg = "Segment_summary discrepancy detected, exiting."
    elif checkSegmentSummary == 'no_test':
        pass
    else:
        errMsg = "checkSegmentSummary kwarg must take a value from 'no_test', "
        errMsg += "'warn', or 'raise_error'."

    # Now need to create the file for SCIENCE_AVAILABLE
    sci_avlble_dict = segments.segmentlistdict()
    # NOTE: Should this be overrideable in the config file?
    sci_avlble_name = "SCIENCE_AVAILABLE"
    for ifo in scienceSegs.keys():
        sci_avlble_dict[ifo + ':' + sci_avlble_name] = scienceSegs[ifo]

    sci_avlble_file = SegFile.from_segment_list_dict('SCIENCE_AVAILABLE',
                            sci_avlble_dict, ifo_list = scienceSegs.keys(),
                            valid_segment=workflow.analysis_time,
                            extension='.xml', tags=tags, directory=outputDir)

    logging.info("Leaving datafind module")
    if datafind_method == "AT_RUNTIME_FAKE_DATA":
        datafindouts = None
    else:
        datafindouts = FileList(datafindouts) 

    return datafindouts, sci_avlble_file, scienceSegs, sci_avlble_name

def setup_datafind_runtime_cache_multi_calls_perifo(cp, scienceSegs,
                                                    outputDir, tags=None):
    """
    This function uses the `gwdatafind` library to obtain the location of all
    the frame files that will be needed to cover the analysis of the data
    given in scienceSegs. This function will not check if the returned frames
    cover the whole time requested, such sanity checks are done in the
    pycbc.workflow.setup_datafind_workflow entry function. As opposed to
    setup_datafind_runtime_single_call_perifo this call will one call to the
    datafind server for every science segment. This function will return a list
    of output files that correspond to the cache .lcf files that are produced,
    which list the locations of all frame files. This will cause problems with
    pegasus, which expects to know about all input files (ie. the frame files
    themselves.)

    cp : ConfigParser.ConfigParser instance
        This contains a representation of the information stored within the
        workflow configuration files
    tags : list of strings, optional (default=None)

    Returns
    --------
    datafindcaches : list of glue.lal.Cache instances
       The glue.lal.Cache representations of the various calls to the datafind
       server and the returned frame files.
    datafindOuts : pycbc.workflow.core.FileList

    """
        tags = []

    # First job is to do setup for the datafind jobs
    # First get the server name
    logging.info("Setting up connection to datafind server.")
    connection = setup_datafind_server_connection(cp, tags=tags)

    # Now ready to loop over the input segments
    datafindouts = []
    datafindcaches = []
    logging.info("Querying datafind server for all science segments.")
    for ifo, scienceSegsIfo in scienceSegs.items():
        observatory = ifo[0].upper()
        frameType = cp.get_opt_tags("workflow-datafind",
                                "datafind-%s-frame-type" % (ifo.lower()), tags)
        for seg in scienceSegsIfo:
            msg = "Finding data between %d and %d " %(seg[0],seg[1])
            msg += "for ifo %s" %(ifo)
            logging.info(msg)
            # WARNING: For now the workflow will expect times to be in integer seconds
            startTime = int(seg[0])
            endTime = int(seg[1])

            # Sometimes the connection can drop, so try a backup here
            try:
                cache, cache_file = run_datafind_instance(cp, outputDir,
                                           connection, observatory, frameType,
                                           startTime, endTime, ifo, tags=tags)
            except:
                connection = setup_datafind_server_connection(cp, tags=tags)
            datafindouts.append(cache_file)
            datafindcaches.append(cache)
    return datafindcaches, datafindouts

def setup_datafind_runtime_cache_single_call_perifo(cp, scienceSegs, outputDir,
                                              tags=None):
    """
    setup_datafind_runtime_generated this call will only run one call to
    datafind per ifo, spanning the whole time. This function will return a list

    Returns
    --------

    """
        tags = []

    # We want to ignore gaps as the detectors go up and down and calling this
    # way will give gaps. See the setup_datafind_runtime_generated function
    # for datafind calls that only query for data that will exist
    cp.set("datafind","on_gaps","ignore")

        checked_times = segments.segmentlist([])
        frame_types = cp.get_opt_tags(
            "workflow-datafind",
            "datafind-%s-frame-type" % (ifo.lower()), tags
        )
        # Check if this is one type, or time varying
        frame_types = frame_types.replace(' ', '').strip().split(',')
        for ftype in frame_types:
            # Check the times, default to full time initially
            # This REQUIRES a coalesced segment list to work
            start = int(scienceSegsIfo[0][0])
            end = int(scienceSegsIfo[-1][1])
            # Then check for limits. We're expecting something like:
            # value[start:end], so need to extract value, start and end
            if '[' in ftype:
                # This gets start and end out
                bopt = ftype.split('[')[1].split(']')[0]
                newstart, newend = bopt.split(':')
                # Then check if the times are within science time
                start = max(int(newstart), start)
                end = min(int(newend), end)
                if end <= start:
                    continue
                # This extracts value
                ftype = ftype.split('[')[0]
            curr_times = segments.segment(start, end)
            # The times here must be distinct. We cannot have two different
            # frame files at the same time from the same ifo.
            if checked_times.intersects_segment(curr_times):
                err_msg = "Different frame types cannot overlap in time."
            checked_times.append(curr_times)

            # Ask datafind where the frames are
            try:
                cache, cache_file = run_datafind_instance(
                    cp,
                    outputDir,
                    connection,
                    observatory,
                    ftype,
                    start,
                    end,
                    ifo,
                    tags=tags
                )
            except:
                    cp,
                    ftype,
                    start,
                    end,
                    ifo,
                    tags=tags
                )

def setup_datafind_runtime_frames_single_call_perifo(cp, scienceSegs,
                                              outputDir, tags=None):
    """
    of files corresponding to the individual frames returned by the datafind
    query. This will allow pegasus to more easily identify all the files used
    as input, but may cause problems for codes that need to take frame cache
    files as input.

    Returns
    --------

    """
    datafindcaches, _ = \
        setup_datafind_runtime_cache_single_call_perifo(cp, scienceSegs,
                                                        outputDir, tags=tags)

    datafindouts = convert_cachelist_to_filelist(datafindcaches)

def setup_datafind_runtime_frames_multi_calls_perifo(cp, scienceSegs,
                                                     outputDir, tags=None):
    """

    Returns
    --------

    """
        setup_datafind_runtime_cache_multi_calls_perifo(cp, scienceSegs,

def setup_datafind_from_pregenerated_lcf_files(cp, ifos, outputDir, tags=None):
    """
    This function is used if you want to run with pregenerated lcf frame
    cache files.

    ifos : list of ifo strings
        List of ifos to get pregenerated files for.
        directory. Currently this sub-module writes no output.

    Returns
    --------
    """
        tags = []

        search_string = "datafind-pregenerated-cache-file-%s" %(ifo.lower(),)
        frame_cache_file_name = cp.get_opt_tags("workflow-datafind",
                                                search_string, tags=tags)
        curr_cache = lal.Cache.fromfilenames([frame_cache_file_name],
                                             coltype=lal.LIGOTimeGPS)
        curr_cache.ifo = ifo
        datafindcaches.append(curr_cache)

def convert_cachelist_to_filelist(datafindcache_list):
    """
    Take as input a list of glue.lal.Cache objects and return a pycbc FileList
    containing all frames within those caches.

    datafindcache_list : list of glue.lal.Cache objects
        The list of cache files to convert.

    Returns
    --------
    datafind_filelist : FileList of frame File objects
        The list of frame files.
    """
    prev_file = None
    prev_name = None
    this_name = None

    datafind_filelist = FileList([])

    for cache in datafindcache_list:
        # sort the cache into time sequential order
        cache.sort()
        curr_ifo = cache.ifo
        for frame in cache:
            # Pegasus doesn't like "localhost" in URLs.
            frame.url = frame.url.replace('file://localhost','file://')

            # Create one File() object for each unique frame file that we
            # get back in the cache.
            if prev_file:
                prev_name = os.path.basename(prev_file.cache_entry.url)
                this_name = os.path.basename(frame.url)

            if (prev_file is None) or (prev_name != this_name):
                currFile = File(curr_ifo, frame.description,
                    frame.segment, file_url=frame.url, use_tmp_subdirs=True)
                datafind_filelist.append(currFile)
                prev_file = currFile

            # Populate the PFNs for the File() we just created
            if frame.url.startswith('file://'):
                currFile.PFN(frame.url, site='local')
                if frame.url.startswith(
                    'file:///cvmfs/oasis.opensciencegrid.org/ligo/frames'):
                    # Datafind returned a URL valid on the osg as well
                    # so add the additional PFNs to allow OSG access.
                    currFile.PFN(frame.url, site='osg')
                    currFile.PFN(frame.url.replace(
                        'file:///cvmfs/oasis.opensciencegrid.org/',
                        'root://xrootd-local.unl.edu/user/'), site='osg')
                        'gsiftp://red-gridftp.unl.edu/user/'), site='osg')
                        'gsiftp://ldas-grid.ligo.caltech.edu/hdfs/'), site='osg')
                elif frame.url.startswith(
                    'file:///cvmfs/gwosc.osgstorage.org/'):
                    for s in ['osg', 'orangegrid', 'osgconnect']:
                        currFile.PFN(frame.url, site=s)
                        currFile.PFN(frame.url, site="{}-scratch".format(s))
            else:
                currFile.PFN(frame.url, site='notlocal')

    return datafind_filelist

def get_science_segs_from_datafind_outs(datafindcaches):
    """
    This function will calculate the science segments that are covered in
    the OutGroupList containing the frame files returned by various
    calls to the datafind server. This can then be used to check whether this
    list covers what it is expected to cover.

    datafindcaches : OutGroupList
        List of all the datafind output files.

    Returns
    --------
    newScienceSegs : Dictionary of ifo keyed glue.segment.segmentlist instances
        The times covered by the frames found in datafindOuts.
    """
    newScienceSegs = {}
    for cache in datafindcaches:
        if len(cache) > 0:
            groupSegs = segments.segmentlist(e.segment for e in cache).coalesce()
            ifo = cache.ifo
                newScienceSegs[ifo] = groupSegs
            else:
                newScienceSegs[ifo].extend(groupSegs)
                newScienceSegs[ifo].coalesce()
    return newScienceSegs

def get_missing_segs_from_frame_file_cache(datafindcaches):
    """
    This function will use os.path.isfile to determine if all the frame files
    returned by the local datafind server actually exist on the disk. This can
    then be used to update the science times if needed.

    Returns
    --------
    missingFrameSegs : Dict. of ifo keyed glue.segment.segmentlist instances
        The times corresponding to missing frames found in datafindOuts.
    missingFrames: Dict. of ifo keyed lal.Cache instances
        The list of missing frames
    """
    missingFrameSegs = {}
    missingFrames = {}
            # Don't bother if these are not file:// urls, assume all urls in
            # one cache file must be the same type
            if not cache[0].scheme == 'file':
                warn_msg = "We have %s entries in the " %(cache[0].scheme,)
                warn_msg += "cache file. I do not check if these exist."
                logging.info(warn_msg)
                continue
            _, currMissingFrames = cache.checkfilesexist(on_missing="warn")
            missingSegs = segments.segmentlist(e.segment \
                                         for e in currMissingFrames).coalesce()
            if ifo not in missingFrameSegs:
                missingFrameSegs[ifo] = missingSegs
                missingFrames[ifo] = lal.Cache(currMissingFrames)
            else:
                missingFrameSegs[ifo].extend(missingSegs)
                # NOTE: This .coalesce probably isn't needed as the segments
                # should be disjoint. If speed becomes an issue maybe remove it?
                missingFrameSegs[ifo].coalesce()
                missingFrames[ifo].extend(currMissingFrames)
    return missingFrameSegs, missingFrames

def setup_datafind_server_connection(cp, tags=None):
    """
    This function is resposible for setting up the connection with the datafind
    server.

    cp : pycbc.workflow.configuration.WorkflowConfigParser
        The memory representation of the ConfigParser
    Returns
    --------
    """
        tags = []

                          "datafind-ligo-datafind-server", tags):
        datafind_server = cp.get_opt_tags("workflow-datafind",
                                        "datafind-ligo-datafind-server", tags)
    else:
        datafind_server = None

    return datafind_connection(datafind_server)

def get_segment_summary_times(scienceFile, segmentName):
    """
    This function will find the times for which the segment_summary is set
    for the flag given by segmentName.

    scienceFile : SegFile
        The segment file that we want to use to determine this.
    segmentName : string
        The DQ flag to search for times in the segment_summary table.

    Returns
    ---------
    summSegList : ligo.segments.segmentlist
        The times that are covered in the segment summary table.
    """
    # Parse the segmentName
    segmentName = segmentName.split(':')
    if not len(segmentName) in [2,3]:
        raise ValueError("Invalid channel name %s." %(segmentName))
    ifo = segmentName[0]
    channel = segmentName[1]
    version = ''
    if len(segmentName) == 3:
        version = int(segmentName[2])

    # Load the filename
    xmldoc = utils.load_filename(scienceFile.cache_entry.path,
                             gz=scienceFile.cache_entry.path.endswith("gz"),
                             contenthandler=ContentHandler)

    # Get the segment_def_id for the segmentName
    segmentDefTable = table.get_table(xmldoc, "segment_definer")
    for entry in segmentDefTable:
        if (entry.ifos == ifo) and (entry.name == channel):
            if len(segmentName) == 2 or (entry.version==version):
                segDefID = entry.segment_def_id
                break
    else:
        raise ValueError("Cannot find channel %s in segment_definer table."\
                         %(segmentName))

    # Get the segmentlist corresponding to this segmentName in segment_summary
    segmentSummTable = table.get_table(xmldoc, "segment_summary")
    summSegList = segments.segmentlist([])
    for entry in segmentSummTable:
        if entry.segment_def_id == segDefID:
            segment = segments.segment(entry.start_time, entry.end_time)
            summSegList.append(segment)
    summSegList.coalesce()

    return summSegList

def run_datafind_instance(cp, outputDir, connection, observatory, frameType,
                          startTime, endTime, ifo, tags=None):
    """
    This function will query the datafind server once to find frames between
    the specified times for the specified frame type and observatory.

        Source for any kwargs that should be sent to the datafind module
    outputDir : Output cache files will be written here. We also write the
        commands for reproducing what is done in this function to this
    connection : datafind connection object
        Initialized through the `gwdatafind` module, this is the open
        connection to the datafind server.
    observatory : string
        The observatory to query frames for. Ex. 'H', 'L' or 'V'.  NB: not
        'H1', 'L1', 'V1' which denote interferometers.
    frameType : string
        The frame type to query for.
    startTime : int
        Integer start time to query the datafind server for frames.
    endTime : int
        Integer end time to query the datafind server for frames.
        The interferometer to use for naming output. Ex. 'H1', 'L1', 'V1'.
        Maybe this could be merged with the observatory string, but this
        could cause issues if running on old 'H2' and 'H1' data.
        the Files and uniquify the actual filename.

    Returns
    --------
    dfCache : glue.lal.Cache instance
       The glue.lal.Cache representation of the call to the datafind
    cacheFile : pycbc.workflow.core.File
        Cache file listing all of the datafind output files for use later in the pipeline.

    """
        tags = []

    seg = segments.segment([startTime, endTime])
    # Take the datafind kwargs from config (usually urltype=file is
    # given).
    dfKwargs = {}
    # By default ignore missing frames, this case is dealt with outside of here
    dfKwargs['on_gaps'] = 'ignore'
    if cp.has_section("datafind"):
        for item, value in cp.items("datafind"):
            dfKwargs[item] = value
        if cp.has_section('datafind-%s' %(tag)):
            for item, value in cp.items("datafind-%s" %(tag)):
                dfKwargs[item] = value

    # It is useful to print the corresponding command to the logs
    # directory to check if this was expected.
    log_datafind_command(observatory, frameType, startTime, endTime,
                         os.path.join(outputDir,'logs'), **dfKwargs)
    logging.debug("Asking datafind server for frames.")
    dfCache = lal.Cache.from_urls(
        connection.find_frame_urls(observatory, frameType,
                                   startTime, endTime, **dfKwargs),
    )
    logging.debug("Frames returned")
    # workflow format output file
    cache_file = File(ifo, 'DATAFIND', seg, extension='lcf',
                      directory=outputDir, tags=tags)
    cache_file.PFN(cache_file.cache_entry.path, site='local')

    dfCache.ifo = ifo
    # Dump output to file
    fP = open(cache_file.storage_path, "w")
    # FIXME: CANNOT use dfCache.tofile because it will print 815901601.00000
    #        as a gps time which is incompatible with the lal cache format
    #        (and the C codes) which demand an integer.
    #dfCache.tofile(fP)
    for entry in dfCache:
        start = str(int(entry.segment[0]))
        duration = str(int(abs(entry.segment)))
        print("%s %s %s %s %s" \
              % (entry.observatory, entry.description, start, duration, entry.url), file=fP)
        entry.segment = segments.segment(int(entry.segment[0]), int(entry.segment[1]))

    fP.close()
    return dfCache, cache_file

def log_datafind_command(observatory, frameType, startTime, endTime,
                         outputDir, **dfKwargs):
    """
    This command will print an equivalent gw_data_find command to disk that
    can be used to debug why the internal datafind module is not working.
    """
    # FIXME: This does not accurately reproduce the call as assuming the
    # kwargs will be the same is wrong, so some things need to be converted
    # "properly" to the command line equivalent.
    gw_command = ['gw_data_find', '--observatory', observatory,
                  '--type', frameType,
                  '--gps-start-time', str(startTime),
                  '--gps-end-time', str(endTime)]

    for name, value in dfKwargs.items():
        if name == 'match':
            gw_command.append("--match")
            gw_command.append(str(value))
        elif name == 'urltype':
            gw_command.append("--url-type")
        elif name == 'on_gaps':
            pass
        else:
            errMsg = "Unknown datafind kwarg given: %s. " %(name)
            errMsg+= "This argument is stripped in the logged .sh command."
            logging.warn(errMsg)

    fileName = "%s-%s-%d-%d.sh" \
               %(observatory, frameType, startTime, endTime-startTime)
    filePath = os.path.join(outputDir, fileName)
    fP = open(filePath, 'w')
    fP.write(' '.join(gw_command))

def datafind_keep_unique_backups(backup_outs, orig_outs):
    """This function will take a list of backup datafind files, presumably
    obtained by querying a remote datafind server, e.g. CIT, and compares
    these against a list of original datafind files, presumably obtained by
    querying the local datafind server. Only the datafind files in the backup
    list that do not appear in the original list are returned. This allows us
    to use only files that are missing from the local cluster.

    backup_outs : FileList
        List of datafind files from the remote datafind server.
    orig_outs : FileList
        List of datafind files from the local datafind server.

    Returns
    --------
    FileList
        List of datafind files in backup_outs and not in orig_outs.
    """
    # NOTE: This function is not optimized and could be made considerably
    #       quicker if speed becomes in issue. With 4s frame files this might
    #       be slow, but for >1000s files I don't foresee any issue, so I keep
    #       this simple.
    return_list = FileList([])
    # We compare the LFNs to determine uniqueness
    # Is there a way to associate two paths with one LFN??
    orig_names = [f.name for f in orig_outs]
    for file in backup_outs:
        if file.name not in orig_names:
            return_list.append(file)
        else:
            index_num = orig_names.index(file.name)
            orig_out = orig_outs[index_num]
            pfns = list(file.pfns)
            # This shouldn't happen, but catch if it does
            assert(len(pfns) == 1)
            orig_out.PFN(pfns[0].url, site='notlocal')

    return return_list

# Copyright (C) 2015  Andrew Williamson
#
#
#

#
#
#                                   Preamble
#
#

"""
This library code contains functions and classes that are used in the
generation of pygrb workflows. For details about pycbc.workflow see here:
http://pycbc.org/pycbc/latest/html/workflow.html
"""

import sys
import os
from glue.ligolw import ligolw, lsctables, utils, ilwd
from pycbc.workflow.core import File, FileList, resolve_url_to_file
from pycbc.workflow.jobsetup import select_generic_executable

def set_grb_start_end(cp, start, end):
    """
    Function to update analysis boundaries as workflow is generated

    cp : pycbc.workflow.configuration.WorkflowConfigParser object
    The parsed configuration options of a pycbc.workflow.core.Workflow.

    start : int
    The start of the workflow analysis time.

    end : int
    The end of the workflow analysis time.

    Returns
    --------
    The modified WorkflowConfigParser object.

    """
    cp.set("workflow", "start-time", str(start))
    cp.set("workflow", "end-time", str(end))

    return cp

def get_coh_PTF_files(cp, ifos, run_dir, bank_veto=False, summary_files=False):
    """
    Retrieve files needed to run coh_PTF jobs within a PyGRB workflow

    ifos : str
    String containing the analysis interferometer IDs.

    run_dir : str
    The run directory, destination for retrieved files.

    bank_veto : Boolean
    If true, will retrieve the bank_veto_bank.xml file.

    summary_files : Boolean
    If true, will retrieve the summary page style files.

    Returns
    -------
    file_list : pycbc.workflow.FileList object
    A FileList containing the retrieved files.
    """
    if os.getenv("LAL_SRC") is None:
        raise ValueError("The environment variable LAL_SRC must be set to a "
                         "location containing the file lalsuite.git")
    else:
        lalDir = os.getenv("LAL_SRC")
        sci_seg = segments.segment(int(cp.get("workflow", "start-time")),
                                   int(cp.get("workflow", "end-time")))
        file_list = FileList([])

        # Bank veto
        if bank_veto:
            shutil.copy("%s/lalapps/src/ring/coh_PTF_config_files/" \
                        "bank_veto_bank.xml" % lalDir, "%s" % run_dir)
            bank_veto_url = "file://localhost%s/bank_veto_bank.xml" % run_dir
            bank_veto = File(ifos, "bank_veto_bank", sci_seg,
                             file_url=bank_veto_url)
            bank_veto.PFN(bank_veto.cache_entry.path, site="local")
            file_list.extend(FileList([bank_veto]))

        if summary_files:
            # summary.js file
                        "coh_PTF_html_summary.js" % lalDir, "%s" % run_dir)
            summary_js_url = "file://localhost%s/coh_PTF_html_summary.js" \
                             % run_dir
            summary_js = File(ifos, "coh_PTF_html_summary_js", sci_seg,
                              file_url=summary_js_url)
            summary_js.PFN(summary_js.cache_entry.path, site="local")
            file_list.extend(FileList([summary_js]))

            # summary.css file
                        "coh_PTF_html_summary.css" % lalDir, "%s" % run_dir)
            summary_css_url = "file://localhost%s/coh_PTF_html_summary.css" \
                              % run_dir
            summary_css = File(ifos, "coh_PTF_html_summary_css", sci_seg,
                               file_url=summary_css_url)
            summary_css.PFN(summary_css.cache_entry.path, site="local")
            file_list.extend(FileList([summary_css]))

        return file_list

def make_exttrig_file(cp, ifos, sci_seg, out_dir):
    '''
    Make an ExtTrig xml file containing information on the external trigger

    ifos : str

    sci_seg : ligo.segments.segment
    The science segment for the analysis run.

    out_dir : str
    The output directory, destination for xml file.

    Returns
    -------
    xml_file : pycbc.workflow.File object
    The xml file with external trigger information.

    '''
    # Initialise objects
    xmldoc = ligolw.Document()
    xmldoc.appendChild(ligolw.LIGO_LW())
    tbl = lsctables.New(lsctables.ExtTriggersTable)
    cols = tbl.validcolumns
    xmldoc.childNodes[-1].appendChild(tbl)
    row = tbl.appendRow()

    # Add known attributes for this GRB
    setattr(row, "event_ra", float(cp.get("workflow", "ra")))
    setattr(row, "event_dec", float(cp.get("workflow", "dec")))
    setattr(row, "start_time", int(cp.get("workflow", "trigger-time")))
    setattr(row, "event_number_grb", str(cp.get("workflow", "trigger-name")))

    # Fill in all empty rows
        if not hasattr(row, entry):
                setattr(row,entry,0.)
                setattr(row,entry,0)
                setattr(row,entry,'')
                row.process_id = ilwd.ilwdchar("external_trigger:process_id:0")
                row.event_id = ilwd.ilwdchar("external_trigger:event_id:0")
            else:
                print("Column %s not recognized" %(entry), file=sys.stderr)
                raise ValueError

    # Save file
    xml_file_name = "triggerGRB%s.xml" % str(cp.get("workflow",
                                                    "trigger-name"))
    xml_file_path = os.path.join(out_dir, xml_file_name)
    utils.write_filename(xmldoc, xml_file_path)
    xml_file_url = urljoin("file:", pathname2url(xml_file_path))
    xml_file = File(ifos, xml_file_name, sci_seg, file_url=xml_file_url)
    xml_file.PFN(xml_file_url, site="local")

    return xml_file

def get_ipn_sky_files(workflow, file_url, tags=None):
    '''
    Retreive the sky point files for searching over the IPN error box and
    populating it with injections.

        An instanced class that manages the constructed workflow.
    file_url : string
        The URL of the IPN sky points file.
    tags : list of strings
        If given these tags are used to uniquely name and identify output files
        that would be produced in multiple calls to this function.

    Returns
    --------
    sky_points_file : pycbc.workflow.core.File
        File object representing the IPN sky points file.
    '''
    tags = tags or []
    file_attrs = {
        'ifos': workflow.ifos,
        'segs': workflow.analysis_time,
        'exe_name': "IPN_SKY_POINTS",
        'tags': tags
    }
    sky_points_file = resolve_url_to_file(file_url, attrs=file_attrs)

    return sky_points_file

def make_gating_node(workflow, datafind_files, outdir=None, tags=None):
    '''
    Generate jobs for autogating the data for PyGRB runs.

    datafind_files : pycbc.workflow.core.FileList
        A FileList containing the frame files to be gated.
    outdir : string
        Path of the output directory

    Returns
    --------
    condition_strain_nodes : list
        List containing the pycbc.workflow.core.Node objects representing the
        autogating jobs.
    condition_strain_outs : pycbc.workflow.core.FileList
        FileList containing the pycbc.workflow.core.File objects representing
        the gated frame files.
    '''

        tags = []

    condition_strain_class = select_generic_executable(workflow,
                                                       "condition_strain")
    condition_strain_nodes = []
    condition_strain_outs = FileList([])
        input_files = FileList([datafind_file for datafind_file in \
                                datafind_files if datafind_file.ifo == ifo])
        condition_strain_jobs = condition_strain_class(cp, "condition_strain",
                ifo=ifo, out_dir=outdir, tags=tags)
        condition_strain_node, condition_strain_out = \
                condition_strain_jobs.create_node(input_files, tags=tags)
        condition_strain_nodes.append(condition_strain_node)
        condition_strain_outs.extend(FileList([condition_strain_out]))

    return condition_strain_nodes, condition_strain_outs

def get_sky_grid_scale(sky_error, sigma_sys=6.8359):
    """
    Calculate suitable 3-sigma radius of the search patch, incorporating Fermi
    GBM systematic if necessary.
    """
    return 1.65 * (sky_error**2 + sigma_sys**2)**0.5

# Copyright (C) 2016 Christopher M. Biwer, Alexander Harvey Nitz, Collin Capano
#
#
#
"""
Module that contains functions for setting up the inference workflow.
"""

import logging, os.path
from Pegasus import DAX3 as dax
from pycbc.workflow.core import (Executable, FileList, Node, makedir, File,
                                 Workflow)
from pycbc.workflow.plotting import PlotExecutable, requirestr, excludestr
from pycbc.results import layout
from pycbc.workflow import pegasus_workflow as wdax

def make_inference_plot(workflow, input_file, output_dir,
                        name, analysis_seg=None,
                        tags=None, input_file_opt='input-file',
                        output_file_extension='.png',
                        add_to_workflow=False):
    """Boiler-plate function for creating a standard plotting job.

    workflow: pycbc.workflow.Workflow
        The core workflow instance we are populating
    input_file: (list of) pycbc.workflow.File
        The file used for the input. May provide either a single file or a
        list of files.
    output_dir: str
        The directory to store result plots.
    name: str
        The name in the [executables] section of the configuration file
        to use.
    analysis_segs: ligo.segments.Segment, optional
       The segment this job encompasses. If None then use the total analysis
       time from the workflow.
    tags: list, optional
        Tags to add to the inference executables.
    input_file_opt : str, optional
        The name of the input-file option used by the executable. Default
        is ``input-file``.
    output_file_extension : str, optional
        What file type to create. Default is ``.png``.
    add_to_workflow : bool, optional
        If True, the node will be added to the workflow before being returned.
        **This means that no options may be added to the node afterward.**
        Default is ``False``.

    Returns
    -------
    pycbc.workflow.plotting.PlotExecutable
        The job node for creating the plot.
    """
    # default values
        tags = []
    if analysis_seg is None:
        analysis_seg = workflow.analysis_time
    # make the directory that will contain the output files
    makedir(output_dir)
    # Catch if a parameters option was specified:
    # we need to do this because PlotExecutable will automatically add any
    # option in the section to the node. However, we need to add the
    # appropriate escapes to the parameters option so pegasus will render it
    # properly (see _params_for_pegasus for details).
    parameters = None
    if workflow.cp.has_option(name, 'parameters'):
        parameters = workflow.cp.get(name, 'parameters')
        workflow.cp.remove_option(name, 'parameters')
    # make a node for plotting the posterior as a corner plot
    node = PlotExecutable(workflow.cp, name, ifos=workflow.ifos,
                          out_dir=output_dir,
                          tags=tags).create_node()
    # add back the parameters option if it was specified
    if parameters is not None:
        node.add_opt("--parameters", _params_for_pegasus(parameters))
        # and put the opt back in the config file in memory
        workflow.cp.set(name, 'parameters', parameters)
    # add input and output options
    if isinstance(input_file, list):
        # list of input files are given, use input_list_opt
        node.add_input_list_opt("--{}".format(input_file_opt), input_file)
    else:
        # assume just a single file
        node.add_input_opt("--{}".format(input_file_opt), input_file)
    node.new_output_file_opt(analysis_seg, output_file_extension,
                             "--output-file")
    # add node to workflow
    if add_to_workflow:
    return node

def make_inference_prior_plot(workflow, config_file, output_dir,
                              name="plot_prior",
                              analysis_seg=None, tags=None):
    """Sets up the corner plot of the priors in the workflow.

    config_file: pycbc.workflow.File
        The WorkflowConfigParser parasable inference configuration file..
        The directory to store result plots and files.
    name: str
        to use, and the section to read for additional arguments to pass to
        the executable. Default is ``plot_prior``.

    Returns
    -------
    pycbc.workflow.FileList
        A list of the output files.
    """
    node = make_inference_plot(workflow, config_file, output_dir,
                               name, analysis_seg=analysis_seg, tags=tags,
                               input_file_opt='config-file',
                               add_to_workflow=True)
    return node.output_files

def create_posterior_files(workflow, samples_files, output_dir,
                           parameters=None, name="extract_posterior",
                           analysis_seg=None, tags=None):
    """Sets up job to create posterior files from some given samples files.

        The workflow instance we are populating
    samples_files : str or list of str
        One or more files to extract the posterior samples from.
    name: str, optional
        the executable. Default is ``extract_posterior``.

    Returns
    -------
        A list of output files.
    """
        tags = []
    # we need to do this because Executable will automatically add any
    extract_posterior_exe = Executable(workflow.cp, name,
                                       ifos=workflow.ifos,
                                       out_dir=output_dir)
    node = extract_posterior_exe.create_node()
    if not isinstance(samples_files, list):
        samples_files = [samples_files]
    node.add_input_list_opt("--input-file", samples_files)
    node.new_output_file_opt(analysis_seg, ".hdf", "--output-file", tags=tags)

def create_fits_file(workflow, inference_file, output_dir,
                     name="create_fits_file",
                     analysis_seg=None, tags=None):
    """Sets up job to create fits files from some given samples files.

    inference_file: pycbc.workflow.File
        The file with posterior samples.
        the executable. Default is ``create_fits_file``.

    Returns
    -------
    """
        tags = []
    create_fits_exe = Executable(workflow.cp, name,
                                 ifos=workflow.ifos,
                                 out_dir=output_dir)
    node = create_fits_exe.create_node()
    node.add_input_opt("--input-file", inference_file)
    node.new_output_file_opt(analysis_seg, ".fits", "--output-file", tags=tags)

def make_inference_skymap(workflow, fits_file, output_dir,
                          name="plot_skymap", analysis_seg=None,
                          tags=None):
    """Sets up the skymap plot.

    fits_file: pycbc.workflow.File
        The fits file with the sky location.
        the executable. Default is ``plot_skymap``.

    Returns
    -------
        A list of result and output files.
    """
    node = make_inference_plot(workflow, fits_file, output_dir,

def make_inference_summary_table(workflow, inference_file, output_dir,
                                 parameters=None, print_metadata=None,
                                 name="table_summary",
                                 analysis_seg=None, tags=None):
    """Sets up the html table summarizing parameter estimates.

    parameters : list or str
        A list or string of parameters to generate the table for. If a string
        is provided, separate parameters should be space or new-line separated.
    print_metadata : list or str
        A list or string of metadata parameters to print. Syntax is the same
        as for ``parameters``.
        the executable. Default is ``table_summary``.

    Returns
    -------
    """
    # we'll use make_inference_plot even though this isn't a plot; the
    # setup is the same, we just change the file extension
    node = make_inference_plot(workflow, inference_file, output_dir,
                               output_file_extension='.html',
                               add_to_workflow=False)
    # now add the parameters and print metadata options; these are pulled
    # from separate sections in the workflow config file, which is why we
    # add them separately here
    if print_metadata is not None:
        node.add_opt("--print-metadata", _params_for_pegasus(print_metadata))

def make_inference_posterior_plot(workflow, inference_file, output_dir,
                                  parameters=None, plot_prior_from_file=None,
                                  name="plot_posterior",
                                  analysis_seg=None, tags=None):
    """Sets up the corner plot of the posteriors in the workflow.

        The parameters to plot.
    plot_prior_from_file : str, optional
        Plot the prior from the given config file on the 1D marginal plots.
        the executable. Default is ``plot_posterior``.

    Returns
    -------
    """
    # create the node, but delay adding it to the workflow so we can add
    # the prior file if it is requested
    if plot_prior_from_file is not None:
        node.add_input_opt('--plot-prior', plot_prior_from_file)
    # now add the node to workflow

def make_inference_samples_plot(workflow, inference_file, output_dir,
                                name="plot_samples",
                                analysis_seg=None, tags=None):
    """Sets up a plot of the samples versus iteration (for MCMC samplers).

        the executable. Default is ``plot_samples``.

    Returns
    -------
    """

def make_inference_acceptance_rate_plot(workflow, inference_file, output_dir,
                                        name="plot_acceptance_rate",
                                        analysis_seg=None, tags=None):
    """Sets up a plot of the acceptance rate (for MCMC samplers).

        the executable. Default is ``plot_acceptance_rate``.

    Returns
    -------
    """

def make_inference_plot_mcmc_history(workflow, inference_file, output_dir,
                                     name="plot_mcmc_history",
                                     analysis_seg=None, tags=None):
    """Sets up a plot showing the checkpoint history of an MCMC sampler.

        the executable. Default is ``plot_mcmc_history``.

    Returns
    -------
    """

def make_inference_dynesty_run_plot(workflow, inference_file, output_dir,
                                    name="plot_dynesty_run",
                                    analysis_seg=None, tags=None):
    """Sets up a debugging plot for the dynesty run (for Dynesty sampler).

        the executable. Default is ``plot_dynesty_run``.

    Returns
    -------
    """

def make_inference_dynesty_trace_plot(workflow, inference_file, output_dir,
                                      name="plot_dynesty_traceplot",
                                      analysis_seg=None, tags=None):
    """Sets up a trace plot for the dynesty run (for Dynesty sampler).

        the executable. Default is ``plot_dynesty_traceplot``.

    Returns
    -------
    """

def make_inference_pp_table(workflow, posterior_files, output_dir,
                            parameters=None, injection_samples_map=None,
                            name="pp_table_summary",
                            analysis_seg=None, tags=None):
    """Performs a PP, writing results to an html table.

    workflow : pycbc.workflow.Workflow
    posterior_files : pycbc.workflow.core.FileList
        List of files with posteriors of injections.
    output_dir : str
    parameters : list or str, optional
    injection_samples_map : (list of) str, optional
        Map between injection parameters and parameters in the posterior file.
        Format is ``INJECTION_PARAM:SAMPLES_PARAM``.
    name : str, optional
    analysis_segs : ligo.segments.Segment, optional
    tags : list, optional

    Returns
    -------
    """
    node = make_inference_plot(workflow, posterior_files, output_dir,
    # add the parameters and inj/samples map
    if injection_samples_map is not None:
        node.add_opt("--injection-samples-map",
                     _params_for_pegasus(injection_samples_map))

def make_inference_pp_plot(workflow, posterior_files, output_dir,
                           parameters=None, injection_samples_map=None,
                           name="plot_pp",
    """Sets up a pp plot in the workflow.

    posterior_files: pycbc.workflow.core.FileList
        the executable. Default is ``plot_pp``.

    Returns
    -------
    """

def make_inference_inj_recovery_plot(workflow, posterior_files, output_dir,
                                     parameter, injection_samples_map=None,
                                     name="inj_recovery",
    """Sets up the recovered versus injected parameter plot in the workflow.

    inference_files: pycbc.workflow.core.FileList
    parameter : str
        The parameter to plot.
        the executable. Default is ``inj_recovery``.

    Returns
    -------
    """
    # arguments are the same as plot_pp, so just call that with the
    # different executable name
    return make_inference_pp_plot(
        workflow, posterior_files, output_dir, parameters=parameter,
        injection_samples_map=injection_samples_map,
        name=name, analysis_seg=analysis_seg, tags=tags)

def get_plot_group(cp, section_tag):
    """Gets plotting groups from ``[workflow-section_tag]``."""
    group_prefix = "plot-group-"
    # parameters for the summary plots
    plot_groups = {}
    opts = [opt for opt in cp.options("workflow-{}".format(section_tag))
            if opt.startswith(group_prefix)]
    for opt in opts:
        group = opt.replace(group_prefix, "").replace("-", "_")
        plot_groups[group] = cp.get_opt_tag("workflow", opt, section_tag)
    return plot_groups

def get_diagnostic_plots(workflow):
    """Determines what diagnostic plots to create based on workflow.

    The plots to create are based on what executable's are specified in the
    workflow's config file. A list of strings is returned giving the diagnostic
    plots to create. This list may contain:

    * ``samples``: For MCMC samplers, a plot of the sample chains as a function
      of iteration. This will be created if ``plot_samples`` is in the
      executables section.
    * ``acceptance_rate``: For MCMC samplers, a plot of the acceptance rate.
      This will be created if ``plot_acceptance_rate`` is in the executables
      section.

    Returns
    -------
    list :
        List of names of diagnostic plots.
    """
    diagnostics = []
    if "plot_samples" in workflow.cp.options("executables"):
        diagnostics.append('samples')
    if "plot_acceptance_rate" in workflow.cp.options("executables"):
        diagnostics.append('acceptance_rate')
    if "plot_mcmc_history" in workflow.cp.options("executables"):
        diagnostics.append('mcmc_history')
    if "plot_dynesty_run" in workflow.cp.options("executables"):
        diagnostics.append('dynesty_run')
    if "plot_dynesty_traceplot" in workflow.cp.options("executables"):
        diagnostics.append('dynesty_traceplot')
    return diagnostics

def make_diagnostic_plots(workflow, diagnostics, samples_file, label, rdir,
    """Makes diagnostic plots.

    Diagnostic plots are sampler-specific plots the provide information on
    how the sampler performed. All diagnostic plots use the output file
    produced by ``pycbc_inference`` as their input. Diagnostic plots are added
    to the results directory ``rdir/NAME`` where ``NAME`` is the name of the
    diagnostic given in ``diagnostics``.

    workflow : pycbc.workflow.core.Workflow
        The workflow to add the plotting jobs to.
    diagnostics : list of str
        The names of the diagnostic plots to create. See
        :py:func:`get_diagnostic_plots` for recognized names.
    samples_file : (list of) pycbc.workflow.File
        One or more samples files with which to create the diagnostic plots.
        If a list of files is provided, a diagnostic plot for each file will
    label : str
        Event label for the diagnostic plots.
    rdir : pycbc.results.layout.SectionNumber
        Results directory layout.
    tags : list of str, optional
        Additional tags to add to the file names.

    Returns
    -------
    dict :
        Dictionary of diagnostic name -> list of files giving the plots that
        will be created.
    """
        tags = []
    out = {}
    if not isinstance(samples_file, list):
        samples_file = [samples_file]
    if 'samples' in diagnostics:
        # files for samples summary subsection
        base = "samples/{}".format(label)
        samples_plots = []
        for kk, sf in enumerate(samples_file):
            samples_plots += make_inference_samples_plot(
                workflow, sf, rdir[base],
                analysis_seg=workflow.analysis_time,
                tags=tags+[label, str(kk)])
        out['samples'] = samples_plots
        layout.group_layout(rdir[base], samples_plots)

    if 'acceptance_rate' in diagnostics:
        # files for samples acceptance_rate subsection
        base = "acceptance_rate/{}".format(label)
        acceptance_plots = []
            acceptance_plots += make_inference_acceptance_rate_plot(
        out['acceptance_rate'] = acceptance_plots
        layout.single_layout(rdir[base], acceptance_plots)

    if 'mcmc_history' in diagnostics:
        # files for samples mcmc history subsection
        base = "mcmc_history/{}".format(label)
        history_plots = []
            history_plots += make_inference_plot_mcmc_history(
        out['mcmc_history'] = history_plots
        layout.single_layout(rdir[base], history_plots)

    if 'dynesty_run' in diagnostics:
        # files for dynesty run subsection
        base = "dynesty_run/{}".format(label)
        dynesty_run_plots = []
            dynesty_run_plots += make_inference_dynesty_run_plot(
        out['dynesty_run'] = dynesty_run_plots
        layout.single_layout(rdir[base], dynesty_run_plots)

    if 'dynesty_traceplot' in diagnostics:
        # files for samples dynesty tyrace plots subsection
        base = "dynesty_traceplot/{}".format(label)
        dynesty_trace_plots = []
            dynesty_trace_plots += make_inference_dynesty_trace_plot(
        out['dynesty_traceplot'] = dynesty_trace_plots
        layout.single_layout(rdir[base], dynesty_trace_plots)

    return out

def make_posterior_workflow(workflow, samples_files, config_file, label,
                            rdir, posterior_file_dir='posterior_files',
    """Adds jobs to a workflow that make a posterior file and subsequent plots.

    A posterior file is first created from the given samples file(s). The
    settings for extracting the posterior are set by the
    ``[extract_posterior]`` section. If that section has a ``parameters``
    argument, then the parameters in the posterior file (and for use in all
    subsequent plotting) will be whatever that option is set to. Otherwise,
    the parameters in the posterior file will be whatever is common to
    all of the given samples file(s).

    Except for prior plots (which use the given inference config file), all
    subsequent jobs use the posterior file. The following are created:

    * **Summary table**: an html table created using the ``table_summary``
      executable. The parameters to print in the table are retrieved from the
      ``table-params`` option in the ``[workflow-summary_table]`` section.
      Metadata may also be printed by adding a ``print-metadata`` option to
      that section.
    * **Summary posterior plots**: a collection of posterior plots to include
      in the summary page, after the summary table. The parameters to plot
      are read from ``[workflow-summary_plots]``. Parameters should be grouped
      together by providing
      ``plot-group-NAME = PARAM1[:LABEL1] PARAM2[:LABEL2]`` in that section,
      where ``NAME`` is a unique name for each group. One posterior plot will
      be created for each plot group. For clarity, only one or two parameters
      should be plotted in each summary group, but this is not enforced.
      Settings for the plotting executable are read from the
      ``plot_posterior_summary`` section; likewise, the executable used
      is read from ``plot_posterior_summary`` in the
      ``[executables]`` section.
    * **Sky maps**: if *both* ``create_fits_file`` and ``plot_skymap``
      are listed in the ``[executables]`` section, then a ``.fits`` file and
      sky map plot will be produced. The sky map plot will be included in
      the summary plots. You must be running in a python 3 environment to
      create these.
    * **Prior plots**: plots of the prior will be created using the
      ``plot_prior`` executable. By default, all of the variable
      parameters will be plotted. The prior plots are added to
      ``priors/LALBEL/`` in the results directory, where ``LABEL`` is the
      given ``label``.
    * **Posterior plots**: additional posterior plots are created using the
      ``plot_posterior`` executable. The parameters to plot are
      read from ``[workflow-plot_params]`` section. As with the summary
      posterior plots, parameters are grouped together by providing
      ``plot-group-NAME`` options in that section. A posterior plot will be
      created for each group, and added to the ``posteriors/LABEL/`` directory.
      Plot settings are read from the ``[plot_posterior]`` section; this
      is kept separate from the posterior summary so that different settings
      can be used. For example, you may want to make a density plot for the
      summary plots, but a scatter plot colored by SNR for the posterior plots.

    samples_file : pycbc.workflow.core.FileList
        List of samples files to combine into a single posterior file.
    config_file : pycbc.worfkow.File
        The inference configuration file used to generate the samples file(s).
        This is needed to make plots of the prior.
    label : str
        Unique label for the plots. Used in file names.
        The results directory to save the plots to.
    posterior_file_dir : str, optional
        The name of the directory to save the posterior file to. Default is
        ``posterior_files``.

    Returns
    -------
    posterior_file : pycbc.workflow.File
        The posterior file that was created.
    summary_files : list
        List of files to go on the summary results page.
    prior_plots : list
        List of prior plots that will be created. These will be saved to
        ``priors/LABEL/`` in the resuls directory, where ``LABEL`` is the
        provided label.
    posterior_plots : list
        List of posterior plots that will be created. These will be saved to
        ``posteriors/LABEL/`` in the results directory.
    """
    # the list of plots to go in the summary
    summary_files = []

        tags = []

    analysis_seg = workflow.analysis_time

    # figure out what parameters user wants to plot from workflow configuration
    summary_plot_params = get_plot_group(workflow.cp, 'summary_plots')
    # parameters to plot in large corner plots
    plot_params = get_plot_group(workflow.cp, 'plot_params')
    # get parameters for the summary tables
    table_params = workflow.cp.get_opt_tag('workflow', 'table-params',
                                           'summary_table')
    # get any metadata that should be printed
    if workflow.cp.has_option('workflow-summary_table', 'print-metadata'):
        table_metadata = workflow.cp.get_opt_tag('workflow', 'print-metadata',
                                                 'summary_table')
    else:
        table_metadata = None

    # figure out if we are making a skymap
    make_skymap = ("create_fits_file" in workflow.cp.options("executables") and
                   "plot_skymap" in workflow.cp.options("executables"))

    # make node for running extract samples
    posterior_file = create_posterior_files(
        workflow, samples_files, posterior_file_dir,
        analysis_seg=analysis_seg, tags=tags+[label])[0]

    # summary table
    summary_files += (make_inference_summary_table(
        workflow, posterior_file, rdir.base,
        parameters=table_params, print_metadata=table_metadata,
        analysis_seg=analysis_seg,
        tags=tags+[label]),)

    # summary posteriors
    summary_plots = []
    for group, params in summary_plot_params.items():
        summary_plots += make_inference_posterior_plot(
            workflow, posterior_file, rdir.base,
            name='plot_posterior_summary',
            parameters=params, plot_prior_from_file=config_file,
            analysis_seg=analysis_seg,
            tags=tags+[label, group])

    # sky map
    if make_skymap:
        # create the fits file
        fits_file = create_fits_file(
            workflow, posterior_file, rdir.base, analysis_seg=analysis_seg,
            tags=tags+[label])[0]
        # now plot the skymap
        skymap_plot = make_inference_skymap(
            workflow, fits_file, rdir.base, analysis_seg=analysis_seg,
            tags=tags+[label])
        summary_plots += skymap_plot

    summary_files += list(layout.grouper(summary_plots, 2))

    # files for priors summary section
    base = "priors/{}".format(label)
    prior_plots = make_inference_prior_plot(
        workflow, config_file, rdir[base],
        analysis_seg=workflow.analysis_time,
        tags=tags+[label])
    layout.single_layout(rdir[base], prior_plots)

    # files for posteriors summary subsection
    base = "posteriors/{}".format(label)
    posterior_plots = []
    for group, params in plot_params.items():
        posterior_plots += make_inference_posterior_plot(
            workflow, posterior_file, rdir[base],
    layout.single_layout(rdir[base], posterior_plots)

    return posterior_file, summary_files, prior_plots, posterior_plots

def _params_for_pegasus(parameters):
    """Escapes $ and escapes in parameters string for pegasus.

    Pegaus kickstart tries to do variable substitution if it sees a ``$``, and
    it will strip away back slashes. This can be problematic when trying to use
    LaTeX in parameter labels. This function adds escapes to all ``$`` and
    backslashes in a parameters argument, so the argument can be safely passed
    through pegasus-kickstart.

        The parameters argument to modify. If a list, the output will be
        converted to a space-separated string.
    """
    if isinstance(parameters, list):
        parameters = " ".join(parameters)
    return parameters.replace('\\', '\\\\').replace('$', '\$')

# Copyright (C) 2015  Ian Harry, Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
"""
This module is responsible for setting up the part of a pycbc workflow that
will generate the injection files to be used for assessing the workflow's
ability to detect predicted signals. (In ihope parlance, this sets up the
inspinj jobs). Full documentation for this module can be found here:
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/NOTYETCREATED.html
"""

from pycbc.workflow.core import FileList, make_analysis_dir, Node
from pycbc.workflow.core import Executable, resolve_url_to_file
from pycbc.workflow.jobsetup import (LalappsInspinjExecutable,
        LigolwCBCJitterSkylocExecutable, LigolwCBCAlignTotalSpinExecutable,
        PycbcDarkVsBrightInjectionsExecutable, LigolwAddExecutable,
        select_generic_executable)

def veto_injections(workflow, inj_file, veto_file, veto_name, out_dir, tags=None):

    node = Executable(workflow.cp, 'strip_injections', ifos=workflow.ifos,
                          out_dir=out_dir, tags=tags).create_node()
    node.add_input_opt('--injection-file', inj_file)
    node.add_opt('--ifos', ' '.join(workflow.ifos))

class PyCBCOptimalSNRExecutable(Executable):
    """Compute optimal SNR for injections"""

    def create_node(self, workflow, inj_file, precalc_psd_files, group_str):
        _, ext = os.path.splitext(inj_file.name)
        node.add_input_opt('--input-file', inj_file)
        node.add_opt('--injection-fraction-range', group_str)
        node.add_input_list_opt('--time-varying-psds', precalc_psd_files)
        node.new_output_file_opt(workflow.analysis_time, ext,
                                 '--output-file')

def compute_inj_optimal_snr(workflow, inj_file, precalc_psd_files, out_dir,
    "Set up a job for computing optimal SNRs of a sim_inspiral file."
        tags = []

    try:
        factor = int(workflow.cp.get_opt_tags('workflow-optimal-snr',
                                              'parallelization-factor',
                                              tags))
    except ConfigParser.Error:
        factor = 1

    if factor == 1:
        # parallelization factor not given - default to single optimal snr job
        opt_snr_exe = PyCBCOptimalSNRExecutable(workflow.cp, 'optimal_snr',
                                                ifos=workflow.ifos,
                                                out_dir=out_dir, tags=tags)
        node = opt_snr_exe.create_node(workflow, inj_file,
                                       precalc_psd_files, '0/1')

        return node.output_files[0]

    opt_snr_split_files = []
                                                out_dir=out_dir,
                                                tags=tags + [str(i)])
        opt_snr_exe.update_current_retention_level(
            Executable.INTERMEDIATE_PRODUCT)
        node = opt_snr_exe.create_node(workflow, inj_file, precalc_psd_files,
                                       group_str)
        opt_snr_split_files += [node.output_files[0]]

    llwadd_exe = LigolwAddExecutable(workflow.cp, 'optimal_snr_merge',
                                     ifos=workflow.ifos, out_dir=out_dir,
                                     tags=tags)
    llwadd_exe.update_current_retention_level(Executable.MERGED_TRIGGERS)
    merge_node = llwadd_exe.create_node(workflow.analysis_time,
                                        opt_snr_split_files,
                                        use_tmp_subdirs=False)
    workflow += merge_node

    return merge_node.output_files[0]

def cut_distant_injections(workflow, inj_file, out_dir, tags=None):
    "Set up a job for removing injections that are too distant to be seen"
        tags = []

    node = Executable(workflow.cp, 'inj_cut', ifos=workflow.ifos,
    node.add_input_opt('--input', inj_file)

def inj_to_hdf(workflow, inj_file, out_dir, tags=None):
    """ Convert injection file to hdf format if not already one
    """
    _, ext = os.path.splitext(inj_file.name)
    if ext == '.hdf':
        return inj_file

        tags = []

    node = Executable(workflow.cp, 'inj2hdf', ifos=workflow.ifos,
    node.new_output_file_opt(workflow.analysis_time, '.hdf', '--output-file')

def setup_injection_workflow(workflow, output_dir=None,
                             inj_section_name='injections', exttrig_file=None,
                             tags =None):
    """
    This function is the gateway for setting up injection-generation jobs in a
    workflow. It should be possible for this function to support a number
    of different ways/codes that could be used for doing this, however as this
    will presumably stay as a single call to a single code (which need not be
    inspinj) there are currently no subfunctions in this moudle.

        The Workflow instance that the coincidence jobs will be added to.
    output_dir : path
        The directory in which injection files will be stored.
    inj_section_name : string (optional, default='injections')
        The string that corresponds to the option describing the exe location
        in the [executables] section of the .ini file and that corresponds to
        the section (and sub-sections) giving the options that will be given to
        the code at run time.
    tags : list of strings (optional, default = [])
        A list of the tagging strings that will be used for all jobs created
        by this call to the workflow. This will be used in output names.

    Returns
    --------
    inj_files : pycbc.workflow.core.FileList
        The list of injection files created by this call.
    inj_tags : list of strings
        The tag corresponding to each injection file and used to uniquely
        identify them. The FileList class contains functions to search
        based on tags.
    """
        tags = []
    logging.info("Entering injection module.")
    make_analysis_dir(output_dir)

    # Get full analysis segment for output file naming
    full_segment = workflow.analysis_time
    ifos = workflow.ifos

    # Identify which injections to do by presence of sub-sections in
    # the configuration file
    inj_tags = []
    inj_files = FileList([])

    for section in  workflow.cp.get_subsections(inj_section_name):
        inj_tag = section.upper()
        curr_tags = tags + [inj_tag]

        # Parse for options in ini file
        injection_method = workflow.cp.get_opt_tags("workflow-injections",
                                                    "injections-method",
                                                    curr_tags)

        if injection_method in ["IN_WORKFLOW", "AT_RUNTIME"]:
            exe = select_generic_executable(workflow, 'injections')
            inj_job = exe(workflow.cp, inj_section_name,
                          out_dir=output_dir, ifos='HL',
                          tags=curr_tags)
            node = inj_job.create_node(full_segment)
            if injection_method == "AT_RUNTIME":
                workflow.execute_node(node)
            else:
                workflow.add_node(node)
            inj_file = node.output_files[0]
            inj_files.append(inj_file)
        elif injection_method == "PREGENERATED":
            file_attrs = {
                'ifos': ['HL'],
                'segs': full_segment,
                'tags': curr_tags
            }
            injection_path = workflow.cp.get_opt_tags(
                "workflow-injections",
                "injections-pregenerated-file",
                curr_tags
            )
            curr_file = resolve_url_to_file(injection_path, attrs=file_attrs)
            inj_files.append(curr_file)
        elif injection_method in ["IN_COH_PTF_WORKFLOW", "AT_COH_PTF_RUNTIME"]:
            inj_job = LalappsInspinjExecutable(workflow.cp, inj_section_name,
                                               out_dir=output_dir, ifos=ifos,
                                               tags=curr_tags)
            node = inj_job.create_node(full_segment, exttrig_file)
            if injection_method == "AT_COH_PTF_RUNTIME":
            else:

            if workflow.cp.has_option("workflow-injections",
                                      "em-bright-only"):
                em_filter_job = PycbcDarkVsBrightInjectionsExecutable(
                                                 workflow.cp,
                                                 'em_bright_filter',
                                                 tags=curr_tags,
                                                 out_dir=output_dir,
                                                 ifos=ifos)
                node = em_filter_job.create_node(inj_file, full_segment,
                                                 curr_tags)
                if injection_method == "AT_COH_PTF_RUNTIME":
                    workflow.execute_node(node)
                else:
                    workflow.add_node(node)
                inj_file = node.output_files[0]

                                      "do-jitter-skyloc"):
                jitter_job = LigolwCBCJitterSkylocExecutable(workflow.cp,
                                                             'jitter_skyloc',
                                                             tags=curr_tags,
                                                             out_dir=output_dir,
                                                             ifos=ifos)
                node = jitter_job.create_node(inj_file, full_segment, curr_tags)
                else:

                                      "do-align-total-spin"):
                align_job = LigolwCBCAlignTotalSpinExecutable(workflow.cp,
                        'align_total_spin', tags=curr_tags, out_dir=output_dir,
                        ifos=ifos)
                node = align_job.create_node(inj_file, full_segment, curr_tags)

                else:

        else:
            err = "Injection method must be one of IN_WORKFLOW, "
            err += "AT_RUNTIME or PREGENERATED. Got %s." % (injection_method)

        inj_tags.append(inj_tag)

    logging.info("Leaving injection module.")
    return inj_files, inj_tags

#
#
#

#
#
#                                   Preamble
#
#

"""
This library code contains functions and classes that are used to set up
and add jobs/nodes to a pycbc workflow. For details about pycbc.workflow see:
"""

import math, os
import lal
import Pegasus.DAX3 as dax
from pycbc.workflow.core import Executable, File, FileList, Node

def int_gps_time_to_str(t):
    """Takes an integer GPS time, either given as int or lal.LIGOTimeGPS, and
    converts it to a string. If a LIGOTimeGPS with nonzero decimal part is
    given, raises a ValueError."""

    if isinstance(t, int):
        return str(t)
    elif isinstance(t, float):
        # Wouldn't this just work generically?
        int_t = int(t)
        if abs(t - int_t) > 0.:
            raise ValueError('Need an integer GPS time, got %s' % str(t))
        return str(int_t)
    elif isinstance(t, lal.LIGOTimeGPS):
        if t.gpsNanoSeconds == 0:
            return str(t.gpsSeconds)
        else:
    else:
        err_msg = "Didn't understand input type {}".format(type(t))

def select_tmpltbank_class(curr_exe):
    """ This function returns a class that is appropriate for setting up
    template bank jobs within workflow.

    curr_exe : string
        The name of the executable to be used for generating template banks.

    Returns
    --------
    exe_class : Sub-class of pycbc.workflow.core.Executable that holds utility
        functions appropriate for the given executable.  Instances of the class
        ('jobs') **must** have methods
        * job.create_node()
        and
        * job.get_valid_times(ifo, )
    """
    exe_to_class_map = {
        'pycbc_geom_nonspinbank'  : PyCBCTmpltbankExecutable,
        'pycbc_aligned_stoch_bank': PyCBCTmpltbankExecutable
    }
    try:
        return exe_to_class_map[curr_exe]
        raise NotImplementedError(
            "No job class exists for executable %s, exiting" % curr_exe)

def select_matchedfilter_class(curr_exe):
    matched-filtering jobs within workflow.

        The name of the matched filter executable to be used.

    Returns
    --------
        and
    """
        'pycbc_inspiral'          : PyCBCInspiralExecutable,
        'pycbc_inspiral_skymax'   : PyCBCInspiralExecutable,
        'pycbc_multi_inspiral'    : PyCBCMultiInspiralExecutable,
    }
    try:
        # also conceivable to introduce a default class??

def select_generic_executable(workflow, exe_tag):
    """ Returns a class that is appropriate for setting up jobs to run executables
    having specific tags in the workflow config.
    Executables should not be "specialized" jobs fitting into one of the
    select_XXX_class functions above, i.e. not a matched filter or template
    bank job, which require extra setup.

        The Workflow instance.

    exe_tag : string
        The name of the config section storing options for this executable and
        the option giving the executable path in the [executables] section.

    Returns
    --------
        ('jobs') **must** have a method job.create_node()
    """
    exe_path = workflow.cp.get("executables", exe_tag)
    exe_name = os.path.basename(exe_path)
        'ligolw_add'               : LigolwAddExecutable,
        'ligolw_cbc_sstinca'           : LigolwSSthincaExecutable,
        'pycbc_sqlite_simplify'    : PycbcSqliteSimplifyExecutable,
        'ligolw_cbc_cluster_coincs': SQLInOutExecutable,
        'ligolw_cbc_repop_coinc'   : SQLInOutExecutable,
        'repop_coinc_expfit'       : SQLInOutExecutable,
        'ligolw_cbc_dbinjfind'         : SQLInOutExecutable,
        'lalapps_inspinj'          : LalappsInspinjExecutable,
        'pycbc_create_injections'  : PyCBCCreateInjectionsExecutable,
        'pycbc_dark_vs_bright_injections' : PycbcDarkVsBrightInjectionsExecutable,
        'pycbc_timeslides'         : PycbcTimeslidesExecutable,
        'pycbc_compute_durations'  : ComputeDurationsExecutable,
        'pycbc_calculate_far'      : PycbcCalculateFarExecutable,
        "pycbc_run_sqlite"         : SQLInOutExecutable,
        # FIXME: We may end up with more than one class for using ligolw_sqlite
        #        How to deal with this?
        "ligolw_sqlite"            : ExtractToXMLExecutable,
        "pycbc_inspinjfind"        : InspinjfindExecutable,
        "pycbc_pickle_horizon_distances" : PycbcPickleHorizonDistsExecutable,
        "pycbc_combine_likelihood" : PycbcCombineLikelihoodExecutable,
        "pycbc_gen_ranking_data"   : PycbcGenerateRankingDataExecutable,
        "pycbc_calculate_likelihood"     : PycbcCalculateLikelihoodExecutable,
        "gstlal_inspiral_marginalize_likelihood"      : GstlalMarginalizeLikelihoodExecutable,
        "pycbc_compute_far_from_snr_chisq_histograms" : GstlalFarfromsnrchisqhistExecutable,
        "gstlal_inspiral_plot_sensitivity"            : GstlalPlotSensitivity,
        "gstlal_inspiral_plot_background" : GstlalPlotBackground,
        "gstlal_inspiral_plotsummary"     : GstlalPlotSummary,
        "gstlal_inspiral_summary_page"    : GstlalSummaryPage,
        "pycbc_condition_strain"         : PycbcConditionStrainExecutable
    }
    try:
        return exe_to_class_map[exe_name]
        # Should we try some sort of default class??
            "No job class exists for executable %s, exiting" % exe_name)

def sngl_ifo_job_setup(workflow, ifo, out_files, curr_exe_job, science_segs,
                       datafind_outs, parents=None,
                       link_job_instance=None, allow_overlap=True,
                       compatibility_mode=True):
    """ This function sets up a set of single ifo jobs. A basic overview of how this
    works is as follows:

    * (1) Identify the length of data that each job needs to read in, and what
      part of that data the job is valid for.
    * START LOOPING OVER SCIENCE SEGMENTS
    * (2) Identify how many jobs are needed (if any) to cover the given science
      segment and the time shift between jobs. If no jobs continue.
    * START LOOPING OVER JOBS
    * (3) Identify the time that the given job should produce valid output (ie.
      inspiral triggers) over.
    * (4) Identify the data range that the job will need to read in to produce
      the aforementioned valid output.
    * (5) Identify all parents/inputs of the job.
    * (6) Add the job to the workflow
    * END LOOPING OVER JOBS
    * END LOOPING OVER SCIENCE SEGMENTS

        An instance of the Workflow class that manages the constructed workflow.
        The name of the ifo to set up the jobs for
    out_files : pycbc.workflow.core.FileList
        The FileList containing the list of jobs. Jobs will be appended
        to this list, and it does not need to be empty when supplied.
    curr_exe_job : Job
        An instanced of the Job class that has a get_valid times method.
    science_segs : ligo.segments.segmentlist
        The list of times that the jobs should cover
    datafind_outs : pycbc.workflow.core.FileList
        The file list containing the datafind files.
    parents : pycbc.workflow.core.FileList (optional, kwarg, default=None)
        The FileList containing the list of jobs that are parents to
        the one being set up.
    link_job_instance : Job instance (optional),
        Coordinate the valid times with another Executable.
    allow_overlap : boolean (optional, kwarg, default = True)
        If this is set the times that jobs are valid for will be allowed to
        overlap. This may be desired for template banks which may have some
        overlap in the times they cover. This may not be desired for inspiral
        jobs, where you probably want triggers recorded by jobs to not overlap
        at all.
    compatibility_mode : boolean (optional,  kwarg, default = False)
        If given the jobs will be tiled in the same method as used in inspiral
        hipe. This requires that link_job_instance is also given. If not given
        workflow's methods are used.

    Returns
    --------
        A list of the files that will be generated by this step in the
        workflow.
    """
    if compatibility_mode and not link_job_instance:
        errMsg = "Compability mode requires a link_job_instance."

    ########### (1) ############
    # Get the times that can be analysed and needed data lengths
    data_length, valid_chunk, valid_length = identify_needed_data(curr_exe_job,
                                           link_job_instance=link_job_instance)

    # Loop over science segments and set up jobs
    for curr_seg in science_segs:
        ########### (2) ############
        # Initialize the class that identifies how many jobs are needed and the
        # shift between them.
        segmenter = JobSegmenter(data_length, valid_chunk, valid_length,
                                 curr_seg, curr_exe_job,
                                 compatibility_mode=compatibility_mode)

        for job_num in range(segmenter.num_jobs):
            ############## (3) #############
            # Figure out over what times this job will be valid for
            job_valid_seg = segmenter.get_valid_times_for_job(job_num,
                                                   allow_overlap=allow_overlap)

            ############## (4) #############
            # Get the data that this job should read in
            job_data_seg = segmenter.get_data_times_for_job(job_num)

            ############# (5) ############
            # Identify parents/inputs to the job
            if parents:
                # Find the set of files with the best overlap
                curr_parent = parents.find_outputs_in_range(ifo, job_valid_seg,
                                                            useSplitLists=True)
                if not curr_parent:
                    err_string = ("No parent jobs found overlapping %d to %d."
                                  %(job_valid_seg[0], job_valid_seg[1]))
                    err_string += "\nThis is a bad error! Contact a developer."
                    raise ValueError(err_string)
            else:
                curr_parent = [None]

            curr_dfouts = None
            if datafind_outs:
                curr_dfouts = datafind_outs.find_all_output_in_range(ifo,
                                              job_data_seg, useSplitLists=True)
                if not curr_dfouts:
                    err_str = ("No datafind jobs found overlapping %d to %d."
                                %(job_data_seg[0],job_data_seg[1]))
                    err_str += "\nThis shouldn't happen. Contact a developer."
                    raise ValueError(err_str)

            ############## (6) #############
            # Make node and add to workflow

            # Note if I have more than one curr_parent I need to make more than
            # one job. If there are no curr_parents it is set to [None] and I
            # make a single job. This catches the case of a split template bank
            # where I run a number of jobs to cover a single range of time.

            # Sort parent jobs to ensure predictable order
            sorted_parents = sorted(curr_parent,
                                    key=lambda fobj: fobj.tagged_description)
            for pnum, parent in enumerate(sorted_parents):
                if len(curr_parent) != 1:
                    tag = ["JOB%d" %(pnum,)]
                else:
                    tag = []
                # To ensure output file uniqueness I add a tag
                # We should generate unique names automatically, but it is a
                # pain until we can set the output names for all Executables
                node = curr_exe_job.create_node(job_data_seg, job_valid_seg,
                                                parent=parent,
                                                dfParents=curr_dfouts,
                                                tags=tag)
                curr_out_files = node.output_files
                # FIXME: Here we remove PSD files if they are coming through.
                #        This should be done in a better way. On to-do list.
                curr_out_files = [i for i in curr_out_files if 'PSD_FILE'\
                                                                 not in i.tags]
                out_files += curr_out_files

    return out_files

def multi_ifo_coherent_job_setup(workflow, out_files, curr_exe_job,
                                 science_segs, datafind_outs, output_dir,
                                 parents=None, slide_dict=None, tags=None):
    """
    Method for setting up coherent inspiral jobs.
    """
        tags = []
    data_seg, job_valid_seg = curr_exe_job.get_valid_times()
    curr_out_files = FileList([])
    if 'IPN' in datafind_outs[-1].description \
            and 'bank_veto_bank' in datafind_outs[-2].description:
        ipn_sky_points = datafind_outs[-1]
        bank_veto = datafind_outs[-2]
        frame_files = datafind_outs[:-2]
    else:
        ipn_sky_points = None
        bank_veto = datafind_outs[-1]
        frame_files = datafind_outs[:-1]

    split_bank_counter = 0

    if curr_exe_job.injection_file is None:
        for split_bank in parents:
            tag = list(tags)
            tag.append(split_bank.tag_str)
            node = curr_exe_job.create_node(data_seg, job_valid_seg,
                    parent=split_bank, dfParents=frame_files,
                    bankVetoBank=bank_veto, ipn_file=ipn_sky_points,
                    slide=slide_dict, tags=tag)
            workflow.add_node(node)
            split_bank_counter += 1
            curr_out_files.extend(node.output_files)
    else:
        for inj_file in curr_exe_job.injection_file:
            for split_bank in parents:
                tag = list(tags)
                tag.append(inj_file.tag_str)
                tag.append(split_bank.tag_str)
                node = curr_exe_job.create_node(data_seg, job_valid_seg,
                        parent=split_bank, inj_file=inj_file, tags=tag,
                        dfParents=frame_files, bankVetoBank=bank_veto,
                        ipn_file=ipn_sky_points)
                split_bank_counter += 1
                curr_out_files.extend(node.output_files)

    # FIXME: Here we remove PSD files if they are coming
    #        through. This should be done in a better way. On
    #        to-do list.
    curr_out_files = [i for i in curr_out_files if 'PSD_FILE'\
                      not in i.tags]
    out_files += curr_out_files

def identify_needed_data(curr_exe_job, link_job_instance=None):
    """ This function will identify the length of data that a specific executable
    needs to analyse and what part of that data is valid (ie. inspiral doesn't
    analyse the first or last 64+8s of data it reads in).

    In addition you can supply a second job instance to "link" to, which will
    ensure that the two jobs will have a one-to-one correspondence (ie. one
    template bank per one matched-filter job) and the corresponding jobs will
    be "valid" at the same times.

        An instance of the Job class that has a get_valid times method.
        Coordinate the valid times with another executable.

    Returns
    --------
    dataLength : float
        The amount of data (in seconds) that each instance of the job must read
        in.
    valid_chunk : glue.segment.segment
        The times within dataLength for which that jobs output **can** be
        valid (ie. for inspiral this is (72, dataLength-72) as, for a standard
        setup the inspiral job cannot look for triggers in the first 72 or
        last 72 seconds of data read in.)
    valid_length : float
        The maximum length of data each job can be valid for. If not using
        link_job_instance this is abs(valid_segment), but can be smaller than
        that if the linked job only analyses a small amount of data (for e.g.).
    """
    # Set up the condorJob class for the current executable
    data_lengths, valid_chunks = curr_exe_job.get_valid_times()

    # Begin by getting analysis start and end, and start and end of time
    # that the output file is valid for
    valid_lengths = [abs(valid_chunk) for valid_chunk in valid_chunks]

    if link_job_instance:
        # FIXME: Should we remove this, after testing is complete??
        # EURGHH! What we are trying to do here is, if this option is given,
        # line up the template bank and inspiral jobs so that there is one
        # template bank for each inspiral job. This is proving a little messy
        # and probably still isn't perfect.

        # What data does the linked exe use?
        link_data_length, link_valid_chunk = link_job_instance.get_valid_times()
        if len(link_data_length) > 1 or len(valid_lengths) > 1:
            raise ValueError('Linking job instances for tiling is not supported'
                             ' between jobs that allow variable tile size')

        # What data is lost at start of both jobs? Take the maximum.
        start_data_loss = max(valid_chunks[0][0], link_valid_chunk[0][0])
        # What data is lost at end of both jobs? Take the maximum.
        end_data_loss = max(data_lengths[0] - valid_chunks[0][1],\
                            link_data_length[0] - link_valid_chunk[0][1])
        # Calculate valid_segments for both jobs based on the combined data
        # loss.

        valid_chunks[0] = segments.segment(start_data_loss, \
                                       data_lengths[0] - end_data_loss)
        link_valid_chunk = segments.segment(start_data_loss, \
                                       link_data_length[0] - end_data_loss)

        # The maximum valid length should be the minimum of the two
        link_valid_length = abs(link_valid_chunk)

        # Which one is now longer? Use this is valid_length
        if link_valid_length < valid_lengths[0]:
            valid_lengths[0] = link_valid_length

    return data_lengths, valid_chunks, valid_lengths

class JobSegmenter(object):
    """ This class is used when running sngl_ifo_job_setup to determine what times
    should be analysed be each job and what data is needed.
    """

    def __init__(self, data_lengths, valid_chunks, valid_lengths, curr_seg,
                 curr_exe_class, compatibility_mode = False):
        """ Initialize class. """
        self.exe_class = curr_exe_class
        self.curr_seg = curr_seg
        self.curr_seg_length = float(abs(curr_seg))

        self.data_length, self.valid_chunk, self.valid_length = \
                      self.pick_tile_size(self.curr_seg_length, data_lengths,
                                                  valid_chunks, valid_lengths)

        self.data_chunk = segments.segment([0, self.data_length])
        self.data_loss = self.data_length - abs(self.valid_chunk)

        if self.data_loss < 0:
            raise ValueError("pycbc.workflow.jobsetup needs fixing! Please contact a developer")

        if self.curr_seg_length < self.data_length:
            self.num_jobs = 0
            return

        # How many jobs do we need
        self.num_jobs = int( math.ceil( (self.curr_seg_length \
                                - self.data_loss) / float(self.valid_length) ))

        self.compatibility_mode = compatibility_mode
        if compatibility_mode and (self.valid_length != abs(self.valid_chunk)):
            errMsg = "In compatibility mode the template bank and matched-"
            errMsg += "filter jobs must read in the same amount of data."
            print(self.valid_length, self.valid_chunk)
        elif compatibility_mode and len(data_lengths) > 1:
            raise ValueError("Cannot enable compatibility mode tiling with "
                             "variable tile size")
        elif compatibility_mode:
            # What is the incremental shift between jobs
            self.job_time_shift = self.valid_length
        elif self.curr_seg_length == self.data_length:
            # If the segment length is identical to the data length then I
            # will have exactly 1 job!
            self.job_time_shift = 0
        else:
            self.job_time_shift = (self.curr_seg_length - self.data_length) / \
                                   float(self.num_jobs - 1)

    def pick_tile_size(self, seg_size, data_lengths, valid_chunks, valid_lengths):
        """ Choose job tiles size based on science segment length """

        if len(valid_lengths) == 1:
            return data_lengths[0], valid_chunks[0], valid_lengths[0]
        else:
            # Pick the tile size that is closest to 1/3 of the science segment
            target_size = seg_size / 3
            pick, pick_diff = 0, abs(valid_lengths[0] - target_size)
            for i, size in enumerate(valid_lengths):
                if abs(size - target_size) < pick_diff:
                    pick, pick_diff  = i, abs(size - target_size)
            return data_lengths[pick], valid_chunks[pick], valid_lengths[pick]

    def get_valid_times_for_job(self, num_job, allow_overlap=True):
        """ Get the times for which this job is valid. """
        if self.compatibility_mode:
            return self.get_valid_times_for_job_legacy(num_job)
        else:
            return self.get_valid_times_for_job_workflow(num_job,

    def get_valid_times_for_job_workflow(self, num_job, allow_overlap=True):
        """ Get the times for which the job num_job will be valid, using workflow's
        method.
        """
        # small factor of 0.0001 to avoid float round offs causing us to
        # miss a second at end of segments.
        shift_dur = self.curr_seg[0] + int(self.job_time_shift * num_job\
                                           + 0.0001)
        job_valid_seg = self.valid_chunk.shift(shift_dur)
        # If we need to recalculate the valid times to avoid overlap
        if not allow_overlap:
            data_per_job = (self.curr_seg_length - self.data_loss) / \
                           float(self.num_jobs)
            lower_boundary = num_job*data_per_job + \
                                 self.valid_chunk[0] + self.curr_seg[0]
            upper_boundary = data_per_job + lower_boundary
            # NOTE: Convert to int after calculating both boundaries
            # small factor of 0.0001 to avoid float round offs causing us to
            # miss a second at end of segments.
            lower_boundary = int(lower_boundary)
            upper_boundary = int(upper_boundary + 0.0001)
            if lower_boundary < job_valid_seg[0] or \
                    upper_boundary > job_valid_seg[1]:
                err_msg = ("Workflow is attempting to generate output "
                          "from a job at times where it is not valid.")
            job_valid_seg = segments.segment([lower_boundary,
                                              upper_boundary])
        return job_valid_seg

    def get_valid_times_for_job_legacy(self, num_job):
        """ Get the times for which the job num_job will be valid, using the method
        use in inspiral hipe.
        """
        # All of this should be integers, so no rounding factors needed.
        shift_dur = self.curr_seg[0] + int(self.job_time_shift * num_job)

        # If this is the last job, push the end back
        if num_job == (self.num_jobs - 1):
            dataPushBack = self.data_length - self.valid_chunk[1]
            job_valid_seg = segments.segment(job_valid_seg[0],
                                               self.curr_seg[1] - dataPushBack)

    def get_data_times_for_job(self, num_job):
        """ Get the data that this job will read in. """
            job_data_seg =  self.get_data_times_for_job_legacy(num_job)
        else:
            job_data_seg =  self.get_data_times_for_job_workflow(num_job)

        # Sanity check that all data is used
        if num_job == 0:
            if job_data_seg[0] != self.curr_seg[0]:
                err= "Job is not using data from the start of the "
                err += "science segment. It should be using all data."
            if job_data_seg[1] != self.curr_seg[1]:
                err = "Job is not using data from the end of the "

        if hasattr(self.exe_class, 'zero_pad_data_extend'):
            job_data_seg = self.exe_class.zero_pad_data_extend(job_data_seg,
                                                                 self.curr_seg)

        return job_data_seg

    def get_data_times_for_job_workflow(self, num_job):
        """ Get the data that this job will need to read in. """
                                           + 0.0001)
        job_data_seg = self.data_chunk.shift(shift_dur)

    def get_data_times_for_job_legacy(self, num_job):
        # Should all be integers, so no rounding needed

            dataPushBack = job_data_seg[1] - self.curr_seg[1]
            assert dataPushBack >= 0
            job_data_seg = segments.segment(job_data_seg[0] - dataPushBack,
                                                 self.curr_seg[1])
            assert (abs(job_data_seg) == self.data_length)

class PyCBCInspiralExecutable(Executable):
    """ The class used to create jobs for pycbc_inspiral Executable. """

    file_input_options = ['--gating-file']
    time_dependent_options = ['--channel-name']

    def __init__(self, cp, exe_name, ifo=None, out_dir=None,
                 injection_file=None, tags=None, reuse_executable=False):
            tags = []
        super(PyCBCInspiralExecutable, self).__init__(
            cp,
            exe_name,
            None,
            ifo,
            out_dir,
            tags=tags,
            reuse_executable=reuse_executable
        )
        self.set_memory(2000)
        self.ext = '.hdf'

        self.num_threads = 1
        if self.get_opt('processing-scheme') is not None:
            stxt = self.get_opt('processing-scheme')
            if len(stxt.split(':')) > 1:
                self.num_threads = stxt.split(':')[1]

    def create_node(self, data_seg, valid_seg, parent=None, dfParents=None, tags=None):
            tags = []
        node = Node(self, valid_seg=valid_seg)
        if not self.has_opt('pad-data'):
            raise ValueError("The option pad-data is a required option of "
                             "%s. Please check the ini file." % self.name)
        pad_data = int(self.get_opt('pad-data'))

        # set remaining options flags
        node.add_opt('--gps-start-time',
                     int_gps_time_to_str(data_seg[0] + pad_data))
        node.add_opt('--gps-end-time',
                     int_gps_time_to_str(data_seg[1] - pad_data))
        node.add_opt('--trig-start-time', int_gps_time_to_str(valid_seg[0]))
        node.add_opt('--trig-end-time', int_gps_time_to_str(valid_seg[1]))
        node.add_profile('condor', 'request_cpus', self.num_threads)

            node.add_input_opt('--injection-file', self.injection_file)

        # set the input and output files
        fil = node.new_output_file_opt(valid_seg, self.ext, '--output', tags=tags,
                         store_file=self.retain_files, use_tmp_subdirs=True)
        fil.add_metadata('data_seg', data_seg)
        node.add_input_opt('--bank-file', parent)
        if dfParents is not None:
            node.add_input_list_opt('--frame-files', dfParents)

    def get_valid_times(self):
        """ Determine possible dimensions of needed input and valid output
        """

        if self.cp.has_option('workflow-matchedfilter',
                                                      'min-analysis-segments'):
            min_analysis_segs = int(self.cp.get('workflow-matchedfilter',
                                                'min-analysis-segments'))
        else:
            min_analysis_segs = 0

                                                      'max-analysis-segments'):
            max_analysis_segs = int(self.cp.get('workflow-matchedfilter',
                                                'max-analysis-segments'))
        else:
            # Choose ridiculously large default value
            max_analysis_segs = 1000

        if self.cp.has_option('workflow-matchedfilter', 'min-analysis-length'):
            min_analysis_length = int(self.cp.get('workflow-matchedfilter',
                                                  'min-analysis-length'))
        else:
            min_analysis_length = 0

        if self.cp.has_option('workflow-matchedfilter', 'max-analysis-length'):
            max_analysis_length = int(self.cp.get('workflow-matchedfilter',
                                                  'max-analysis-length'))
        else:
            # Choose a ridiculously large default value
            max_analysis_length = 100000

        segment_length = int(self.get_opt('segment-length'))
        pad_data = 0
        if self.has_opt('pad-data'):
            pad_data += int(self.get_opt('pad-data'))

        # NOTE: Currently the tapered data is ignored as it is short and
        #       will lie within the segment start/end pad. This means that
        #       the tapered data *will* be used for PSD estimation (but this
        #       effect should be small). It will also be in the data segments
        #       used for SNR generation (when in the middle of a data segment
        #       where zero-padding is not being used) but the templates should
        #       not be long enough to use this data assuming segment start/end
        #       pad take normal values. When using zero-padding this data will
        #       be used for SNR generation.

        #if self.has_opt('taper-data'):
        #    pad_data += int(self.get_opt( 'taper-data' ))
        if self.has_opt('allow-zero-padding'):
            self.zero_padding=True
        else:
            self.zero_padding=False

        start_pad = int(self.get_opt( 'segment-start-pad'))
        end_pad = int(self.get_opt('segment-end-pad'))

        seg_ranges = range(min_analysis_segs, max_analysis_segs + 1)
        data_lengths = []
        valid_regions = []
        for nsegs in seg_ranges:
            analysis_length = (segment_length - start_pad - end_pad) * nsegs
            if not self.zero_padding:
                data_length = analysis_length + pad_data * 2 \
                              + start_pad + end_pad
                start = pad_data + start_pad
                end = data_length - pad_data - end_pad
            else:
                data_length = analysis_length + pad_data * 2
                start = pad_data
                end = data_length - pad_data
            if data_length > max_analysis_length: continue
            if data_length < min_analysis_length: continue
            data_lengths += [data_length]
            valid_regions += [segments.segment(start, end)]
        # If min_analysis_length is given, ensure that it is added as an option
        # for job analysis length.
        if min_analysis_length:
            data_length = min_analysis_length
            else:
            if end > start:
                data_lengths += [data_length]
                valid_regions += [segments.segment(start, end)]
        return data_lengths, valid_regions

    def zero_pad_data_extend(self, job_data_seg, curr_seg):
        """When using zero padding, *all* data is analysable, but the setup
        functions must include the padding data where it is available so that
        we are not zero-padding in the middle of science segments. This
        function takes a job_data_seg, that is chosen for a particular node
        and extends it with segment-start-pad and segment-end-pad if that
        data is available.
        """
        if self.zero_padding is False:
            return job_data_seg
        else:
            start_pad = int(self.get_opt( 'segment-start-pad'))
            end_pad = int(self.get_opt('segment-end-pad'))
            new_data_start = max(curr_seg[0], job_data_seg[0] - start_pad)
            new_data_end = min(curr_seg[1], job_data_seg[1] + end_pad)
            new_data_seg = segments.segment([new_data_start, new_data_end])
            return new_data_seg

class PyCBCMultiInspiralExecutable(Executable):
    """
    The class responsible for setting up jobs for the
    pycbc_multi_inspiral executable.
    """
    def __init__(self, cp, name, universe=None, ifo=None, injection_file=None,
                 gate_files=None, out_dir=None, tags=None):
            tags = []
        super(PyCBCMultiInspiralExecutable, self).__init__(cp, name, universe,
                ifo, out_dir=out_dir, tags=tags)
        self.data_seg = segments.segment(int(cp.get('workflow', 'start-time')),
                                         int(cp.get('workflow', 'end-time')))

    def create_node(self, data_seg, valid_seg, parent=None, inj_file=None,
                    dfParents=None, bankVetoBank=None, ipn_file=None,
                    slide=None, tags=None):
            tags = []

        if not dfParents:
            raise ValueError("%s must be supplied with frame or cache files"
                              % self.name)

        # If doing single IFO search, make sure slides are disabled
        if len(self.ifo_list) < 2 and \
                (node.get_opt('--do-short-slides') is not None or \
                 node.get_opt('--short-slide-offset') is not None):
            raise ValueError("Cannot run with time slides in a single IFO "
                             "configuration! Please edit your configuration "
                             "file accordingly.")

        # Set instuments
        node.add_opt("--instruments", " ".join(self.ifo_list))

        pad_data = self.get_opt('pad-data')
        if pad_data is None:

        # Feed in bank_veto_bank.xml
        if self.cp.has_option('inspiral', 'do-bank-veto'):
            if not bankVetoBank:
                raise ValueError("%s must be given a bank veto file if the "
                                 "argument 'do-bank-veto' is given"
                                 % self.name)
            node.add_input_opt('--bank-veto-templates', bankVetoBank)

        # Set time options
        node.add_opt('--gps-start-time', data_seg[0] + int(pad_data))
        node.add_opt('--gps-end-time', data_seg[1] - int(pad_data))
        node.add_opt('--trig-start-time', valid_seg[0])
        node.add_opt('--trig-end-time', valid_seg[1])

        # Set the input and output files
        node.new_output_file_opt(data_seg, '.hdf', '--output',
                                 tags=tags, store_file=self.retain_files)
        node.add_input_opt('--bank-file', parent, )

        # TODO: isn't there a cleaner way of doing this?
            node.add_arg('--frame-cache %s' % \
                         " ".join([":".join([frameCache.ifo, frameCache.name])\
                                   for frameCache in dfParents]))
            for frameCache in dfParents:
                node._add_input(frameCache)
            #node.add_input_list_opt('--frame-cache', dfParents)

        if ipn_file is not None:
            node.add_input_opt('--sky-positions-file', ipn_file)

        if inj_file is not None:
            if self.get_opt('--do-short-slides') is not None or \
                    self.get_opt('--short-slide-offset') is not None:
                raise ValueError("Cannot run with short slides in an "
                                 "injection job. Please edit your "
                                 "configuration file accordingly.")
            node.add_input_opt('--injection-file', inj_file)

        if slide is not None:
            for ifo in self.ifo_list:
                node.add_opt('--%s-slide-segment' % ifo.lower(), slide[ifo])

        # Channels
        channel_names = {}
        for ifo in self.ifo_list:
            channel_names[ifo] = self.cp.get_opt_tags(
                               "workflow", "%s-channel-name" % ifo.lower(), "")
        channel_names_str = \
            " ".join([val for key, val in channel_names.items()])
        node.add_opt("--channel-name", channel_names_str)

        if self.has_opt("segment-start-pad"):
            pad_data = int(self.get_opt("pad-data"))
            start_pad = int(self.get_opt("segment-start-pad"))
            end_pad = int(self.get_opt("segment-end-pad"))
            valid_start = self.data_seg[0] + pad_data + start_pad
            valid_end = self.data_seg[1] - pad_data - end_pad
        elif self.has_opt('analyse-segment-end'):
            safety = 1
            deadtime = int(self.get_opt('segment-length')) / 2
            spec_len = int(self.get_opt('inverse-spec-length')) / 2
            valid_start = (self.data_seg[0] + deadtime - spec_len + pad_data -
                           safety)
            valid_end = self.data_seg[1] - spec_len - pad_data - safety
        else:
            overlap = int(self.get_opt('segment-length')) / 4
            valid_start = self.data_seg[0] + overlap + pad_data
            valid_end = self.data_seg[1] - overlap - pad_data

        return self.data_seg, segments.segment(valid_start, valid_end)

class PyCBCTmpltbankExecutable(Executable):
    """ The class used to create jobs for pycbc_geom_nonspin_bank Executable and
    any other Executables using the same command line option groups.
    """

                 tags=None, write_psd=False, psd_files=None):
            tags = []
        super(PyCBCTmpltbankExecutable, self).__init__(cp, exe_name, 'vanilla', ifo, out_dir, tags=tags)
        self.write_psd = write_psd
        self.psd_files = psd_files

            tags = []

            raise ValueError("%s must be supplied with data file(s)"

        # set the remaining option flags

        # Add the PSD file if needed
        if self.write_psd:
            node.new_output_file_opt(valid_seg, '.txt', '--psd-output',
                                     tags=tags+['PSD_FILE'], store_file=self.retain_files)
        node.new_output_file_opt(valid_seg, '.xml.gz', '--output-file',
        node.add_input_list_opt('--frame-files', dfParents)

    def create_nodata_node(self, valid_seg, tags=None):
        """ A simplified version of create_node that creates a node that does not
        need to read in data.

        valid_seg : glue.segment
            The segment over which to declare the node valid. Usually this
            would be the duration of the analysis.

        Returns
        --------
        node : pycbc.workflow.core.Node
            The instance corresponding to the created node.
        """
            tags = []

        # Set the output file

                                 store_file=self.retain_files)

        if self.psd_files is not None:
            should_add = False

            # If any of the ifos for this job are in the set
            # of ifos for which a static psd was provided.
                for psd_file in self.psd_files:
                    if ifo in psd_file.ifo_list:
                        should_add = True

            if should_add:
                node.add_input_opt('--psd-file', psd_file)

        pad_data = int(self.get_opt( 'pad-data'))
        analysis_length = int(self.cp.get('workflow-tmpltbank', 'analysis-length'))

        #FIXME this should not be hard coded
        data_length = analysis_length + pad_data * 2
        start = pad_data
        end = data_length - pad_data
        return [data_length], [segments.segment(start, end)]

class LigoLWCombineSegsExecutable(Executable):
    """ This class is used to create nodes for the ligolw_combine_segments
    Executable
    """

    # Always want to keep the segments
    current_retention_level = Executable.FINAL_RESULT
    def create_node(self, valid_seg, veto_files, segment_name):
        node.add_opt('--segment-name', segment_name)
        for fil in veto_files:
            node.add_input_arg(fil)
        node.new_output_file_opt(valid_seg, '.xml', '--output',

class LigolwAddExecutable(Executable):
    """ The class used to create nodes for the ligolw_add Executable. """

    current_retention_level = Executable.INTERMEDIATE_PRODUCT
    def __init__(self, *args, **kwargs):
        super(LigolwAddExecutable, self).__init__(*args, **kwargs)

    def create_node(self, jobSegment, input_files, output=None,
                    use_tmp_subdirs=True, tags=None):
            tags = []

        # Very few options to ligolw_add, all input files are given as a long
        # argument list. If this becomes unwieldy we could dump all these files
        # to a cache file and read that in. ALL INPUT FILES MUST BE LISTED AS
        # INPUTS (with .add_input_opt_file) IF THIS IS DONE THOUGH!
        for fil in input_files:

        # Currently we set the output file using the name of *all* active ifos,
        # even if one or more of these ifos is not active within jobSegment.
        # In ihope, these files were named with only participating ifos. Not
        # sure this is worth doing, can can be done with replacing self.ifo
        # here if desired
        if output:
            node.add_output_opt('--output', output)
        else:
            node.new_output_file_opt(jobSegment, '.xml.gz', '--output',
                                    tags=tags, store_file=self.retain_files,
                                    use_tmp_subdirs=use_tmp_subdirs)

class LigolwSSthincaExecutable(Executable):
    """ The class responsible for making jobs for ligolw_sstinca. """

    def __init__(self, cp, exe_name, universe=None, ifo=None, out_dir=None,
                 dqVetoName=None, tags=None):
            tags = []
        super(LigolwSSthincaExecutable, self).__init__(cp, exe_name, universe, ifo, out_dir, tags=tags)
        if dqVetoName:
            self.add_opt("--vetoes-name", dqVetoName)

    def create_node(self, jobSegment, coincSegment, inputFile, tags=None,
                    write_likelihood=False):
            tags = []
        node.add_input_arg(inputFile)

        # Add the start/end times
        segString = ""
        if coincSegment[0]:
            segString += str(coincSegment[0])
        segString += ":"
        if coincSegment[1]:
            segString += str(coincSegment[1])

        node.add_opt('--coinc-end-time-segment', segString)

        node.new_output_file_opt(jobSegment, '.xml.gz', '--output-file',
                        tags=self.tags+tags, store_file=self.retain_files,
                        use_tmp_subdirs=True)

        if write_likelihood:
            node.new_output_file_opt(jobSegment, '.xml.gz',
                        '--likelihood-output-file',
                        tags=['DIST_STATS']+self.tags+tags,
                        store_file=self.retain_files)

class PycbcSqliteSimplifyExecutable(Executable):
    """ The class responsible for making jobs for pycbc_sqlite_simplify. """

    def __init__(self, cp, exe_name, universe=None, ifo=None, out_dir=None, tags=None):
            tags = []
        super(PycbcSqliteSimplifyExecutable, self).__init__(cp, exe_name, universe, ifo, out_dir, tags=tags)

    def create_node(self, job_segment, inputFiles, injFile=None,
                    injString=None, workflow=None):
        if injFile and not injString:
            raise ValueError("injString needed if injFile supplied.")
        # Need to check if I am dealing with a single split job
        extra_tags = []
        if len(inputFiles) == 1:
            for tag in inputFiles[0].tags:
                if tag.startswith('JOB'):
                    extra_tags.append(tag)
        # If the number of input files is large, split this up
        num_inputs = len(inputFiles)
        if num_inputs > 20 and workflow is not None:
            reduced_inputs = FileList([])
            count = 0
            testing = 0
            curr_node = Node(self)
            # hack the retention level ..
            if self.global_retention_threshold > \
                                 Executable.INTERMEDIATE_PRODUCT:
                curr_store_file = False
            else:
                curr_store_file = True
            for i, file in enumerate(inputFiles):
                curr_node.add_input_arg(file)
                if ( (not (i % 20)) and (i != 0) ) or ((i+1) == num_inputs):
                    count_tag = ['SPLIT%d' %(count)]
                    curr_node.new_output_file_opt(job_segment, '.sqlite',
                                    '--output-file',
                                     tags=self.tags+extra_tags+count_tag,
                                     store_file=curr_store_file)
                    workflow.add_node(curr_node)
                    reduced_inputs.append(curr_node.output_file)
                    testing += len(curr_node._inputs)
                    curr_node = Node(self)
                    count+=1
            inputFiles = reduced_inputs
        elif num_inputs > 20:
            err_msg = "Please provide the workflow keyword to the "
            err_msg += "pycbc_sqlite_simplify create node function if "
            err_msg += "supplying a large number of inputs. This allows us "
            err_msg += "to parallelize the operation and speedup the workflow."
            logging.warn(err_msg)

        for file in inputFiles:
            node.add_input_arg(file)
        if injFile:
            node.add_input_opt("--injection-file", injFile)
            node.add_opt("--simulation-tag", injString)
        node.new_output_file_opt(job_segment, '.sqlite', '--output-file',
                        tags=self.tags+extra_tags, store_file=self.retain_files)

class SQLInOutExecutable(Executable):
    """
    The class responsible for making jobs for SQL codes taking one input and
    one output.
    """
    current_retention_level=Executable.ALL_TRIGGERS
            tags = []
        super(SQLInOutExecutable, self).__init__(cp, exe_name, universe, ifo, out_dir, tags=tags)

    def create_node(self, job_segment, input_file):
        # Need to follow through the split job tag if present
        for tag in input_file.tags:
            if tag.startswith('JOB'):
                extra_tags.append(tag)

        node.add_input_opt('--input', input_file)
        node.new_output_file_opt(job_segment, '.sqlite', '--output',

class PycbcCalculateFarExecutable(SQLInOutExecutable):
    """
    The class responsible for making jobs for the FAR calculation code. This
    only raises the default retention level
    """
    current_retention_level=Executable.FINAL_RESULT

class ExtractToXMLExecutable(Executable):
    """
    This class is responsible for running ligolw_sqlite jobs that will take an
    SQL file and dump it back to XML.
    """
                 tags=None):
            tags = []
        Executable.__init__(self, cp, exe_name, universe, ifo, out_dir,
                                  tags=tags)
        node.add_input_opt('--database', input_file)
        node.new_output_file_opt(job_segment, '.xml', '--extract',
                                 tags=self.tags, store_file=self.retain_files)

class ComputeDurationsExecutable(SQLInOutExecutable):
    """
    The class responsible for making jobs for pycbc_compute_durations.
    """
    def create_node(self, job_segment, input_file, summary_xml_file):
        node.add_input_opt('--segment-file', summary_xml_file)

class InspinjfindExecutable(Executable):
    """
    The class responsible for running jobs with pycbc_inspinjfind
    """
            tags = []
        node.add_input_opt('--input-file', input_file)
        node.new_output_file_opt(job_segment, '.xml', '--output-file',

class PycbcSplitInspinjExecutable(Executable):
    """
    The class responsible for running the pycbc_split_inspinj executable
    """
    def __init__(self, cp, exe_name, num_splits, universe=None, ifo=None,
                 out_dir=None):
        super(PycbcSplitInspinjExecutable, self).__init__(cp, exe_name,
                universe, ifo, out_dir, tags=[])
        self.num_splits = int(num_splits)
    def create_node(self, parent, tags=None):
            tags = []

        node.add_input_opt('--input-file', parent)

        if parent.name.endswith("gz"):
            ext = ".xml.gz"
        else:
            ext = ".xml"

        out_files = FileList([])
        for i in range(self.num_splits):
            curr_tag = 'split%d' % i
            curr_tags = parent.tags + [curr_tag]
            job_tag = parent.description + "_" + self.name.upper()
            out_file = File(parent.ifo_list, job_tag, parent.segment,
                            extension=ext, directory=self.out_dir,
                            tags=curr_tags, store_file=self.retain_files)
            out_files.append(out_file)

        node.add_output_list_opt('--output-files', out_files)

class PycbcPickleHorizonDistsExecutable(Executable):
    """
    The class responsible for running the pycbc_pickle_horizon_distances
    executable which is part 1 of 4 of the gstlal_inspiral_calc_likelihood
    functionality
    """
    # FIXME: This class will soon be removed when gstlal post-proc is updated
            tags = []
    def create_node(self, job_segment, trigger_files):
        for file in trigger_files:
        node.new_output_file_opt(job_segment, '.pickle', '--output-file',

class PycbcCombineLikelihoodExecutable(Executable):
    """
    The class responsible for running the pycbc_combine_likelihood
    executable which is part 2 of 4 of the gstlal_inspiral_calc_likelihood
    """
            tags = []
    def create_node(self, job_segment, likelihood_files, horizon_dist_file):
        node.add_input_list_opt('--likelihood-urls', likelihood_files)
        node.add_input_opt('--horizon-dist-file', horizon_dist_file)
        node.new_output_file_opt(job_segment, '.xml.gz', '--output-file',

class PycbcGenerateRankingDataExecutable(Executable):
    """
    The class responsible for running the pycbc_gen_ranking_data
    executable which is part 3 of 4 of the gstlal_inspiral_calc_likelihood
    """
            tags = []
        self.set_num_cpus(4)
        self.set_memory('8000')
    def create_node(self, job_segment, likelihood_file, horizon_dist_file):
        node.add_input_opt('--likelihood-file', likelihood_file)

class PycbcCalculateLikelihoodExecutable(Executable):
    """
    The class responsible for running the pycbc_calculate_likelihood
    executable which is part 4 of 4 of the gstlal_inspiral_calc_likelihood
    """
            tags = []
    def create_node(self, job_segment, trigger_file, likelihood_file,
                    horizon_dist_file):
        for tag in trigger_file.tags:

        node.add_input_opt('--trigger-file', trigger_file)
                                 tags=self.tags + extra_tags,

class GstlalMarginalizeLikelihoodExecutable(Executable):
    """
    The class responsible for running the gstlal marginalize_likelihood jobs
    """
            tags = []
        node.add_input_arg(input_file)
        node.new_output_file_opt(job_segment, '.xml.gz', '--output',

class GstlalFarfromsnrchisqhistExecutable(Executable):
    """
    The class responsible for running the gstlal far from chisq hist jobs
    """
            tags = []
    def create_node(self, job_segment, non_inj_db, marg_input_file,
                   inj_database=None, write_background_bins=False):
        node.add_input_opt("--non-injection-db", non_inj_db)
        if inj_database is not None:
            node.add_input_opt("--input-database", inj_database)
        node.add_input_opt("--background-bins-file", marg_input_file)
        node.new_output_file_opt(job_segment, '.sqlite', '--output-database',
        # FIXME: Not supported yet
        if write_background_bins:
            node.new_output_file_opt(job_segment, '.xml.gz',
                                  "--background-bins-out-file",
                                  tags=["POSTMARG"] + self.tags,
                                  store_file=self.retain_files)

class GstlalPlotSensitivity(Executable):
    """
    The class responsible for running gstlal_plot_sensitivity
    """
            tags = []
        self.set_memory('4000')
    def create_node(self, non_inj_db, injection_dbs):
        node.add_input_opt("--zero-lag-database", non_inj_db)
        for file in injection_dbs:

class GstlalPlotSummary(Executable):
    """
    The class responsible for running gstlal_plot_summary
    """
            tags = []
        node.add_input_arg(non_inj_db)

class GstlalPlotBackground(Executable):
    """
    The class responsible for running gstlal_plot_background
    """
            tags = []
    def create_node(self, non_inj_db, likelihood_file):
        node.add_input_opt("--database", non_inj_db)
        node.add_input_arg(likelihood_file)

class GstlalSummaryPage(Executable):
    """
    The class responsible for running gstlal_inspiral_summary_page
    """
            tags = []
    def create_and_add_node(self, workflow, parent_nodes):
        # FIXME: As the parents of this job (the plotting scripts) do not track
        # the location of their output files, we must set explicit parent-child
        # relations the "old-fashioned" way here. Possibly this is how the AEI
        # DB stuff could work?
        workflow.add_node(node)
        for parent in parent_nodes:
            dep = dax.Dependency(parent=parent._dax_node, child=node._dax_node)
            workflow._adag.addDependency(dep)

class PyCBCCreateInjectionsExecutable(Executable):
    """
    The class used to create jobs for the lalapps_inspinj Executable.
    """
    file_input_options = ['--config-files']
    def create_node(self, segment, tags=None):
            tags = []
        node.new_output_file_opt(segment, '.hdf', '--output-file',
                                 store_file=self.retain_files, tags=tags)
        node.add_opt('--gps-start-time', int_gps_time_to_str(segment[0]))
        node.add_opt('--gps-end-time', int_gps_time_to_str(segment[1]))

class LalappsInspinjExecutable(Executable):
    """
    """
    def create_node(self, segment, exttrig_file=None, tags=None):
            tags = []

        curr_tags = self.tags + tags
        # This allows the desired number of injections to be given explicitly
        # in the config file. Used for coh_PTF as segment length is unknown
        # before run time.
        if self.get_opt('write-compress') is not None:
            ext = '.xml.gz'
        else:
            ext = '.xml'

        # Check if these injections are using trigger information to choose
        # sky positions for the simulated signals
        if (self.get_opt('l-distr') == 'exttrig' and exttrig_file is not None \
                and 'trigger' in exttrig_file.description):
            # Use an XML file containing trigger information
            triggered = True
            node.add_input_opt('--exttrig-file', exttrig_file)
        elif (self.get_opt('l-distr') == 'ipn' and exttrig_file is not None \
                and 'IPN' in exttrig_file.description):
            # Use an IPN sky points file
            node.add_input_opt('--ipn-file', exttrig_file)
        elif (self.get_opt('l-distr') != 'exttrig') \
                and (self.get_opt('l-distr') != 'ipn' and not \
                     self.has_opt('ipn-file')):
            # Use no trigger information for generating injections
            triggered = False
        else:
            err_msg = "The argument 'l-distr' passed to the "
            err_msg += "%s job has the value " % self.tagged_name
            err_msg += "'%s' but you have not " % self.get_opt('l-distr')
            err_msg += "provided the corresponding ExtTrig or IPN file. "
            err_msg += "Please check your configuration files and try again."

        if triggered:
            num_injs = int(self.cp.get_opt_tags('workflow-injections',
                                                'num-injs', curr_tags))
            inj_tspace = float(segment[1] - segment[0]) / num_injs
            node.add_opt('--time-interval', inj_tspace)
            node.add_opt('--time-step', inj_tspace)

        node.new_output_file_opt(segment, ext, '--output',

class PycbcDarkVsBrightInjectionsExecutable(Executable):
    """
    The clase used to create jobs for the pycbc_dark_vs_bright_injections Executable.
    """
    def __init__(self, cp, exe_name, universe=None, ifos=None, out_dir=None,
            tags = []
        Executable.__init__(self, cp, exe_name, universe, ifos, out_dir,
                            tags=tags)
        self.out_dir = out_dir
        self.exe_name = exe_name

    def create_node(self, parent, segment, tags=None):
            tags = []
        if not parent:
            raise ValueError("Must provide an input file.")

        # Standard injection file produced by lalapps_inspinj
        # becomes the input here
        node.add_input_opt('-i', parent)
        if self.has_opt('write-compress'):
        else:
        # The output are two files:
        # 1) the list of potentially EM bright injections
        tag=['POTENTIALLY_BRIGHT']
        node.new_output_file_opt(segment, ext, '--output-bright',
                                 store_file=self.retain_files, tags=tags+tag)
        # 2) the list of EM dim injections
        tag=['DIM_ONLY']
        node.new_output_file_opt(segment,
                                 ext, '--output-dim',

class LigolwCBCJitterSkylocExecutable(Executable):
    """
    The class used to create jobs for the ligolw_cbc_skyloc_jitter executable.
    """
            tags = []

            tags = []

        output_file = File(parent.ifo_list, self.exe_name,
                           segment, extension='.xml', store_file=True,
                           directory=self.out_dir, tags=tags)
        node.add_output_opt('--output-file', output_file)

class LigolwCBCAlignTotalSpinExecutable(Executable):
    """
    """
            tags = []

            tags = []

        output_file = File(parent.ifo_list, self.exe_name, segment,
                           extension='.xml', store_file=self.retain_files,
        node.add_input_arg(parent)

class PycbcTimeslidesExecutable(Executable):
    """
    The class used to create jobs for the pycbc_timeslides Executable.
    """
    def create_node(self, segment):

        node.new_output_file_opt(segment, '.xml.gz', '--output-files',

class PycbcSplitBankExecutable(Executable):
    """ The class responsible for creating jobs for pycbc_hdf5_splitbank. """

    extension = '.hdf'
    def __init__(self, cp, exe_name, num_banks,
                 ifo=None, out_dir=None, universe=None):
        super(PycbcSplitBankExecutable, self).__init__(cp, exe_name, universe,
                ifo, out_dir, tags=[])
        self.num_banks = int(num_banks)

    def create_node(self, bank, tags=None):
        """
        Set up a CondorDagmanNode class to run splitbank code

        bank : pycbc.workflow.core.File
            The File containing the template bank to be split

        Returns
        --------
            The node to run the job
        """
            tags = []
        node.add_input_opt('--bank-file', bank)

        # Get the output (taken from inspiral.py)
        for i in range( 0, self.num_banks):
            curr_tag = 'bank%d' %(i)
            # FIXME: What should the tags actually be? The job.tags values are
            #        currently ignored.
            curr_tags = bank.tags + [curr_tag] + tags
            job_tag = bank.description + "_" + self.name.upper()
            out_file = File(bank.ifo_list, job_tag, bank.segment,
                            extension=self.extension, directory=self.out_dir,
        node.add_output_list_opt('--output-filenames', out_files)

class PycbcSplitBankXmlExecutable(PycbcSplitBankExecutable):
    """ Subclass resonsible for creating jobs for pycbc_splitbank. """

    extension='.xml.gz'

class PycbcConditionStrainExecutable(Executable):
    """ The class responsible for creating jobs for pycbc_condition_strain. """

    def __init__(self, cp, exe_name, ifo=None, out_dir=None, universe=None,
            tags=None):
        super(PycbcConditionStrainExecutable, self).__init__(cp, exe_name, universe,
              ifo, out_dir, tags)

    def create_node(self, input_files, tags=None):
            tags = []
        start_time = self.cp.get("workflow", "start-time")
        end_time = self.cp.get("workflow", "end-time")
        node.add_opt('--gps-start-time', start_time)
        node.add_opt('--gps-end-time', end_time)
        node.add_input_list_opt('--frame-files', input_files)

        out_file = File(self.ifo, "gated",
                        segments.segment(int(start_time), int(end_time)),
                        directory=self.out_dir, store_file=self.retain_files,
                        extension=input_files[0].name.split('.', 1)[-1],
                        tags=tags)
        node.add_output_opt('--output-strain-file', out_file)

        out_gates_file = File(self.ifo, "output_gates",
                              segments.segment(int(start_time), int(end_time)),
                              directory=self.out_dir, extension='txt',
                              store_file=self.retain_files, tags=tags)
        node.add_output_opt('--output-gates-file', out_gates_file)

        return node, out_file

class PycbcCreateInjectionsExecutable(Executable):
    """ The class responsible for creating jobs
    for ``pycbc_create_injections``.
    """

                 universe=None, tags=None):
        super(PycbcCreateInjectionsExecutable, self).__init__(
                               cp, exe_name, universe, ifo, out_dir, tags)

    def create_node(self, config_file=None, seed=None, tags=None):
        """ Set up a CondorDagmanNode class to run ``pycbc_create_injections``.

        config_file : pycbc.workflow.core.File
            A ``pycbc.workflow.core.File`` for inference configuration file
            to be used with ``--config-files`` option.
        seed : int
            Seed to use for generating injections.
        tags : list
            A list of tags to include in filenames.

        Returns
        --------
            The node to run the job.
        """

        # default for tags is empty list
        tags = [] if tags is None else tags

        # get analysis start and end time
        analysis_time = segments.segment(int(start_time), int(end_time))

        # make node for running executable
        node.add_input_opt("--config-files", config_file)
        if seed:
            node.add_opt("--seed", seed)
        injection_file = node.new_output_file_opt(analysis_time,
                                                  ".hdf", "--output-file",
                                                  tags=tags)

        return node, injection_file

class PycbcInferenceExecutable(Executable):
    """ The class responsible for creating jobs for ``pycbc_inference``.
    """

    def __init__(self, cp, exe_name, ifos=None, out_dir=None,
        super(PycbcInferenceExecutable, self).__init__(cp, exe_name,
                                                       universe=universe,
                                                       ifos=ifos,
                                                       out_dir=out_dir,
                                                       tags=tags)

    def create_node(self, config_file, seed=None, tags=None,
                    analysis_time=None):
        """ Set up a CondorDagmanNode class to run ``pycbc_inference``.

        seed : int
            An ``int`` to be used with ``--seed`` option.
        tags : list

        Returns
        --------
        """
        # if analysis time not provided, try to get it from the config file
        if analysis_time is None:
            start_time = self.cp.get("workflow", "start-time")
            end_time = self.cp.get("workflow", "end-time")
            analysis_time = segments.segment(int(start_time), int(end_time))
        node.add_input_opt("--config-file", config_file)
        if seed is not None:
        inference_file = node.new_output_file_opt(analysis_time,
        if self.cp.has_option("pegasus_profile-inference",
                              "condor|+CheckpointSig"):
            ckpt_file_name = "{}.checkpoint".format(inference_file.name)
            ckpt_file = dax.File(ckpt_file_name)
            node._dax_node.uses(ckpt_file, link=dax.Link.OUTPUT,
                                register=False, transfer=False)

        return node, inference_file

#
#
#

#
#
#                                   Preamble
#
#

"""
This module is responsible for setting up the matched-filtering stage of
"""

import os, logging
from math import radians
from pycbc.workflow.core import FileList, make_analysis_dir
from pycbc.workflow.jobsetup import (select_matchedfilter_class,
        select_tmpltbank_class, sngl_ifo_job_setup,
        multi_ifo_coherent_job_setup)

def setup_matchedfltr_workflow(workflow, science_segs, datafind_outs,
                               tmplt_banks, output_dir=None,
                               injection_file=None, tags=None):
    '''
    This function aims to be the gateway for setting up a set of matched-filter
    jobs in a workflow. This function is intended to support multiple
    different ways/codes that could be used for doing this. For now the only
    supported sub-module is one that runs the matched-filtering by setting up
    a serious of matched-filtering jobs, from one executable, to create
    matched-filter triggers covering the full range of science times for which
    there is data and a template bank file.

    Workflow : pycbc.workflow.core.Workflow
        The workflow instance that the coincidence jobs will be added to.
        An FileList of the datafind files that are needed to obtain the
        data used in the analysis.
    tmplt_banks : pycbc.workflow.core.FileList
        An FileList of the template bank files that will serve as input
        in this stage.
        The directory in which output will be stored.
    injection_file : pycbc.workflow.core.File, optional (default=None)
        If given the file containing the simulation file to be sent to these
        jobs on the command line. If not given no file will be sent.
        by this call to the workflow. An example might be ['BNSINJECTIONS'] or
        ['NOINJECTIONANALYSIS']. This will be used in output names.

    Returns
    -------
    inspiral_outs : pycbc.workflow.core.FileList
        A list of output files written by this stage. This *will not* contain
        any intermediate products produced within this stage of the workflow.
        If you require access to any intermediate products produced at this
        stage you can call the various sub-functions directly.
    '''
        tags = []
    logging.info("Entering matched-filtering setup module.")

    # Parse for options in .ini file
    mfltrMethod = cp.get_opt_tags("workflow-matchedfilter", "matchedfilter-method",
                                  tags)

    # Could have a number of choices here
    if mfltrMethod == "WORKFLOW_INDEPENDENT_IFOS":
        logging.info("Adding matched-filter jobs to workflow.")
        if cp.has_option_tags("workflow-matchedfilter",
                              "matchedfilter-link-to-tmpltbank", tags):
            if not cp.has_option_tags("workflow-tmpltbank",
                              "tmpltbank-link-to-matchedfilter", tags):
                errMsg = "If using matchedfilter-link-to-tmpltbank, you should "
                errMsg += "also use tmpltbank-link-to-matchedfilter."
                logging.warn(errMsg)
            linkToTmpltbank = True
        else:
            linkToTmpltbank = False
                              "matchedfilter-compatibility-mode", tags):
            if not linkToTmpltbank:
                errMsg = "Compatibility mode requires that the "
                errMsg += "matchedfilter-link-to-tmpltbank option is also set."
                              "tmpltbank-compatibility-mode", tags):
                errMsg = "If using compatibility mode it must be set both in "
                errMsg += "the template bank and matched-filtering stages."
            compatibility_mode = True
        else:
            compatibility_mode = False

        inspiral_outs = setup_matchedfltr_dax_generated(workflow, science_segs,
                                      datafind_outs, tmplt_banks, output_dir,
                                      injection_file=injection_file,
                                      tags=tags,
                                      link_to_tmpltbank=linkToTmpltbank,
                                      compatibility_mode=compatibility_mode)
    elif mfltrMethod == "WORKFLOW_MULTIPLE_IFOS":
        inspiral_outs = setup_matchedfltr_dax_generated_multi(workflow,
                                      science_segs, datafind_outs, tmplt_banks,
                                      output_dir, injection_file=injection_file,
                                      tags=tags)
    else:
        errMsg = "Matched filter method not recognized. Must be one of "
        errMsg += "WORKFLOW_INDEPENDENT_IFOS (currently only one option)."

    logging.info("Leaving matched-filtering setup module.")
    return inspiral_outs

def setup_matchedfltr_dax_generated(workflow, science_segs, datafind_outs,
                                    tmplt_banks, output_dir,
                                    injection_file=None,
                                    tags=None, link_to_tmpltbank=False,
                                    compatibility_mode=False):
    '''
    Setup matched-filter jobs that are generated as part of the workflow.
    This
    module can support any matched-filter code that is similar in principle to
    lalapps_inspiral, but for new codes some additions are needed to define
    Executable and Job sub-classes (see jobutils.py).

    link_to_tmpltbank : boolean, optional (default=True)
        If this option is given, the job valid_times will be altered so that there
        will be one inspiral file for every template bank and they will cover the
        same time span. Note that this option must also be given during template
        bank generation to be meaningful.

    Returns
    -------
    '''
        tags = []
    # Need to get the exe to figure out what sections are analysed, what is
    # discarded etc. This should *not* be hardcoded, so using a new executable
    # will require a bit of effort here ....

    ifos = science_segs.keys()
    match_fltr_exe = os.path.basename(cp.get('executables','inspiral'))
    # Select the appropriate class
    exe_class = select_matchedfilter_class(match_fltr_exe)

    if link_to_tmpltbank:
        # Use this to ensure that inspiral and tmpltbank jobs overlap. This
        # means that there will be 1 inspiral job for every 1 tmpltbank and
        # the data read in by both will overlap as much as possible. (If you
        # ask the template bank jobs to use 2000s of data for PSD estimation
        # and the matched-filter jobs to use 4000s, you will end up with
        # twice as many matched-filter jobs that still use 4000s to estimate a
        # PSD but then only generate triggers in the 2000s of data that the
        # template bank jobs ran on.
        tmpltbank_exe = os.path.basename(cp.get('executables', 'tmpltbank'))
        link_exe_instance = select_tmpltbank_class(tmpltbank_exe)
    else:
        link_exe_instance = None

    # Set up class for holding the banks
    inspiral_outs = FileList([])

    # Matched-filtering is done independently for different ifos, but might not be!
    # If we want to use multi-detector matched-filtering or something similar to this
    # it would probably require a new module
        logging.info("Setting up matched-filtering for %s." %(ifo))
        job_instance = exe_class(workflow.cp, 'inspiral', ifo=ifo,
                                               out_dir=output_dir,
                                               injection_file=injection_file,
                                               tags=tags)
        if link_exe_instance:
            link_job_instance = link_exe_instance(cp, 'tmpltbank', ifo=ifo,
                                               out_dir=output_dir, tags=tags)
        else:
            link_job_instance = None

        sngl_ifo_job_setup(workflow, ifo, inspiral_outs, job_instance,
                           science_segs[ifo], datafind_outs,
                           parents=tmplt_banks, allow_overlap=False,
                           link_job_instance=link_job_instance,
                           compatibility_mode=compatibility_mode)

def setup_matchedfltr_dax_generated_multi(workflow, science_segs, datafind_outs,
                                          tmplt_banks, output_dir,
                                          injection_file=None,
                                          tags=None, link_to_tmpltbank=False,
                                          compatibility_mode=False):
    '''
    Setup matched-filter jobs that are generated as part of the workflow in
    which a single job reads in and generates triggers over multiple ifos.
    This
    pycbc_multi_inspiral or lalapps_coh_PTF_inspiral, but for new codes some
    additions are needed to define Executable and Job sub-classes
    (see jobutils.py).

    Returns
    -------
    '''
        tags = []

    ifos = sorted(science_segs.keys())

    # List for holding the output

    logging.info("Setting up matched-filtering for %s." %(' '.join(ifos),))

    if match_fltr_exe == 'pycbc_multi_inspiral':
        exe_class = select_matchedfilter_class(match_fltr_exe)
        cp.set('inspiral', 'longitude',\
               str(radians(float(cp.get('workflow', 'ra')))))
        cp.set('inspiral', 'latitude',\
               str(radians(float(cp.get('workflow', 'dec')))))
        # At the moment we aren't using sky grids, but when we do this code
        # might be used then. 
        # from pycbc.workflow.grb_utils import get_sky_grid_scale
        # if cp.has_option("jitter_skyloc", "apply-fermi-error"):
        #     cp.set('inspiral', 'sky-error',
        #            str(get_sky_grid_scale(float(cp.get('workflow',
        #                                                'sky-error')))))
        # else:
        #                                                'sky-error')),
        #                                   sigma_sys=0.0)))
        # cp.set('inspiral', 'trigger-time',\
        #        cp.get('workflow', 'trigger-time'))
        # cp.set('inspiral', 'block-duration',
        #        str(abs(science_segs[ifos[0]][0]) - \
        #                2 * int(cp.get('inspiral', 'pad-data'))))

        job_instance = exe_class(workflow.cp, 'inspiral', ifo=ifos,
                                 out_dir=output_dir,
                                 injection_file=injection_file,
                                 tags=tags)
        if cp.has_option("workflow", "do-long-slides") and "slide" in tags[-1]:
            slide_num = int(tags[-1].replace("slide", ""))
            logging.info("Setting up matched-filtering for slide {}"
                         .format(slide_num))
            slide_shift = int(cp.get("inspiral", "segment-length"))
            time_slide_dict = {ifo: (slide_num + 1) * ix * slide_shift
                               for ix, ifo in enumerate(ifos)}
            multi_ifo_coherent_job_setup(workflow, inspiral_outs, job_instance,
                                         science_segs, datafind_outs,
                                         output_dir, parents=tmplt_banks,
                                         slide_dict=time_slide_dict)
        else:
                                         output_dir, parents=tmplt_banks)
    else:
        # Select the appropriate class
        raise ValueError("Not currently supported.")

# Copyright (C) 2015 Christopher M. Biwer, Alexander Harvey Nitz
#
#
#

from pycbc.workflow.core import Executable, FileList, Node, makedir, File, Workflow
try:
    # Python 3
    from itertools import zip_longest
    # Python 2
    from itertools import izip_longest as zip_longest

    """ Create a list of n length tuples
    """

def setup_foreground_minifollowups(workflow, coinc_file, single_triggers,
                       tmpltbank_file, insp_segs, insp_data_name,
                       insp_anal_name, dax_output, out_dir, tags=None):
    """ Create plots that followup the Nth loudest coincident injection
    from a statmap produced HDF file.

    coinc_file:
    single_triggers: list of pycbc.workflow.File
        A list cointaining the file objects associated with the merged
        single detector trigger files for each ifo.
    tmpltbank_file: pycbc.workflow.File
        The file object pointing to the HDF format template bank
    insp_segs: SegFile
       The segment file containing the data read and analyzed by each inspiral
       job.
    insp_data_name: str
        The name of the segmentlist storing data read.
    insp_anal_name: str
        The name of the segmentlist storing data analyzed.
    out_dir: path
        The directory to store minifollowups result plots and files
    tags: {None, optional}
        Tags to add to the minifollowups executables

    Returns
    -------
    layout: list
        A list of tuples which specify the displayed file layout for the
        minifollops plots.
    """
    logging.info('Entering minifollowups module')

    if not workflow.cp.has_section('workflow-minifollowups'):
        logging.info('There is no [workflow-minifollowups] section in configuration file')
        logging.info('Leaving minifollowups')
        return

    makedir(dax_output)

    # turn the config file into a File class
    config_path = os.path.abspath(dax_output + '/' + '_'.join(tags) + 'foreground_minifollowup.ini')
    workflow.cp.write(open(config_path, 'w'))

    config_file = wdax.File(os.path.basename(config_path))
    config_file.PFN(urljoin('file:', pathname2url(config_path)), site='local')

    exe = Executable(workflow.cp, 'foreground_minifollowup',
                     ifos=workflow.ifos, out_dir=dax_output, tags=tags)

    node.add_input_opt('--config-files', config_file)
    node.add_input_opt('--bank-file', tmpltbank_file)
    node.add_input_opt('--statmap-file', coinc_file)
    node.add_multiifo_input_list_opt('--single-detector-triggers',
                                     single_triggers)
    node.add_input_opt('--inspiral-segments', insp_segs)
    node.add_opt('--inspiral-data-read-name', insp_data_name)
    node.add_opt('--inspiral-data-analyzed-name', insp_anal_name)
    if tags:
        node.add_list_opt('--tags', tags)
    node.new_output_file_opt(workflow.analysis_time, '.dax', '--output-file')
    node.new_output_file_opt(workflow.analysis_time, '.dax.map', '--output-map')
    node.new_output_file_opt(workflow.analysis_time, '.tc.txt',
                             '--transformation-catalog')

    name = node.output_files[0].name
    map_file = node.output_files[1]
    tc_file = node.output_files[2]

    node.add_opt('--workflow-name', name)
    node.add_opt('--output-dir', out_dir)

    # execute this in a sub-workflow
    fil = node.output_files[0]

    # determine if a staging site has been specified
    try:
        staging_site = workflow.cp.get('workflow-foreground_minifollowups',
                                       'staging-site')
    except:
        staging_site = None

    job = dax.DAX(fil)
    job.addArguments('--basename %s' % os.path.splitext(os.path.basename(name))[0])
    Workflow.set_job_properties(job, map_file, tc_file, staging_site=staging_site)
    workflow._adag.addJob(job)
    dep = dax.Dependency(parent=node._dax_node, child=job)
    workflow._adag.addDependency(dep)
    logging.info('Leaving minifollowups module')

def setup_single_det_minifollowups(workflow, single_trig_file, tmpltbank_file,
                                   insp_segs, insp_data_name, insp_anal_name,
                                   dax_output, out_dir, veto_file=None,
                                   veto_segment_name=None, statfiles=None,
                                   tags=None):
    """ Create plots that followup the Nth loudest clustered single detector
    triggers from a merged single detector trigger HDF file.

    single_trig_file: pycbc.workflow.File
        The File class holding the single detector triggers.
       The segment file containing the data read by each inspiral job.
    statfiles: FileList (optional, default=None)
        Supplementary files necessary for computing the single-detector
        statistic.
    Returns
    -------
    """

    if not workflow.cp.has_section('workflow-sngl_minifollowups'):
        msg = 'There is no [workflow-sngl_minifollowups] section in '
        msg += 'configuration file'
        logging.info(msg)
        return

    curr_ifo = single_trig_file.ifo
    config_path = os.path.abspath(dax_output + '/' + curr_ifo + \
                                   '_'.join(tags) + 'singles_minifollowup.ini')

    exe = Executable(workflow.cp, 'singles_minifollowup',
                     ifos=curr_ifo, out_dir=dax_output, tags=tags)

    wikifile = curr_ifo + '_'.join(tags) + 'loudest_table.txt'

    node.add_input_opt('--single-detector-file', single_trig_file)
    node.add_opt('--instrument', curr_ifo)
    node.add_opt('--wiki-file', wikifile)
    if veto_file is not None:
        assert(veto_segment_name is not None)
        node.add_opt('--veto-segment-name', veto_segment_name)
    if statfiles:
        statfiles = statfiles.find_output_with_ifo(curr_ifo)
        node.add_input_list_opt('--statistic-files', statfiles)
    if tags:
    node.new_output_file_opt(workflow.analysis_time, '.dax.map',
                             '--output-map')

    try:
        staging_site = workflow.cp.get('workflow-sngl_minifollowups',
    except:

    job.addArguments('--basename %s' \
                     % os.path.splitext(os.path.basename(name))[0])
    Workflow.set_job_properties(job, map_file, tc_file,
                                staging_site=staging_site)

def setup_injection_minifollowups(workflow, injection_file, inj_xml_file,
                                  single_triggers, tmpltbank_file,
                                  insp_segs, insp_data_name, insp_anal_name,
                                  dax_output, out_dir, tags=None):
    """ Create plots that followup the closest missed injections

    Returns
    -------
    """
    logging.info('Entering injection minifollowups module')

    if not workflow.cp.has_section('workflow-injection_minifollowups'):
        logging.info('There is no [workflow-injection_minifollowups] section in configuration file')
        return

    config_path = os.path.abspath(dax_output + '/' + '_'.join(tags) + 'injection_minifollowup.ini')

    exe = Executable(workflow.cp, 'injection_minifollowup', ifos=workflow.ifos, out_dir=dax_output)

    node.add_input_opt('--injection-file', injection_file)
    node.add_input_opt('--injection-xml-file', inj_xml_file)
    node.add_multiifo_input_list_opt('--single-detector-triggers', single_triggers)
    if tags:
    node.new_output_file_opt(workflow.analysis_time, '.dax', '--output-file', tags=tags)
    node.new_output_file_opt(workflow.analysis_time, '.dax.map', '--output-map', tags=tags)
    node.new_output_file_opt(workflow.analysis_time, '.tc.txt', '--transformation-catalog', tags=tags)

    try:
        staging_site = workflow.cp.get('workflow-injection_minifollowups',
    except:

    logging.info('Leaving injection minifollowups module')

class SingleTemplateExecutable(PlotExecutable):
    """Class to be used for to create workflow.Executable instances for the
    pycbc_single_template executable. Basically inherits directly from
    PlotExecutable but adds the file_input_options.
    """
    time_dependent_options = ['--channel-name', '--frame-type']

class SingleTimeFreqExecutable(PlotExecutable):
    pycbc_plot_singles_timefreq executable. Basically inherits directly from
    """

class PlotQScanExecutable(PlotExecutable):
    pycbc_plot_qscan executable. Basically inherits directly from
    """

def make_single_template_plots(workflow, segs, data_read_name, analyzed_name,
                                  params, out_dir, inj_file=None, exclude=None,
                                  require=None, tags=None, params_str=None,
                                  use_exact_inj_params=False):
    """Function for creating jobs to run the pycbc_single_template code and
    to run the associated plotting code pycbc_single_template_plots and add
    these jobs to the workflow.

    workflow : workflow.Workflow instance
        The pycbc.workflow.Workflow instance to add these jobs to.
    segs : workflow.File instance
        The pycbc.workflow.File instance that points to the XML file containing
        the segment lists of data read in and data analyzed.
    data_read_name : str
        The name of the segmentlist containing the data read in by each
        inspiral job in the segs file.
    analyzed_name : str
        The name of the segmentlist containing the data analyzed by each
    params : dictionary
        A dictionary containing the parameters of the template to be used.
        params[ifo+'end_time'] is required for all ifos in workflow.ifos.
        If use_exact_inj_params is False then also need to supply values for
        [mass1, mass2, spin1z, spin2x]. For precessing templates one also
        needs to supply [spin1y, spin1x, spin2x, spin2y, inclination]
        additionally for precession one must supply u_vals or
        u_vals_+ifo for all ifos. u_vals is the ratio between h_+ and h_x to
        use when constructing h(t). h(t) = (h_+ * u_vals) + h_x.
        Directory in which to store the output files.
    inj_file : workflow.File (optional, default=None)
        If given send this injection file to the job so that injections are
        made into the data.
    exclude : list (optional, default=None)
        If given, then when considering which subsections in the ini file to
        parse for options to add to single_template_plot, only use subsections
        that *do not* match strings in this list.
    require : list (optional, default=None)
        matching strings in this list.
    tags : list (optional, default=None)
        Add this list of tags to all jobs.
    params_str : str (optional, default=None)
        If given add this string to plot title and caption to describe the
        template that was used.
    use_exact_inj_params : boolean (optional, default=False)
        If True do not use masses and spins listed in the params dictionary
        but instead use the injection closest to the filter time as a template.

    Returns
    --------
    output_files : workflow.FileList
        The list of workflow.Files created in this function.
    """
    makedir(out_dir)
    name = 'single_template_plot'
    secs = requirestr(workflow.cp.get_subsections(name), require)
    secs = excludestr(secs, exclude)
    files = FileList([])
    for tag in secs:
        for ifo in workflow.ifos:
            if params['%s_end_time' % ifo] == -1.0:
                continue
            # Reanalyze the time around the trigger in each detector
            curr_exe = SingleTemplateExecutable(workflow.cp, 'single_template',
                                                ifos=[ifo], out_dir=out_dir,
                                                tags=[tag] + tags)
            start = int(params[ifo + '_end_time'])
            end = start + 1
            cseg = segments.segment([start, end])
            node = curr_exe.create_node(valid_seg=cseg)

            if use_exact_inj_params:
                node.add_opt('--use-params-of-closest-injection')
            else:
                node.add_opt('--mass1', "%.6f" % params['mass1'])
                node.add_opt('--mass2', "%.6f" % params['mass2'])
                node.add_opt('--spin1z',"%.6f" % params['spin1z'])
                node.add_opt('--spin2z',"%.6f" % params['spin2z'])
                node.add_opt('--template-start-frequency',
                             "%.6f" % params['f_lower'])
                # Is this precessing?
                if 'u_vals' in params or 'u_vals_%s' % ifo in params:
                    node.add_opt('--spin1x',"%.6f" % params['spin1x'])
                    node.add_opt('--spin1y',"%.6f" % params['spin1y'])
                    node.add_opt('--spin2x',"%.6f" % params['spin2x'])
                    node.add_opt('--spin2y',"%.6f" % params['spin2y'])
                    node.add_opt('--inclination',"%.6f" % params['inclination'])
                    try:
                        node.add_opt('--u-val',"%.6f" % params['u_vals'])
                    except:
                        node.add_opt('--u-val',
                                     "%.6f" % params['u_vals_%s' % ifo])

            # str(numpy.float64) restricts to 2d.p. BE CAREFUL WITH THIS!!!
            str_trig_time = '%.6f' %(params[ifo + '_end_time'])
            node.add_opt('--trigger-time', str_trig_time)
            node.add_input_opt('--inspiral-segments', segs)
            if inj_file is not None:
                node.add_input_opt('--injection-file', inj_file)
            node.add_opt('--data-read-name', data_read_name)
            node.add_opt('--data-analyzed-name', analyzed_name)
            node.new_output_file_opt(workflow.analysis_time, '.hdf',
                                     '--output-file', store_file=False)
            data = node.output_files[0]
            workflow += node
            # Make the plot for this trigger and detector
            node = PlotExecutable(workflow.cp, name, ifos=[ifo],
                              out_dir=out_dir, tags=[tag] + tags).create_node()
            node.add_input_opt('--single-template-file', data)
            node.new_output_file_opt(workflow.analysis_time, '.png',
                                     '--output-file')
            title="'%s SNR and chi^2 timeseries" %(ifo)
            if params_str is not None:
                title+= " using %s" %(params_str)
            title+="'"
            node.add_opt('--plot-title', title)
            caption = "'The SNR and chi^2 timeseries around the injection"
                caption += " using %s" %(params_str)
                caption += ". The injection itself was used as the template.'"
            else:
                caption += ". The template used has the following parameters: "
                caption += "mass1=%s, mass2=%s, spin1z=%s, spin2z=%s'"\
                       %(params['mass1'], params['mass2'], params['spin1z'],
                         params['spin2z'])
            node.add_opt('--plot-caption', caption)
            files += node.output_files
    return files

def make_plot_waveform_plot(workflow, params, out_dir, ifos, exclude=None,
                            require=None, tags=None):
    """ Add plot_waveform jobs to the workflow.
    """
        node = PlotExecutable(workflow.cp, 'plot_waveform', ifos=ifos,
        node.add_opt('--mass1', "%.6f" % params['mass1'])
        node.add_opt('--mass2', "%.6f" % params['mass2'])
        node.add_opt('--spin1z',"%.6f" % params['spin1z'])
        node.add_opt('--spin2z',"%.6f" % params['spin2z'])
        if 'u_vals' in params:
            # Precessing options
            node.add_opt('--spin1x',"%.6f" % params['spin1x'])
            node.add_opt('--spin2x',"%.6f" % params['spin2x'])
            node.add_opt('--spin1y',"%.6f" % params['spin1y'])
            node.add_opt('--spin2y',"%.6f" % params['spin2y'])
            node.add_opt('--inclination',"%.6f" % params['inclination'])
            node.add_opt('--u-val', "%.6f" % params['u_vals'])
        node.new_output_file_opt(workflow.analysis_time, '.png',
        files += node.output_files

def make_inj_info(workflow, injection_file, injection_index, num, out_dir,
                  tags=None):
    name = 'page_injinfo'
                              out_dir=out_dir, tags=tags).create_node()
    node.add_opt('--injection-index', str(injection_index))
    node.add_opt('--n-nearest', str(num))
    node.new_output_file_opt(workflow.analysis_time, '.html', '--output-file')
    files += node.output_files

def make_coinc_info(workflow, singles, bank, coinc, out_dir,
                    n_loudest=None, trig_id=None, file_substring=None,
                    sort_order=None, sort_var=None, tags=None):
    name = 'page_coincinfo'
    node.add_input_list_opt('--single-trigger-files', singles)
    node.add_input_opt('--statmap-file', coinc)
    node.add_input_opt('--bank-file', bank)
    if sort_order:
        node.add_opt('--sort-order', sort_order)
    if sort_var:
        node.add_opt('--sort-variable', sort_var)
    if n_loudest is not None:
        node.add_opt('--n-loudest', str(n_loudest))
    if trig_id is not None:
        node.add_opt('--trigger-id', str(trig_id))
    if file_substring is not None:
        node.add_opt('--statmap-file-subspace-name', file_substring)

def make_sngl_ifo(workflow, sngl_file, bank_file, trigger_id, out_dir, ifo,
    """Setup a job to create sngl detector sngl ifo html summary snippet.
    """
    name = 'page_snglinfo'
    node = PlotExecutable(workflow.cp, name, ifos=[ifo],
    node.add_input_opt('--single-trigger-file', sngl_file)
    node.add_opt('--trigger-id', str(trigger_id))
    node.add_opt('--instrument', ifo)

def make_trigger_timeseries(workflow, singles, ifo_times, out_dir, special_tids=None,
                            exclude=None, require=None, tags=None):
    name = 'plot_trigger_timeseries'
        node = PlotExecutable(workflow.cp, name, ifos=workflow.ifos,
        node.add_multiifo_input_list_opt('--single-trigger-files', singles)
        node.add_opt('--times', ifo_times)
        node.new_output_file_opt(workflow.analysis_time, '.png', '--output-file')

        if special_tids is not None:
            node.add_opt('--special-trigger-ids', special_tids)

def make_qscan_plot(workflow, ifo, trig_time, out_dir, injection_file=None,
                    data_segments=None, time_window=100, tags=None):
    """ Generate a make_qscan node and add it to workflow.

    This function generates a single node of the singles_timefreq executable
    and adds it to the current workflow. Parent/child relationships are set by
    the input/output files automatically.

    ifo: str
        Which interferometer are we using?
    trig_time: int
        The time of the trigger being followed up.
    out_dir: str
        Location of directory to output to
    injection_file: pycbc.workflow.File (optional, default=None)
        If given, add the injections in the file to strain before making the
        plot.
    data_segments: ligo.segments.segmentlist (optional, default=None)
        The list of segments for which data exists and can be read in. If given
        the start/end times given to singles_timefreq will be adjusted if
        [trig_time - time_window, trig_time + time_window] does not completely
        lie within a valid data segment. A ValueError will be raised if the
        trig_time is not within a valid segment, or if it is not possible to
        find 2*time_window (plus the padding) of continuous data around the
        trigger. This **must** be coalesced.
    time_window: int (optional, default=None)
        The amount of data (not including padding) that will be read in by the
        singles_timefreq job. The default value of 100s should be fine for most
        cases.
    tags: list (optional, default=None)
        List of tags to add to the created nodes, which determine file naming.
    """
    name = 'plot_qscan'

    curr_exe = PlotQScanExecutable(workflow.cp, name, ifos=[ifo],
                          out_dir=out_dir, tags=tags)

    # Determine start/end times, using data segments if needed.
    # Begin by choosing "optimal" times
    start = trig_time - time_window
    end = trig_time + time_window
    node = curr_exe.create_node(valid_seg=segments.segment([start, end]))

    # Then if data_segments is available, check against that, and move if
    # needed
    if data_segments is not None:
        # Assumes coalesced, so trig_time can only be within one segment
        for seg in data_segments:
            if trig_time in seg:
                data_seg = seg
                break
            elif trig_time == -1.0:
                node.add_opt('--gps-start-time', int(trig_time))
                node.add_opt('--gps-end-time', int(trig_time))
                node.add_opt('--center-time', trig_time)
                caption_string = "'No trigger in %s'" % ifo
                node.add_opt('--plot-caption', caption_string)
                node.new_output_file_opt(workflow.analysis_time, '.png', '--output-file')
                workflow += node
                return node.output_files
        else:
            err_msg = "Trig time {} ".format(trig_time)
            err_msg += "does not seem to lie within any data segments. "
            err_msg += "This shouldn't be possible, please ask for help!"
        # Check for pad-data
        if curr_exe.has_opt('pad-data'):
            pad_data = int(curr_exe.get_opt('pad-data'))
        else:
            pad_data = 0
        # We only read data that's available. The code must handle the case
        # of not much data being available.
        if end > (data_seg[1] - pad_data):
            end = data_seg[1] - pad_data
        if start < (data_seg[0] + pad_data):
            start = data_seg[0] + pad_data

    node.add_opt('--gps-start-time', int(start))
    node.add_opt('--gps-end-time', int(end))
    node.add_opt('--center-time', trig_time)

    if injection_file is not None:
        node.add_input_opt('--injection-file', injection_file)

    node.new_output_file_opt(workflow.analysis_time, '.png', '--output-file')

def make_singles_timefreq(workflow, single, bank_file, trig_time, out_dir,
                          veto_file=None, time_window=10, data_segments=None,
    """ Generate a singles_timefreq node and add it to workflow.

    single: pycbc.workflow.core.File instance
        The File object storing the single-detector triggers to followup.
    bank_file: pycbc.workflow.core.File instance
        The File object storing the template bank.
    veto_file: pycbc.workflow.core.File (optional, default=None)
        If given use this file to veto triggers to determine the loudest event.
        FIXME: Veto files *should* be provided a definer argument and not just
        assume that all segments should be read.
        singles_timefreq job. The default value of 10s should be fine for most
        cases.
    """
    name = 'plot_singles_timefreq'

    curr_exe = SingleTimeFreqExecutable(workflow.cp, name, ifos=[single.ifo],

    node.add_input_opt('--trig-file', single)

    # needed
                break

                if veto_file:
                    node.add_input_opt('--veto-file', veto_file)

                node.add_opt('--detector', single.ifo)
        else:
        else:
        if abs(data_seg) < (2 * time_window + 2 * pad_data):
            tl = 2 * time_window + 2 * pad_data
            err_msg = "I was asked to use {} seconds of data ".format(tl)
            err_msg += "to run a plot_singles_timefreq job. However, I have "
            err_msg += "only {} seconds available.".format(abs(data_seg))
        if data_seg[0] > (start - pad_data):
            end = start + 2 * time_window
        if data_seg[1] < (end + pad_data):
            start = end - 2 * time_window
        # Sanity check, shouldn't get here!
            err_msg = "I shouldn't be here! Go ask Ian what he broke."

    if veto_file:

    node.add_opt('--detector', single.ifo)

def make_skipped_html(workflow, skipped_data, out_dir, tags):
    """
    Make a html snippet from the list of skipped background coincidences
    """
    exe = Executable(workflow.cp, 'html_snippet',
                     ifos=workflow.ifos, out_dir=out_dir, tags=tags)

    parsed_data = {}
    for ifo, time in skipped_data:
        if ifo not in parsed_data:
            parsed_data[ifo] = {}
        if time not in parsed_data[ifo]:
            parsed_data[ifo][time] = 1
        else:
            parsed_data[ifo][time] = parsed_data[ifo][time] + 1

    n_events = len(skipped_data)
    html_string = '"{} background events have been skipped '.format(n_events)
    html_string += 'because one of their single triggers already appears '
    html_string += 'in the events followed up above. '
    html_string += 'Specifically, the following single detector triggers '
    html_string += 'were found in these coincidences. '
    html_template = '{} event at time {} appeared {} times. '
    for ifo in parsed_data:
        for time in parsed_data[ifo]:
            n_occurances = parsed_data[ifo][time]
            html_string += html_template.format(ifo, time, n_occurances)

    html_string += '"'

    node.add_opt('--html-text', html_string)
    node.add_opt('--title', '"Events were skipped"')
    files = node.output_files

def create_noop_node():
    """
    Creates a noop node that can be added to a DAX doing nothing. The reason
    for using this is if a minifollowups dax contains no triggers currently
    the dax will contain no jobs and be invalid. By adding a noop node we
    ensure that such daxes will actually run if one adds one such noop node.
    Adding such a noop node into a workflow *more than once* will cause a
    failure.
    """
    exe = wdax.Executable('NOOP')
    pfn = distutils.spawn.find_executable('true')
    exe.add_pfn(pfn)
    node = wdax.Node(exe)

# Copyright (C) 2014  Alex Nitz
#
#
#
#

#
#
#                                   Preamble
#
#
""" This module provides thin wrappers around Pegasus.DAX3 functionality that
provides additional abstraction and argument handling.
"""
import os
from six.moves.urllib.parse import urljoin, urlsplit
from Pegasus.catalogs.transformation_catalog import TransformationCatalog

class ProfileShortcuts(object):
    """ Container of common methods for setting pegasus profile information
    on Executables and nodes. This class expects to be inherited from
    and for a add_profile method to be implemented.
    """
    def set_memory(self, size):
        """ Set the amount of memory that is required in megabytes
        """
        self.add_profile('condor', 'request_memory', '%sM' % size)

    def set_storage(self, size):
        """ Set the amount of storage required in megabytes
        """
        self.add_profile('condor', 'request_disk', '%sM' % size)

    def set_num_cpus(self, number):
        self.add_profile('condor', 'request_cpus', number)

    def set_universe(self, universe):
        if universe is 'standard':
            self.add_profile("pegasus", "gridstart", "none")

        self.add_profile("condor", "universe", universe)

    def set_category(self, category):
        self.add_profile('dagman', 'category', category)

    def set_priority(self, priority):
        self.add_profile('dagman', 'priority', priority)

    def set_num_retries(self, number):
        self.add_profile("dagman", "retry", number)

class Executable(ProfileShortcuts):
    """ The workflow representation of an Executable
    """
    id = 0
    def __init__(self, name, namespace=None, os='linux',
                       arch='x86_64', installed=True, version=None,
                       container=None):
        self.logical_name = name + "_ID%s" % str(Executable.id)
        Executable.id += 1
        self.namespace = namespace
        self.version = version
        if container:
            self._dax_executable = dax.Executable(self.logical_name,
                   namespace=self.namespace, version=version, os=os,
                   arch=arch, installed=installed, container=container)
        else:
                   arch=arch, installed=installed)
        self.in_workflow = False
        self.pfns = {}

    def clear_pfns(self):
        self._dax_executable.clearPFNs()

    def add_pfn(self, url, site='local'):
        self._dax_executable.PFN(url, site)
        self.pfns[site] = url

    def get_pfn(self, site='local'):
        return self.pfns[site]

    def insert_into_dax(self, dax):
        dax.addExecutable(self._dax_executable)

    def add_profile(self, namespace, key, value, force=False):
        """ Add profile information to this executable
        """
        try:
            entry = dax.Profile(namespace, key, value)
            self._dax_executable.addProfile(entry)
        except dax.DuplicateError:
            if force:
                # Replace with the new key
                self._dax_executable.removeProfile(entry)
                self._dax_executable.addProfile(entry)

    def is_same_as(self, other):
        test_vals = ['namespace', 'version', 'arch', 'os', 'osrelease',
                     'osversion', 'glibc', 'installed', 'container']
        # Check for logical name first
        if not self.pegasus_name == other.pegasus_name:

        # Check the properties of the executable
        for val in test_vals:
            sattr = getattr(self._dax_executable, val)
            oattr = getattr(other._dax_executable, val)
            if not sattr == oattr:
        # Also check the "profile". This is things like Universe, RAM/disk/CPU
        # requests, execution site, getenv=True, etc.
        for profile in self._dax_executable.profiles:
            if profile not in other._dax_executable.profiles:
        for profile in other._dax_executable.profiles:
            if profile not in self._dax_executable.profiles:

class Node(ProfileShortcuts):
    def __init__(self, executable):
        self.executable=executable
        self._inputs = []
        self._outputs = []
        self._dax_node = dax.Job(name=executable.logical_name,
                                 version = executable.version,
                                 namespace=executable.namespace)
        self._args = []
        # Each value in _options is added separated with whitespace
        # so ['--option','value'] --> "--option value"
        self._options = []
        # For _raw_options *NO* whitespace is added.
        # so ['--option','value'] --> "--optionvalue"
        # and ['--option',' ','value'] --> "--option value"
        self._raw_options = []

    def add_arg(self, arg):
        """ Add an argument
        """
        if not isinstance(arg, File):
            arg = str(arg)

        self._args += [arg]

    def add_raw_arg(self, arg):
        """ Add an argument to the command line of this job, but do *NOT* add
            white space between arguments. This can be added manually by adding
            ' ' if needed
        """

        self._raw_options += [arg]

        """ Add a option
        """
        if value is not None:
            if not isinstance(value, File):
                value = str(value)
            self._options += [opt, value]
        else:
            self._options += [opt]

    def add_input(self, inp):
        """Declares an input file without adding it as a command-line option.
        """
        self._add_input(inp)

    #private functions to add input and output data sources/sinks
    def _add_input(self, inp):
        """ Add as source of input data
        """
        self._inputs += [inp]
        inp._set_as_input_of(self)

    def _add_output(self, out):
        """ Add as destination of output data
        """
        self._outputs += [out]
        out.node = self
        out._set_as_output_of(self)

    # public functions to add options, arguments with or without data sources
    def add_input_opt(self, opt, inp):
        """ Add an option that determines an input
        """
        self.add_opt(opt, inp._dax_repr())

    def add_output_opt(self, opt, out):
        """ Add an option that determines an output
        """
        self.add_opt(opt, out._dax_repr())
        self._add_output(out)

    def add_output_list_opt(self, opt, outputs):
        """ Add an option that determines a list of outputs
        """
        self.add_opt(opt)
        for out in outputs:
            self.add_opt(out)
            self._add_output(out)

    def add_input_list_opt(self, opt, inputs):
        """ Add an option that determines a list of inputs
        """
        for inp in inputs:
            self.add_opt(inp)

    def add_list_opt(self, opt, values):
        """ Add an option with a list of non-file parameters.
        """
        for val in values:
            self.add_opt(val)

    def add_input_arg(self, inp):
        """ Add an input as an argument
        """
        self.add_arg(inp._dax_repr())

    def add_output_arg(self, out):
        """ Add an output as an argument
        """
        self.add_arg(out._dax_repr())

    def new_output_file_opt(self, opt, name):
        """ Add an option and return a new file handle
        """
        fil = File(name)
        self.add_output_opt(opt, fil)
        return fil

    # functions to describe properties of this node
        """ Add profile information to this node at the DAX level
        """
        try:
            self._dax_node.addProfile(entry)
            if force:
                self._dax_node.removeProfile(entry)
                self._dax_node.addProfile(entry)

    def _finalize(self):
        args = self._args + self._options
        self._dax_node.addArguments(*args)
        if len(self._raw_options):
            raw_args = [' '] + self._raw_options
            self._dax_node.addRawArguments(*raw_args)

class Workflow(object):
    """
    """
    def __init__(self, name='my_workflow'):
        self.name = name
        self._adag = dax.ADAG(name)

        self._executables = []
        self.sub_workflows = []
        self._external_workflow_inputs = []
        self.filename = self.name + '.dax'

        self.as_job = dax.DAX(self.filename)

    def _make_root_dependency(self, inp):
        def root_path(v):
            path = [v]
            while v.in_workflow:
                path += [v.in_workflow]
                v = v.in_workflow
            return path
        workflow_root = root_path(self)
        input_root = root_path(inp)
        for step in workflow_root:
            if step in input_root:
                common = step
                break
        dep = dax.Dependency(
            parent=input_root[input_root.index(common)-1].as_job,
            child=workflow_root[workflow_root.index(common)-1].as_job)
        common._adag.addDependency(dep)

    def add_workflow(self, workflow):
        """ Add a sub-workflow to this workflow

        This function adds a sub-workflow of Workflow class to this workflow.
        Parent child relationships are determined by data dependencies

        workflow : Workflow instance
            The sub-workflow to add to this one
        """
        workflow.in_workflow = self
        self.sub_workflows += [workflow]

        node = workflow.as_job
        self._adag.addJob(node)

        node.file.PFN(os.path.join(os.getcwd(), node.file.name), site='local')
        self._adag.addFile(node.file)

        for inp in workflow._external_workflow_inputs:
            workflow._make_root_dependency(inp.node)

    def add_node(self, node):
        """ Add a node to this workflow

        This function adds nodes to the workflow. It also determines
        parent/child relations from the DataStorage inputs to this job.

        node : pycbc.workflow.pegasus_workflow.Node
            A node that should be executed as part of this workflow.
        """
        node._finalize()
        node.in_workflow = self

        # Record the executable that this node uses
        if not node.executable.in_workflow:
            for exe in self._executables:
                if node.executable.is_same_as(exe):
                    node.executable.in_workflow = True
                    node._dax_node.name = exe.logical_name
                    node.executable.logical_name = exe.logical_name
                    break
            else:
                node.executable.in_workflow = True
                self._executables += [node.executable]

        # Add the node itself
        self._adag.addJob(node._dax_node)

        # Determine the parent child relationships based on the inputs that
        # this node requires.
        added_nodes = []
        for inp in node._inputs:
            if inp.node is not None and inp.node.in_workflow == self:
                if inp.node not in added_nodes:
                    parent = inp.node._dax_node
                    child = node._dax_node
                    dep = dax.Dependency(parent=parent, child=child)
                    self._adag.addDependency(dep)
                    added_nodes.append(inp.node)

            elif inp.node is not None and not inp.node.in_workflow:
                raise ValueError('Parents of this node must be added to the '
                                 'workflow first.')

            elif inp.node is None and not inp.workflow_input:
                self._inputs += [inp]
                inp.workflow_input = True

            elif inp.node is not None and inp.node.in_workflow != self and inp not in self._inputs:
                self._external_workflow_inputs += [inp]

        # Record the outputs that this node generates
        self._outputs += node._outputs

        if isinstance(other, Node):
            return self.add_node(other)
        elif isinstance(other, Workflow):
            return self.add_workflow(other)
        else:
            raise TypeError('Cannot add type %s to this workflow' % type(other))

    def save(self, filename=None, tc=None):
        """ Write this workflow to DAX file
        """
            filename = self.filename

        for sub in self.sub_workflows:
            sub.save()

        # FIXME this is ugly as pegasus 4.9.0 does not support the full
        # transformation catalog in the DAX. I have asked Karan to fix this so
        # that executables and containers can be specified in the DAX itself.
        # Karan says that XML is going away in Pegasus 5.x and so this code
        # will need to be re-written anyway.
        #
        # the transformation catalog is written in the same directory as the
        # DAX.  pycbc_submit_dax needs to know this so that the right
        # transformation catalog is used when the DAX is planned.
        if tc is None:
            tc = '{}.tc.txt'.format(filename)
        p = os.path.dirname(tc)
        f = os.path.basename(tc)
        if not p:
            p = '.'

        tc = TransformationCatalog(p, f)

        for e in self._adag.executables.copy():
            tc.add(e)
            try:
                tc.add_container(e.container)
            except:
                pass
            self._adag.removeExecutable(e)

        f = open(filename, "w")
        self._adag.writeXML(f)
        tc.write()

class DataStorage(object):
    """ A workflow representation of a place to store and read data from.

    The abstract representation of a place to store and read data from. This
    can include files, database, or remote connections. This object is
    used as a handle to pass between functions, and is used a way to logically
    represent the order operation on the physical data.
    """
    def __init__(self, name):
        self.node = None
        self.workflow_input = False

    def _set_as_node_input(self):
        pass

    def _set_as_node_output(self):
        pass

    def _dax_repr(self):
        return self.name

class File(DataStorage, dax.File):
    """ The workflow representation of a physical file

    An object that represents a file from the perspective of setting up a
    workflow. The file may or may not exist at the time of workflow generation.
    If it does, this is represented by containing a physical file name (PFN).
    A storage path is also available to indicate the desired final
    destination of this file.
    """
        DataStorage.__init__(self, name)
        dax.File.__init__(self, name)
        self.storage_path = None

    @property
    def dax_repr(self):
        """Return the dax representation of a File."""
        return self._dax_repr()

    def _set_as_input_of(self, node):
        node._dax_node.uses(self, link=dax.Link.INPUT, register=False,
                                                       transfer=True)
    def _set_as_output_of(self, node):
        if self.storage_path:
            transfer_file = True
        else:
            transfer_file = False
        node._dax_node.uses(self, link=dax.Link.OUTPUT, register=True,
                                                        transfer=transfer_file)
    def output_map_str(self):
            return '%s %s pool="%s"' % (self.name, self.storage_path, 'local')
        else:
            raise ValueError('This file does not have a storage path')

    def has_pfn(self, url, site=None):
        """ Wrapper of the pegasus hasPFN function, that allows it to be called
        outside of specific pegasus functions.
        """
        curr_pfn = dax.PFN(url, site)
        return self.hasPFN(curr_pfn)

        dax.addFile(self)

    def from_path(cls, path):
        """Takes a path and returns a File object with the path as the PFN."""
        urlparts = urlsplit(path)
        site = 'nonlocal'
        if (urlparts.scheme == '' or urlparts.scheme == 'file'):
            if os.path.isfile(urlparts.path):
                path = os.path.abspath(urlparts.path)
                path = urljoin('file:', pathname2url(path)) 
                site = 'local'

        fil = File(os.path.basename(path))
        fil.PFN(path, site)
        return fil

class Database(DataStorage):
    pass

# Copyright (C) 2015  Alex Nitz
#
#
#

#
#
#                                   Preamble
#
#
"""
This module is responsible for setting up plotting jobs.
"""
from pycbc.workflow.core import File, FileList, makedir, Executable

def excludestr(tags, substr):
    if substr is None:
        return tags
    if isinstance(substr, list):
        if len(substr) > 1:
            tags = excludestr(tags, substr[1:])
        substr = substr[0]
    return [tag for tag in tags if substr not in tag]

def requirestr(tags, substr):
    return [tag for tag in tags if substr in tag]

class PlotExecutable(Executable):
    """ plot executable
    """

    # plots and final results should get the highest priority
    # on the job queue
        node = Executable.create_node(self, **kwargs)
        node.set_priority(1000)

def make_template_plot(workflow, bank_file, out_dir, tags=None):
    node = PlotExecutable(workflow.cp, 'plot_bank', ifos=workflow.ifos,

    if workflow.cp.has_option_tags('workflow-coincidence', 'background-bins', tags=tags):
        bins = workflow.cp.get_opt_tags('workflow-coincidence', 'background-bins', tags=tags)
        node.add_opt('--background-bins', bins)

def make_range_plot(workflow, psd_files, out_dir, exclude=None, require=None,
                   tags=None):
    secs = requirestr(workflow.cp.get_subsections('plot_range'), require)
        node = PlotExecutable(workflow.cp, 'plot_range', ifos=workflow.ifos,
        node.add_input_list_opt('--psd-files', psd_files)

def make_spectrum_plot(workflow, psd_files, out_dir, tags=None,
                       hdf_group=None, precalc_psd_files=None):
    node = PlotExecutable(workflow.cp, 'plot_spectrum', ifos=workflow.ifos,
    node.add_input_list_opt('--psd-files', psd_files)

    if hdf_group is not None:
        node.add_opt('--hdf-group', hdf_group)
    if precalc_psd_files is not None and len(precalc_psd_files) == 1:
        node.add_input_list_opt('--psd-file', precalc_psd_files)

def make_segments_plot(workflow, seg_files, out_dir, tags=None):
    node = PlotExecutable(workflow.cp, 'plot_segments', ifos=workflow.ifos,
                         out_dir=out_dir, tags=tags).create_node()
    node.add_input_list_opt('--segment-files', seg_files)

def make_gating_plot(workflow, insp_files, out_dir, tags=None):
    node = PlotExecutable(workflow.cp, 'plot_gating', ifos=workflow.ifos,
    node.add_input_list_opt('--input-file', insp_files)

def make_throughput_plot(workflow, insp_files, out_dir, tags=None):
    node = PlotExecutable(workflow.cp, 'plot_throughput', ifos=workflow.ifos,

def make_foreground_table(workflow, trig_file, bank_file, out_dir,
                          singles=None, extension='.html', tags=None,
                          hierarchical_level=None):

    if hierarchical_level is not None and tags:
        tags = [("HIERARCHICAL_LEVEL_{:02d}".format(
                hierarchical_level))] + tags
    elif hierarchical_level is not None and not tags:
        tags = ["HIERARCHICAL_LEVEL_{:02d}".format(hierarchical_level)]
    elif hierarchical_level is None and not tags:
        tags = []

    exe = PlotExecutable(workflow.cp, 'page_foreground',
                         ifos=trig_file.ifo_list,
                         out_dir=out_dir, tags=tags)
    node.add_input_opt('--trigger-file', trig_file)
    if hierarchical_level is not None:
        node.add_opt('--use-hierarchical-level', hierarchical_level)
    if singles is not None:
        node.add_input_list_opt('--single-detector-triggers', singles)
    node.new_output_file_opt(bank_file.segment, extension, '--output-file')

def make_sensitivity_plot(workflow, inj_file, out_dir, exclude=None,
                         require=None, tags=None):
    secs = requirestr(workflow.cp.get_subsections('plot_sensitivity'), require)
        node = PlotExecutable(workflow.cp, 'plot_sensitivity', ifos=workflow.ifos,
                    out_dir=out_dir, tags=[tag] + tags).create_node()
        node.add_input_opt('--injection-file', inj_file)
        node.new_output_file_opt(inj_file.segment, '.png', '--output-file')

def make_coinc_snrchi_plot(workflow, inj_file, inj_trig, stat_file, trig_file,
                          out_dir, exclude=None, require=None, tags=None):
    secs = requirestr(workflow.cp.get_subsections('plot_coinc_snrchi'), require)
        exe = PlotExecutable(workflow.cp, 'plot_coinc_snrchi',
                             ifos=inj_trig.ifo_list,
                             out_dir=out_dir, tags=[tag] + tags)
        node.add_input_opt('--found-injection-file', inj_file)
        node.add_input_opt('--single-injection-file', inj_trig)
        node.add_input_opt('--coinc-statistic-file', stat_file)
        node.add_input_opt('--single-trigger-file', trig_file)

def make_inj_table(workflow, inj_file, out_dir, missed=False, singles=None,
    node = PlotExecutable(workflow.cp, 'page_injections', ifos=workflow.ifos,
                    out_dir=out_dir, tags=tags).create_node()

    if missed:
        node.add_opt('--show-missed')

    node.new_output_file_opt(inj_file.segment, '.html', '--output-file')

def make_seg_table(workflow, seg_files, seg_names, out_dir, tags=None,
                  title_text=None, description=None):
    """ Creates a node in the workflow for writing the segment summary
    table. Returns a File instances for the output file.
    """
    seg_files = list(seg_files)
    seg_names = list(seg_names)
    if tags is None: tags = []
    node = PlotExecutable(workflow.cp, 'page_segtable', ifos=workflow.ifos,
    quoted_seg_names = []
    for s in seg_names:
        quoted_seg_names.append("'" + s + "'")
    node.add_opt('--segment-names', ' '.join(quoted_seg_names))
    if description:
        node.add_opt('--description', "'" + description + "'")
    if title_text:
        node.add_opt('--title-text', "'" + title_text + "'")

def make_veto_table(workflow, out_dir, vetodef_file=None, tags=None):
    """ Creates a node in the workflow for writing the veto_definer
    """
    if vetodef_file is None:
        if not workflow.cp.has_option_tags("workflow-segments",
                                           "segments-veto-definer-file", []):
        vetodef_file = workflow.cp.get_opt_tags("workflow-segments",
                                           "segments-veto-definer-file", [])
        file_url = urljoin('file:', pathname2url(vetodef_file))
        vdf_file = File(workflow.ifos, 'VETO_DEFINER',
                        workflow.analysis_time, file_url=file_url)
        vdf_file.PFN(file_url, site='local')
    else:
        vdf_file = vetodef_file

    node = PlotExecutable(workflow.cp, 'page_vetotable', ifos=workflow.ifos,
    node.add_input_opt('--veto-definer-file', vdf_file)

def make_seg_plot(workflow, seg_files, out_dir, seg_names=None, tags=None):
    """ Creates a node in the workflow for plotting science, and veto segments.
    """

    node = PlotExecutable(workflow.cp, 'page_segplot', ifos=workflow.ifos,

def make_ifar_plot(workflow, trigger_file, out_dir, tags=None,
                   hierarchical_level=None, executable='page_ifar'):
    """ Creates a node in the workflow for plotting cumulative histogram
    of IFAR values.
    """

        tags = []

    exe = PlotExecutable(workflow.cp, executable, ifos=trigger_file.ifo_list,
    node.add_input_opt('--trigger-file', trigger_file)

def make_snrchi_plot(workflow, trig_files, veto_file, veto_name,
                     out_dir, exclude=None, require=None, tags=None):
    secs = requirestr(workflow.cp.get_subsections('plot_snrchi'), require)
        for trig_file in trig_files:
            exe = PlotExecutable(workflow.cp, 'plot_snrchi',
                                 ifos=trig_file.ifo_list,
                                 out_dir=out_dir,
                                 tags=[tag] + tags)
            node = exe.create_node()
            node.add_input_opt('--trigger-file', trig_file)
            if veto_file is not None:
                node.add_input_opt('--veto-file', veto_file)
                node.add_opt('--segment-name', veto_name)
            node.new_output_file_opt(trig_file.segment, '.png', '--output-file')

def make_foundmissed_plot(workflow, inj_file, out_dir, exclude=None,
        tags = []
    secs = requirestr(workflow.cp.get_subsections('plot_foundmissed'), require)
        exe = PlotExecutable(workflow.cp, 'plot_foundmissed', ifos=workflow.ifos,
                    out_dir=out_dir, tags=[tag] + tags)
        ext = '.html' if exe.has_opt('dynamic') else '.png'
        node.new_output_file_opt(inj_file.segment, ext, '--output-file')

def make_snrratehist_plot(workflow, bg_file, out_dir, closed_box=False,
                          tags=None, hierarchical_level=None):

        tags = []

    exe = PlotExecutable(workflow.cp, 'plot_snrratehist',
                         ifos=bg_file.ifo_list,
    node.add_input_opt('--trigger-file', bg_file)

    if closed_box:
        node.add_opt('--closed-box')

    node.new_output_file_opt(bg_file.segment, '.png', '--output-file')

def make_snrifar_plot(workflow, bg_file, out_dir, closed_box=False,
                     cumulative=True, tags=None, hierarchical_level=None):

        tags = []

    exe = PlotExecutable(workflow.cp, 'plot_snrifar', ifos=bg_file.ifo_list,

    if not cumulative:
        node.add_opt('--not-cumulative')

def make_results_web_page(workflow, results_dir, explicit_dependencies=None):
    template_path = 'templates/orange.html'

    out_dir = workflow.cp.get('results_page', 'output-path')
    node = PlotExecutable(workflow.cp, 'results_page', ifos=workflow.ifos,
                out_dir=out_dir).create_node()
    node.add_opt('--plots-dir', results_dir)
    node.add_opt('--template-file', template_path)
    if explicit_dependencies is not None:
        import Pegasus.DAX3 as dax
        for dep in explicit_dependencies:
            dax_dep = dax.Dependency(parent=dep._dax_node,
                                     child=node._dax_node)
            workflow._adag.addDependency(dax_dep)

def make_single_hist(workflow, trig_file, veto_file, veto_name,
                     out_dir, bank_file=None, exclude=None,
                     require=None, tags=None):
    secs = requirestr(workflow.cp.get_subsections('plot_hist'), require)
        node = PlotExecutable(workflow.cp, 'plot_hist',
                    ifos=trig_file.ifo,
                    out_dir=out_dir,
                    tags=[tag] + tags).create_node()
            node.add_input_opt('--bank-file', bank_file)
        node.new_output_file_opt(trig_file.segment, '.png', '--output-file')

def make_binned_hist(workflow, trig_file, veto_file, veto_name,
                     out_dir, bank_file, exclude=None,
    secs = requirestr(workflow.cp.get_subsections('plot_binnedhist'), require)
        node = PlotExecutable(workflow.cp, 'plot_binnedhist',
        node.add_opt('--ifo', trig_file.ifo)
            node.add_opt('--veto-segment-name', veto_name)

def make_singles_plot(workflow, trig_files, bank_file, veto_file, veto_name,
    secs = requirestr(workflow.cp.get_subsections('plot_singles'), require)
            node = PlotExecutable(workflow.cp, 'plot_singles',
                        ifos=trig_file.ifo,
                        out_dir=out_dir,
                        tags=[tag] + tags).create_node()

            node.add_opt('--detector', trig_file.ifo)
            node.add_input_opt('--single-trig-file', trig_file)

#
#
#

"""This module is responsible for setting up PSD-related jobs in workflows.
"""

from pycbc.workflow.core import FileList, make_analysis_dir, Executable, File
from pycbc.workflow.core import SegFile
from ligo.segments import segmentlist

class CalcPSDExecutable(Executable):

class MergePSDFiles(Executable):

def chunks(l, n):
    """ Yield n successive chunks from l.
    """
    newn = int(len(l) / n)
    for i in range(0, n-1):
        yield l[i*newn:i*newn+newn]
    yield l[n*newn-newn:]

def merge_psds(workflow, files, ifo, out_dir, tags=None):
    tags = [] if not tags else tags
    node = MergePSDFiles(workflow.cp, 'merge_psds',
                         ifos=ifo, out_dir=out_dir,
                         tags=tags).create_node()
    node.add_input_list_opt('--psd-files', files)

def setup_psd_calculate(workflow, frame_files, ifo, segments,
                        segment_name, out_dir, tags=None):
    if workflow.cp.has_option_tags('workflow-psd', 'parallelization-factor', tags=tags):
        num_parts = int(workflow.cp.get_opt_tags('workflow-psd',
                                                 'parallelization-factor',
                                                 tags=tags))
    else:
        num_parts = 1

    # get rid of duplicate segments which happen when splitting the bank
    segments = segmentlist(frozenset(segments))

    segment_lists = list(chunks(segments, num_parts))

    psd_files = FileList([])
    for i, segs in enumerate(segment_lists):
        seg_file = SegFile.from_segment_list('%s_%s' %(segment_name, i),
                         segmentlist(segs), segment_name, ifo,
                         valid_segment=workflow.analysis_time,
                         extension='xml', directory=out_dir)

        psd_files += [make_psd_file(workflow, frame_files, seg_file,
                                    segment_name, out_dir,
                                    tags=tags + ['PART%s' % i])]

    if num_parts > 1:
        return merge_psds(workflow, psd_files, ifo, out_dir, tags=tags)
    else:
        return psd_files[0]

def make_psd_file(workflow, frame_files, segment_file, segment_name, out_dir,
    exe = CalcPSDExecutable(workflow.cp, 'calculate_psd',
                             ifos=segment_file.ifo, out_dir=out_dir,
                             tags=tags)
    node.add_input_opt('--analysis-segment-file', segment_file)
    node.add_opt('--segment-name', segment_name)

    if frame_files and not exe.has_opt('frame-type'):
        node.add_input_list_opt('--frame-files', frame_files)

class AvgPSDExecutable(Executable):

def make_average_psd(workflow, psd_files, out_dir, tags=None,
                     output_fmt='.txt'):
    node = AvgPSDExecutable(workflow.cp, 'average_psd', ifos=workflow.ifos,
    node.add_input_list_opt('--input-files', psd_files)

    if len(workflow.ifos) > 1:
        node.new_output_file_opt(workflow.analysis_time, output_fmt,
                                 '--detector-avg-file')

    node.new_multiifo_output_list_opt('--time-avg-file', workflow.ifos,
                                 workflow.analysis_time, output_fmt, tags=tags)

# keep namespace clean
__all__ = ['make_psd_file', 'make_average_psd', 'setup_psd_calculate', 'merge_psds']

# Copyright (C) 2015 Larne Pekowsky
#
#
#

#
#
#                                   Preamble
#
#

"""
This module is responsible for setting up the psd files used by CBC
workflows.
"""

# FIXME: Is this module still relevant for any code? Can it be removed?

import os
from pycbc.workflow.core import File, FileList
from pycbc.workflow.core import make_analysis_dir, resolve_url_to_file

def setup_psd_workflow(workflow, science_segs, datafind_outs,
                             output_dir=None, tags=None):
    '''
    Setup static psd section of CBC workflow. At present this only supports pregenerated
    psd files, in the future these could be created within the workflow.

    science_segs : Keyed dictionary of glue.segmentlist objects
        scienceSegs[ifo] holds the science segments to be analysed for each
        ifo.
    output_dir : path string
        The directory where data products will be placed.

    Returns
    --------
    psd_files : pycbc.workflow.core.FileList
        The FileList holding the psd files, 0 or 1 per ifo
    '''
        tags = []
    logging.info("Entering static psd module.")

    # Parse for options in ini file.
    try:
        psdMethod = cp.get_opt_tags("workflow-psd", "psd-method",
                                     tags)
    except:
        # Predefined PSD sare optional, just return an empty list if not
        # provided.
        return FileList([])

    if psdMethod == "PREGENERATED_FILE":
        logging.info("Setting psd from pre-generated file(s).")
        psd_files = setup_psd_pregenerated(workflow, tags=tags)
    else:
        errMsg = "PSD method not recognized. Only "
        errMsg += "PREGENERATED_FILE is currently supported."

    logging.info("Leaving psd module.")
    return psd_files

def setup_psd_pregenerated(workflow, tags=None):
    '''
    Setup CBC workflow to use pregenerated psd files.
    The file given in cp.get('workflow','pregenerated-psd-file-(ifo)') will
    be used as the --psd-file argument to geom_nonspinbank, geom_aligned_bank
    and pycbc_plot_psd_file.

    Returns
    --------
        The FileList holding the gating files
    '''
        tags = []

    global_seg = workflow.analysis_time
    file_attrs = {'segs': global_seg, 'tags': tags}

    # Check for one psd for all ifos
    try:
        pre_gen_file = cp.get_opt_tags('workflow-psd',
                        'psd-pregenerated-file', tags)
        file_attrs['ifos'] = workflow.ifos
        curr_file = resolve_url_to_file(pre_gen_file, attrs=file_attrs)
        psd_files.append(curr_file)
        # Check for one psd per ifo
            try:
                pre_gen_file = cp.get_opt_tags('workflow-psd',
                                'psd-pregenerated-file-%s' % ifo.lower(),
                                tags)
                file_attrs['ifos'] = [ifo]
                curr_file = resolve_url_to_file(pre_gen_file, attrs=file_attrs)
                psd_files.append(curr_file)

            except ConfigParser.Error:
                # It's unlikely, but not impossible, that only some ifos
                # will have pregenerated PSDs
                logging.warn("No psd file specified for IFO %s." % (ifo,))
                pass

#
#
#

#
#
#                                   Preamble
#
#
"""
This module is responsible for setting up the segment generation stage of
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/ahope/segments.html
"""

import os, sys, shutil, stat, copy, itertools
from six.moves.urllib.parse import urljoin, urlunparse
import lal
from ligo.segments import utils as segmentsUtils
from pycbc.workflow.core import Executable, FileList, Node, SegFile, make_analysis_dir, make_external_call, File
from pycbc.workflow.core import resolve_url
from pycbc.workflow.jobsetup import LigolwAddExecutable, LigoLWCombineSegsExecutable

def get_science_segments(workflow, out_dir, tags=None):
    """
    Get the analyzable segments after applying ini specified vetoes.

    workflow : Workflow object
        Instance of the workflow object
    out_dir : path
        Location to store output files
        Used to retrieve subsections of the ini file for
        configuration options.

    Returns
    --------
    sci_seg_file : workflow.core.SegFile instance
        The segment file combined from all ifos containing the science segments.
    sci_segs : Ifo keyed dict of ligo.segments.segmentlist instances
        The science segs for each ifo, keyed by ifo
    sci_seg_name : str
        The name with which science segs are stored in the output XML file.
    """
        tags = []
    logging.info('Starting generation of science segments')

    start_time = workflow.analysis_time[0]
    end_time = workflow.analysis_time[1]

    sci_seg_name = "SCIENCE"
    sci_segs = {}
    sci_seg_dict = segments.segmentlistdict()
    sci_seg_summ_dict = segments.segmentlistdict()

        curr_sci_segs, curr_sci_xml, curr_seg_name = get_sci_segs_for_ifo(ifo,
                              workflow.cp, start_time, end_time, out_dir, tags)
        sci_seg_dict[ifo + ':' + sci_seg_name] = curr_sci_segs
        sci_segs[ifo] = curr_sci_segs
        sci_seg_summ_dict[ifo + ':' + sci_seg_name] = \
                          curr_sci_xml.seg_summ_dict[ifo + ':' + curr_seg_name]
    sci_seg_file = SegFile.from_segment_list_dict(sci_seg_name,
                                          sci_seg_dict, extension='xml',
                                          valid_segment=workflow.analysis_time,
                                          seg_summ_dict=sci_seg_summ_dict,
                                          directory=out_dir, tags=tags)
    logging.info('Done generating science segments')
    return sci_seg_file, sci_segs, sci_seg_name

def get_files_for_vetoes(workflow, out_dir,
                         runtime_names=None, in_workflow_names=None, tags=None):
    """
    Get the various sets of veto segments that will be used in this analysis.

    runtime_names : list
        Veto category groups with these names in the [workflow-segment] section
        of the ini file will be generated now.
    in_workflow_names : list
        of the ini file will be generated in the workflow. If a veto category
        appears here and in runtime_names, it will be generated now.

    Returns
    --------
    veto_seg_files : FileList
        List of veto segment files generated
    """
        tags = []
    if runtime_names is None:
        runtime_names = []
    if in_workflow_names is None:
        in_workflow_names = []
    logging.info('Starting generating veto files for analysis')
    save_veto_definer(workflow.cp, out_dir, tags)

    now_cat_sets = []
    for name in runtime_names:
        cat_sets = parse_cat_ini_opt(workflow.cp.get_opt_tags(
                                              'workflow-segments', name, tags))
        now_cat_sets.extend(cat_sets)

    now_cats = set()
    for cset in now_cat_sets:
        now_cats = now_cats.union(cset)

    later_cat_sets = []
    for name in in_workflow_names:
        later_cat_sets.extend(cat_sets)

    later_cats = set()
    for cset in later_cat_sets:
        later_cats = later_cats.union(cset)
        # Avoid duplication
        later_cats = later_cats - now_cats

    veto_gen_job = create_segs_from_cats_job(workflow.cp, out_dir,
                                             workflow.ifo_string, tags=tags)

    cat_files = FileList()
        for category in now_cats:
            cat_files.append(get_veto_segs(workflow, ifo,
                                        cat_to_veto_def_cat(category),
                                        start_time, end_time, out_dir,
                                        veto_gen_job, execute_now=True,
                                        tags=tags))

        for category in later_cats:
                                        veto_gen_job, tags=tags,
                                        execute_now=False))

    logging.info('Done generating veto segments')
    return cat_files

def get_analyzable_segments(workflow, sci_segs, cat_files, out_dir, tags=None):
    """
    Get the analyzable segments after applying ini specified vetoes and any
    other restrictions on the science segs, e.g. a minimum segment length, or
    demanding that only coincident segments are analysed.

    sci_segs : Ifo-keyed dictionary of glue.segmentlists
        The science segments for each ifo to which the vetoes, or any other
        restriction, will be applied.
    cat_files : FileList of SegFiles
        The category veto files generated by get_veto_segs

    Returns
    --------
    sci_ok_seg_file : workflow.core.SegFile instance
        The segment file combined from all ifos containing the analyzable
        science segments.
    sci_ok_segs : Ifo keyed dict of ligo.segments.segmentlist instances
        The analyzable science segs for each ifo, keyed by ifo
    sci_ok_seg_name : str
        The name with which analyzable science segs are stored in the output
        XML file.
    """
        tags = []
    logging.info('Starting reducing to analysable science segments')

    sci_ok_seg_name = "SCIENCE_OK"
    sci_ok_seg_dict = segments.segmentlistdict()
    sci_ok_segs = {}

    cat_sets = parse_cat_ini_opt(workflow.cp.get_opt_tags('workflow-segments',
                                                'segments-science-veto', tags))
    if len(cat_sets) > 1:
        raise ValueError('Provide only 1 category group to determine'
                         ' analyzable segments')
    cat_set = cat_sets[0]

        curr_segs = copy.copy(sci_segs[ifo])
        files = cat_files.find_output_with_ifo(ifo)
        for category in cat_set:
            veto_def_cat = cat_to_veto_def_cat(category)
            file_list = files.find_output_with_tag('VETO_CAT%d' %(veto_def_cat))
            if len(file_list) > 1:
                err_msg = "Found more than one veto file for %s " %(ifo,)
                err_msg += "and category %s." %(category,)
            if len(file_list) == 0:
                err_msg = "Found no veto files for %s " %(ifo,)
            curr_veto_file = file_list[0]
            cat_segs = curr_veto_file.return_union_seglist()
            curr_segs -= cat_segs
            curr_segs.coalesce()
        sci_ok_seg_dict[ifo + ':' + sci_ok_seg_name] = curr_segs

    sci_ok_seg_file = SegFile.from_segment_list_dict(sci_ok_seg_name,
                                          sci_ok_seg_dict, extension='xml',

    if workflow.cp.has_option_tags("workflow-segments",
                          "segments-minimum-segment-length", tags):
        min_seg_length = int( workflow.cp.get_opt_tags("workflow-segments",
                              "segments-minimum-segment-length", tags) )
        sci_ok_seg_file.remove_short_sci_segs(min_seg_length)

    # FIXME: Another test we can do is limit to coinc time +/- some window
    #        this should *not* be set through segments-method, but currently
    #        is not implemented
    #segments_method = workflow.cp.get_opt_tags("workflow-segments",
    #                                  "segments-method", tags)
    #if segments_method == 'ALL_SINGLE_IFO_TIME':
    #    pass
    #elif segments_method == 'COINC_TIME':
    #    cum_segs = None
    #    for ifo in sci_segs:
    #        if cum_segs is not None:
    #            cum_segs = (cum_segs & sci_segs[ifo]).coalesce()
    #        else:
    #            cum_segs = sci_segs[ifo]
    #
    #        sci_segs[ifo] = cum_segs
    #else:
    #    raise ValueError("Invalid segments-method, %s. Options are "
    #                     "ALL_SINGLE_IFO_TIME and COINC_TIME" % segments_method)

        sci_ok_segs[ifo] = \
                      sci_ok_seg_file.segment_dict[ifo + ':' + sci_ok_seg_name]

    logging.info('Done generating analyzable science segments')
    return sci_ok_seg_file, sci_ok_segs, sci_ok_seg_name

def get_cumulative_veto_group_files(workflow, option, cat_files,
                                    out_dir, execute_now=True, tags=None):
    """
    Get the cumulative veto files that define the different backgrounds
    we want to analyze, defined by groups of vetos.

    option : str
        ini file option to use to get the veto groups
    execute_now : Boolean
        If true outputs are generated at runtime. Else jobs go into the workflow
        and are generated then.

    Returns
    --------
    seg_files : workflow.core.FileList instance
        The cumulative segment files for each veto group.
        The segment names for the corresponding seg_file
    cat_files : workflow.core.FileList instance
        The list of individual category veto files
    """
        tags = []
    logging.info("Starting generating vetoes for groups in %s" %(option))

                                            option, tags))

    cum_seg_files = FileList()
    names = []
    for cat_set in cat_sets:
        segment_name = "CUMULATIVE_CAT_%s" % (''.join(sorted(cat_set)))
        logging.info('getting information for %s' % segment_name)
        categories = [cat_to_veto_def_cat(c) for c in cat_set]

        cum_seg_files += [get_cumulative_segs(workflow, categories, cat_files,
                          out_dir, execute_now=execute_now,
                          segment_name=segment_name, tags=tags)]
        names.append(segment_name)

    logging.info("Done generating vetoes for groups in %s" %(option))

    return cum_seg_files, names, cat_files

def setup_segment_generation(workflow, out_dir, tag=None):
    """
    This function is the gateway for setting up the segment generation steps in a
    workflow. It is designed to be able to support multiple ways of obtaining
    these segments and to combine/edit such files as necessary for analysis.
    The current modules have the capability to generate files at runtime or to
    generate files that are not needed for workflow generation within the workflow.

        This instance also contains the ifos for which to attempt to obtain
        segments for this analysis and the start and end times to search for
        segments over.
    tag : string, optional (default=None)
        Use this to specify a tag. This can be used if this module is being
        options in [workflow-datafind-${TAG}] rather than [workflow-datafind]). This
        is also used to tag the Files returned by the class to uniqueify

    Returns
    -------
    segsToAnalyse : dictionay of ifo-keyed glue.segment.segmentlist instances
        This will contain the times that your code should analyse. By default this
        is science time - CAT_1 vetoes. (This default could be changed if desired)
    segFilesList : pycbc.workflow.core.FileList of SegFile instances
        These are representations of the various segment files that were constructed
        at this stage of the workflow and may be needed at later stages of the
        analysis (e.g. for performing DQ vetoes). If the file was generated at
        run-time the segment lists contained within these files will be an attribute
        of the instance. (If it will be generated in the workflow it will not be
        because I am not psychic).
    """
    logging.info("Entering segment generation module")

    segmentsMethod = cp.get_opt_tags("workflow-segments",
                                     "segments-method", [tag])
    # These only needed if calling setup_segment_gen_mixed
    if segmentsMethod in ['AT_RUNTIME','CAT2_PLUS_DAG','CAT3_PLUS_DAG',
                          'CAT4_PLUS_DAG']:
        veto_cats = cp.get_opt_tags("workflow-segments",
                                    "segments-veto-categories", [tag])
        max_veto_cat = max([int(c) for c in veto_cats.split(',')])
        veto_categories = range(1, max_veto_cat + 1)
        if cp.has_option_tags("workflow-segments",
                              "segments-generate-coincident-segments", [tag]):
            generate_coincident_segs = True
        else:
            generate_coincident_segs = False
        # Need to curl the veto-definer file
        vetoDefUrl = cp.get_opt_tags("workflow-segments",
                                     "segments-veto-definer-url", [tag])
        vetoDefBaseName = os.path.basename(vetoDefUrl)
        vetoDefNewPath = os.path.join(out_dir, vetoDefBaseName)
        resolve_url(vetoDefUrl,out_dir)
        # and update location
        cp.set("workflow-segments", "segments-veto-definer-file",
                vetoDefNewPath)

    if cp.has_option_tags("workflow-segments",
                          "segments-minimum-segment-length", [tag]):
        minSegLength = int( cp.get_opt_tags("workflow-segments",
                           "segments-minimum-segment-length", [tag]) )
    else:
        minSegLength = 0

    if segmentsMethod == "AT_RUNTIME":
        max_veto = 1000
    elif segmentsMethod == "CAT2_PLUS_DAG":
        max_veto = 1
    elif segmentsMethod == "CAT3_PLUS_DAG":
        max_veto = 2
    elif segmentsMethod == "CAT4_PLUS_DAG":
        max_veto = 3
    else:
        msg = "Entry segments-method in [workflow-segments] does not have "
        msg += "expected value. Valid values are AT_RUNTIME, CAT4_PLUS_DAG, "
        msg += "CAT2_PLUS_DAG or CAT3_PLUS_DAG."

    logging.info("Generating segments with setup_segment_gen_mixed")
    segFilesList = setup_segment_gen_mixed(workflow, veto_categories,
                             out_dir, max_veto, tag=tag,
                             generate_coincident_segs=generate_coincident_segs)
    logging.info("Segments obtained")

    # This creates the segsToAnalyse from the segFilesList. Currently it uses
    # the 'SCIENCE_OK' segFilesList, which is science - CAT_1 in
    # setup_segment_gen_mixed.
    # This also applies the minimum science length
    segsToAnalyse = {}
        analSegs = segFilesList.find_output_with_ifo(ifo)
        analSegs = analSegs.find_output_with_tag('SCIENCE_OK')
        assert len(analSegs) == 1
        analSegs = analSegs[0]
        if analSegs.segment_list:
            if minSegLength:
                analSegs.remove_short_sci_segs(minSegLength)
                analSegs.to_segment_xml(override_file_if_exists=True)
            segsToAnalyse[ifo] = analSegs.segment_list
        else:
            msg = "No science segments found for ifo %s. " %(ifo)
            msg += "If this is unexpected check the files that were dumped "
            msg += "in the %s directory. Also the " %(out_dir)
            msg += "commands that can be used to reproduce some of these "
            msg += "in %s/*.sh" %(os.path.join(out_dir,'logs'))

    logging.info("Leaving segment generation module")
    return segsToAnalyse, segFilesList

def setup_segment_gen_mixed(workflow, veto_categories, out_dir,
                            maxVetoAtRunTime, tag=None,
                            generate_coincident_segs=True):
    """
    This function will generate veto files for each ifo and for each veto
    category.
    It can generate these vetoes at run-time or in the workflow (or do some at
    run-time and some in the workflow). However, the CAT_1 vetoes and science
    time must be generated at run time as they are needed to plan the workflow.
    CATs 2 and higher *may* be needed for other workflow construction.
    It can also combine these files to create a set of cumulative,
    multi-detector veto files, which can be used in ligolw_thinca and in
    pipedown. Again these can be created at run time or within the workflow.

    veto_categories : list of ints
        List of veto categories to generate segments for. If this stops being
        integers, this can be changed here.
    maxVetoAtRunTime : int
        Generate veto files at run time up to this category. Veto categories
        beyond this in veto_categories will be generated in the workflow.
        If we move to a model where veto
        categories are not explicitly cumulative, this will be rethought.
    generate_coincident_segs : boolean, optional (default = True)
        If given this module will generate a set of coincident, cumulative veto
        files that can be used with ligolw_thinca and pipedown.

    Returns
    -------
    segFilesList : dictionary of pycbc.workflow.core.SegFile instances
        These are representations of the various segment files that were
        constructed
        run-time the segment lists contained within these files will be an
        attribute
        of the instance. (If it will be generated in the workflow it will
        not be because I am not psychic).
    """
    segFilesList = FileList([])
    segValidSeg = workflow.analysis_time
    # Will I need to add some jobs to the workflow?
    vetoGenJob = create_segs_from_cats_job(cp, out_dir, workflow.ifo_string)

        logging.info("Generating science segments for ifo %s" %(ifo))
        currSciSegs, currSciXmlFile, _ = get_sci_segs_for_ifo(ifo, cp,
                                        start_time, end_time, out_dir, tags=tag)
        segFilesList.append(currSciXmlFile)

        for category in veto_categories:
            if category > maxVetoAtRunTime:
                msg = "Adding creation of CAT_%d segments " %(category)
                msg += "for ifo %s to workflow." %(ifo)
                logging.info(msg)
                execute_status = False

            if category <= maxVetoAtRunTime:
                logging.info("Generating CAT_%d segments for ifo %s." \
                             %(category,ifo))
                execute_status = True

            currVetoXmlFile = get_veto_segs(workflow, ifo, category,
                                                start_time, end_time, out_dir,
                                                vetoGenJob,
                                                execute_now=execute_status)

            segFilesList.append(currVetoXmlFile)
            # Store the CAT_1 veto segs for use below
            if category == 1:
                cat1Segs = currVetoXmlFile.return_union_seglist()

        analysedSegs = currSciSegs - cat1Segs
        analysedSegs.coalesce()
        analysedSegDict = segments.segmentlistdict()
        analysedSegDict[ifo + ':SCIENCE_OK'] = analysedSegs
        analysedXmlFile = os.path.join(out_dir,
                             "%s-SCIENCE_OK_SEGMENTS.xml" %(ifo.upper()) )
        currUrl = urlunparse(['file', 'localhost', analysedXmlFile,
                              None, None, None])
        if tag:
            currTags = [tag, 'SCIENCE_OK']
        else:
            currTags = ['SCIENCE_OK']
        currFile = SegFile(ifo, 'SEGMENTS', analysedSegs,
                           segment_dict=analysedSegDict, file_url=currUrl,
                           tags=currTags)
        segFilesList.append(currFile)
        currFile.to_segment_xml()

    if generate_coincident_segs:
        # Need to make some combined category veto files to use when vetoing
        # segments and triggers.
        ifo_string = workflow.ifo_string
        categories = []
        cum_cat_files = []
            categories.append(category)
            # Set file name in workflow standard
            if tag:
                currTags = [tag]
            else:
                currTags = []

            # And actually make the file (or queue it in the workflow)
            logging.info("Generating combined, cumulative CAT_%d segments."\
                             %(category))
            else:
            currSegFile = get_cumulative_segs(workflow, categories,
                                segFilesList, out_dir,
                                execute_now=execute_status, tags=currTags)

            segFilesList.append(currSegFile)
            cum_cat_files.append(currSegFile)
        # Create a combined file
        # Set file tag in workflow standard
        if tag:
            currTags = [tag, 'COMBINED_CUMULATIVE_SEGMENTS']
        else:
            currTags = ['COMBINED_CUMULATIVE_SEGMENTS']

        combined_veto_file = os.path.join(out_dir,
                               '%s-CUMULATIVE_ALL_CATS_SEGMENTS.xml' \
                               %(ifo_string) )
        curr_url = urlunparse(['file', 'localhost',
                               combined_veto_file, None, None, None])
        curr_file = SegFile(ifo_string, 'SEGMENTS', segValidSeg,
                            file_url=curr_url, tags=currTags)

                break
        else:
            execute_status = False
        add_cumulative_files(workflow, curr_file, cum_cat_files, out_dir,
                             execute_now=execute_status)
        segFilesList.append(curr_file)

    return segFilesList

def get_sci_segs_for_ifo(ifo, cp, start_time, end_time, out_dir, tags=None):
    """
    Obtain science segments for the selected ifo

        The string describing the ifo to obtain science times for.
    start_time : gps time (either int/LIGOTimeGPS)
        The time at which to begin searching for segments.
    end_time : gps time (either int/LIGOTimeGPS)
        The time at which to stop searching for segments.

    Returns
    --------
    sci_segs : ligo.segments.segmentlist
        The segmentlist generated by this call
    sci_xml_file : pycbc.workflow.core.SegFile
        The workflow File object corresponding to this science segments file.
    out_sci_seg_name : string
        The name of the output segment list in the output XML file.

    """
        tags = []
    seg_valid_seg = segments.segment([start_time,end_time])
    sci_seg_name = cp.get_opt_tags(
        "workflow-segments", "segments-%s-science-name" %(ifo.lower()), tags)
    sci_seg_url = cp.get_opt_tags(
        "workflow-segments", "segments-database-url", tags)
    # NOTE: ligolw_segment_query returns slightly strange output. The output
    #       segment list is put in with name "RESULT". So this is hardcoded here
    out_sci_seg_name = "RESULT"
    if tags:
        sci_xml_file_path = os.path.join(
            out_dir, "%s-SCIENCE_SEGMENTS_%s.xml" \
                     %(ifo.upper(), '_'.join(tags)))
        tag_list=tags + ['SCIENCE']
    else:
            out_dir, "%s-SCIENCE_SEGMENTS.xml" %(ifo.upper()) )
        tag_list = ['SCIENCE']

    if file_needs_generating(sci_xml_file_path, cp, tags=tags):
        seg_find_call = [ resolve_url(cp.get("executables","segment_query"),
                permissions=stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR),
            "--query-segments",
            "--segment-url", sci_seg_url,
            "--gps-start-time", str(start_time),
            "--gps-end-time", str(end_time),
            "--include-segments", sci_seg_name,
            "--output-file", sci_xml_file_path ]

        make_external_call(seg_find_call, out_dir=os.path.join(out_dir,'logs'),
                                out_basename='%s-science-call' %(ifo.lower()) )

    # Yes its yucky to generate a file and then read it back in.
    sci_xml_file_path = os.path.abspath(sci_xml_file_path)
    sci_xml_file = SegFile.from_segment_xml(sci_xml_file_path, tags=tag_list,
                                        valid_segment=seg_valid_seg)
    #       segment_summary output does not use RESULT. Therefore move the
    #       segment_summary across.
    sci_xml_file.seg_summ_dict[ifo.upper() + ":" + out_sci_seg_name] = \
             sci_xml_file.seg_summ_dict[':'.join(sci_seg_name.split(':')[0:2])]

    sci_segs = sci_xml_file.return_union_seglist()
    return sci_segs, sci_xml_file, out_sci_seg_name

def get_veto_segs(workflow, ifo, category, start_time, end_time, out_dir,
                  veto_gen_job, tags=None, execute_now=False):
    """
    Obtain veto segments for the selected ifo and veto category and add the job
    to generate this to the workflow.

        An instance of the Workflow class that manages the workflow.
        The string describing the ifo to generate vetoes for.
    category : int
        The veto category to generate vetoes for.
    vetoGenJob : Job
        The veto generation Job class that will be used to create the Node.
    execute_now : boolean, optional
        If true, jobs are executed immediately. If false, they are added to the
        workflow to be run later.

    Returns
    --------
    veto_def_file : pycbc.workflow.core.SegFile
        The workflow File object corresponding to this DQ veto file.
    """
        tags = []
    # FIXME: This job needs an internet connection and X509_USER_PROXY
    #        For internet connection, it may need a headnode (ie universe local)
    #        For X509_USER_PROXY, I don't know what pegasus is doing
    node = Node(veto_gen_job)
    node.add_opt('--veto-categories', str(category))
    node.add_opt('--ifo-list', ifo)
    node.add_opt('--gps-start-time', str(start_time))
    node.add_opt('--gps-end-time', str(end_time))
    if tags:
        veto_xml_file_name = "%s-VETOTIME_CAT%d_%s-%d-%d.xml" \
                               %(ifo, category, '_'.join(tags), start_time,
                                 end_time-start_time)
    else:
        veto_xml_file_name = "%s-VETOTIME_CAT%d-%d-%d.xml" \
                         %(ifo, category, start_time, end_time-start_time)
    veto_xml_file_path = os.path.abspath(os.path.join(out_dir,
                                         veto_xml_file_name))
    curr_url = urlunparse(['file', 'localhost',
                           veto_xml_file_path, None, None, None])
    if tags:
        curr_tags = tags + ['VETO_CAT%d' %(category)]
    else:
        curr_tags = ['VETO_CAT%d' %(category)]

    if file_needs_generating(veto_xml_file_path, workflow.cp, tags=tags):
        if execute_now:
            workflow.execute_node(node, verbatim_exe = True)
            veto_xml_file = SegFile.from_segment_xml(veto_xml_file_path,
                                                 valid_segment=seg_valid_seg)
        else:
            veto_xml_file = SegFile(ifo, 'SEGMENTS', seg_valid_seg,
                                    file_url=curr_url, tags=curr_tags)
            node._add_output(veto_xml_file)
    else:
        veto_xml_file = SegFile.from_segment_xml(veto_xml_file_path,
    return veto_xml_file

def create_segs_from_cats_job(cp, out_dir, ifo_string, tags=None):
    """
    This function creates the CondorDAGJob that will be used to run
    ligolw_segments_from_cats as part of the workflow

        The in-memory representation of the configuration (.ini) files
        Directory in which to put output files
        String containing all active ifos, ie. "H1L1V1"
    tag : list of strings, optional (default=None)
        Use this to specify a tag(s). This can be used if this module is being

    Returns
    --------
    job : Job instance
        The Job instance that will run segments_from_cats jobs
    """
        tags = []

    seg_server_url = cp.get_opt_tags("workflow-segments",
                                   "segments-database-url", tags)
    veto_def_file = cp.get_opt_tags("workflow-segments",
                                  "segments-veto-definer-file", tags)

    job = Executable(cp, 'segments_from_cats', universe='local',
                               ifos=ifo_string, out_dir=out_dir, tags=tags)
    job.add_opt('--separate-categories')
    job.add_opt('--segment-url', seg_server_url)

    job.add_opt('--veto-file', veto_def_file)
    # FIXME: Would like the proxy in the Workflow instance
    # FIXME: Explore using the x509 condor commands
    # If the user has a proxy set in the environment, add it to the job
    return job

def get_cumulative_segs(workflow, categories, seg_files_list, out_dir,
                        tags=None, execute_now=False, segment_name=None):
    """
    Function to generate one of the cumulative, multi-detector segment files
    as part of the workflow.

    categories : int
        The veto categories to include in this cumulative veto.
    seg_files_list : Listionary of SegFiles
        The list of segment files to be used as input for combining.
        The directory to write output to.
    tags : list of strings, optional
        A list of strings that is used to identify this job
    segment_name : str
        The name of the combined, cumulative segments in the output file.
    """
        tags = []
    add_inputs = FileList([])
    valid_segment = workflow.analysis_time
    if segment_name is None:
        segment_name = 'VETO_CAT%d_CUMULATIVE' % (categories[-1])
    # calculate the cumulative veto files for a given ifo
        cum_job = LigoLWCombineSegsExecutable(cp, 'ligolw_combine_segments',
                       out_dir=out_dir, tags=[segment_name]+tags, ifos=ifo)
        inputs = []
        files = seg_files_list.find_output_with_ifo(ifo)
        for category in categories:
            file_list = files.find_output_with_tag('VETO_CAT%d' %(category))
            inputs+=file_list

        cum_node  = cum_job.create_node(valid_segment, inputs, segment_name)
        if file_needs_generating(cum_node.output_files[0].cache_entry.path,
                                 workflow.cp, tags=tags):
            if execute_now:
                workflow.execute_node(cum_node)
            else:
                workflow.add_node(cum_node)
        else:
            cum_node.executed = True
            for fil in cum_node._outputs:
                fil.node = None
                fil.PFN(urljoin('file:', pathname2url(fil.storage_path)),
                        site='local')
        add_inputs += cum_node.output_files

    # add cumulative files for each ifo together
    name = '%s_VETO_SEGMENTS' %(segment_name)
    outfile = File(workflow.ifos, name, workflow.analysis_time,
                                            directory=out_dir, extension='xml',
                                            tags=[segment_name] + tags)
    add_job = LigolwAddExecutable(cp, 'llwadd', ifos=ifo, out_dir=out_dir,
    add_node = add_job.create_node(valid_segment, add_inputs, output=outfile)
    if file_needs_generating(add_node.output_files[0].cache_entry.path,
                             workflow.cp, tags=tags):
            workflow.execute_node(add_node)
        else:
            workflow.add_node(add_node)
    else:
        add_node.executed = True
        for fil in add_node._outputs:
    return outfile

def add_cumulative_files(workflow, output_file, input_files, out_dir,
                         execute_now=False, tags=None):
    """
    Function to combine a set of segment files into a single one. This function
    will not merge the segment lists but keep each separate.

    output_file: pycbc.workflow.core.File
        The output file object
    input_files: pycbc.workflow.core.FileList
        This list of input segment files
    """
        tags = []
    llwadd_job = LigolwAddExecutable(workflow.cp, 'llwadd',
                       ifo=output_file.ifo_list, out_dir=out_dir, tags=tags)
    add_node = llwadd_job.create_node(output_file.segment, input_files,
                                   output=output_file)
        else:
    else:
    return add_node.output_files[0]

def find_playground_segments(segs):
    '''Finds playground time in a list of segments.

      Playground segments include the first 600s of every 6370s stride starting
      at GPS time 729273613.

      Parameters
      ----------
      segs : segmentfilelist
          A segmentfilelist to find playground segments.

      Returns
      -------
      outlist : segmentfilelist
          A segmentfilelist with all playground segments during the input
          segmentfilelist (ie. segs).
    '''

    start_s2 = 729273613
    playground_stride = 6370
    playground_length = 600
    outlist = segments.segmentlist()

    # loop over segments
    for seg in segs:
        start = seg[0]
        end = seg[1]

        # the first playground segment whose end is after the start of seg
        playground_start = start_s2 + playground_stride * ( 1 + \
                     int(start-start_s2-playground_length) / playground_stride)

        while playground_start < end:
            # find start of playground segment
            if playground_start > start:
                ostart = playground_start
            else:
                ostart = start

            playground_end = playground_start + playground_length

            # find end of playground segment
            if playground_end < end:
                oend = playground_end
            else:
                oend = end

            # append segment
            x = segments.segment(ostart, oend)
            outlist.append(x)

            # increment
            playground_start = playground_start + playground_stride

    return outlist

def get_triggered_coherent_segment(workflow, sciencesegs):
    """
    Construct the coherent network on and off source segments. Can switch to
    construction of segments for a single IFO search when coherent segments
    are insufficient for a search.

        The workflow instance that the calculated segments belong to.
    sciencesegs : dict
        Dictionary of all science segments within analysis time.

    Returns
    --------
    onsource : ligo.segments.segmentlistdict
        A dictionary containing the on source segments for network IFOs

    offsource : ligo.segments.segmentlistdict
        A dictionary containing the off source segments for network IFOs
    """

    # Load parsed workflow config options
    triggertime = int(os.path.basename(cp.get('workflow', 'trigger-time')))
    minduration = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                              'min-duration')))
    maxduration = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                              'max-duration')))
    onbefore = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                           'on-before')))
    onafter = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                          'on-after')))
    padding = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                          'pad-data')))
    if cp.has_option("workflow-condition_strain", "do-gating"):
        padding += int(os.path.basename(cp.get("condition_strain",
                                               "pad-data")))
    quanta = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                         'quanta')))

    # Check available data segments meet criteria specified in arguments
    commonsegs = sciencesegs.extract_common(sciencesegs.keys())
    offsrclist = commonsegs[tuple(commonsegs.keys())[0]]
    if len(offsrclist) > 1:
        logging.info("Removing network segments that do not contain trigger "
                     "time")
        for seg in offsrclist:
            if triggertime in seg:
                offsrc = seg
    else:
        offsrc = offsrclist[0]

    if abs(offsrc) < minduration + 2 * padding:
        fail = segments.segment([triggertime - minduration / 2. - padding,
                                 triggertime + minduration / 2. + padding])
        logging.warning("Available network segment shorter than minimum "
                        "allowed duration.")
        return None, fail

    # Will segment duration be the maximum desired length or not?
    if abs(offsrc) >= maxduration + 2 * padding:
        logging.info("Available network science segment duration (%ds) is "
                     "greater than the maximum allowed segment length (%ds). "
                     "Truncating..." % (abs(offsrc), maxduration))
    else:
                     "less than the maximum allowed segment length (%ds)."
                     % (abs(offsrc), maxduration))

    logging.info("%ds of padding applied at beginning and end of segment."
                 % padding)

    # Construct on-source
    onstart = triggertime - onbefore
    onend = triggertime + onafter
    oncentre = onstart + ((onbefore + onafter) / 2)
    onsrc = segments.segment(onstart, onend)
    logging.info("Constructed ON-SOURCE: duration %ds (%ds before to %ds after"
                 " trigger)."
                 % (abs(onsrc), triggertime - onsrc[0],
                    onsrc[1] - triggertime))
    onsrc = segments.segmentlist([onsrc])

    # Maximal, centred coherent network segment
    idealsegment = segments.segment(int(oncentre - padding -
                                    0.5 * maxduration),
                                    int(oncentre + padding +
                                    0.5 * maxduration))

    # Construct off-source
    if (idealsegment in offsrc):
        offsrc = idealsegment

    elif idealsegment[1] not in offsrc:
        offsrc &= segments.segment(offsrc[1] - maxduration - 2 * padding,
                                   offsrc[1])

    elif idealsegment[0] not in offsrc:
        offsrc &= segments.segment(offsrc[0],
                                   offsrc[0] + maxduration + 2 * padding)

    # Trimming off-source
    excess = (abs(offsrc) - 2 * padding) % quanta
    if excess != 0:
        logging.info("Trimming %ds excess time to make OFF-SOURCE duration a "
                     "multiple of %ds" % (excess, quanta))
        offset = (offsrc[0] + abs(offsrc) / 2.) - oncentre
        if 2 * abs(offset) > excess:
            if offset < 0:
                offsrc &= segments.segment(offsrc[0] + excess,
                                           offsrc[1])
            elif offset > 0:
                offsrc &= segments.segment(offsrc[0],
                                           offsrc[1] - excess)
            assert abs(offsrc) % quanta == 2 * padding
        else:
            logging.info("This will make OFF-SOURCE symmetrical about trigger "
                         "time.")
            start = int(offsrc[0] - offset + excess / 2)
            end = int(offsrc[1] - offset - round(float(excess) / 2))
            offsrc = segments.segment(start, end)

    logging.info("Constructed OFF-SOURCE: duration %ds (%ds before to %ds "
                 "after trigger)."
                 % (abs(offsrc) - 2 * padding,
                    triggertime - offsrc[0] - padding,
                    offsrc[1] - triggertime - padding))
    offsrc = segments.segmentlist([offsrc])

    # Put segments into segmentlistdicts
    onsource = segments.segmentlistdict()
    offsource = segments.segmentlistdict()
    ifos = ''
    for iifo in sciencesegs.keys():
        ifos += str(iifo)
        onsource[iifo] = onsrc
        offsource[iifo] = offsrc

    return onsource, offsource

def generate_triggered_segment(workflow, out_dir, sciencesegs):

    if cp.has_option("workflow", "allow-single-ifo-search"):
        min_ifos = 1
    else:
        min_ifos = 2

    minbefore = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                            'min-before')))
    minafter = int(os.path.basename(cp.get('workflow-exttrig_segments',
                                           'min-after')))

    # How many IFOs meet minimum data requirements?
    min_seg = segments.segment(triggertime - onbefore - minbefore - padding,
                               triggertime + onafter + minafter + padding)
    scisegs = segments.segmentlistdict({ifo: sciencesegs[ifo]
            for ifo in sciencesegs.keys() if min_seg in sciencesegs[ifo]
            and abs(sciencesegs[ifo]) >= minduration})

    # Find highest number of IFOs that give an acceptable coherent segment
    num_ifos = len(scisegs.keys())
    while num_ifos >= min_ifos:
        # Consider all combinations for a given number of IFOs
        ifo_combos = itertools.combinations(scisegs.keys(), num_ifos)
        onsource = {}
        offsource = {}
        for ifo_combo in ifo_combos:
            ifos = "".join(ifo_combo)
            logging.info("Calculating optimal segment for %s.", ifos)
            segs = segments.segmentlistdict({ifo: scisegs[ifo]
                                             for ifo in ifo_combo})
            onsource[ifos], offsource[ifos] = get_triggered_coherent_segment(\
                    workflow, segs)

        # Which combination gives the longest coherent segment?
        valid_combs = [iifos for iifos in onsource.keys()
                       if onsource[iifos] is not None]

        if len(valid_combs) == 0:
            # If none, offsource dict will contain segments showing criteria
            # that have not been met, for use in plotting
            if len(offsource.keys()) > 1:
                seg_lens = {ifos: abs(next(offsource[ifos].values())[0])
                            for ifos in offsource.keys()}
                best_comb = max(seg_lens.iterkeys(),
                                key=(lambda key: seg_lens[key]))
            else:
                best_comb = tuple(offsource.keys())[0]
            logging.info("No combination of %d IFOs with suitable science "
                         "segment.", num_ifos)
        else:
            # Identify best analysis segment
            if len(valid_combs) > 1:
                            for ifos in valid_combs}
            else:
                best_comb = valid_combs[0]
            logging.info("Calculated science segments.")

            offsourceSegfile = os.path.join(out_dir, "offSourceSeg.txt")
            segmentsUtils.tosegwizard(open(offsourceSegfile, "w"),
                                      list(offsource[best_comb].values())[0])

            onsourceSegfile = os.path.join(out_dir, "onSourceSeg.txt")
            segmentsUtils.tosegwizard(file(onsourceSegfile, "w"),
                                      list(onsource[best_comb].values())[0])

            bufferleft = int(cp.get('workflow-exttrig_segments',
                                    'num-buffer-before'))
            bufferright = int(cp.get('workflow-exttrig_segments',
                                     'num-buffer-after'))
            onlen = onbefore + onafter
            bufferSegment = segments.segment(\
                    triggertime - onbefore - bufferleft * onlen,
                    triggertime + onafter + bufferright * onlen)
            bufferSegfile = os.path.join(out_dir, "bufferSeg.txt")
            segmentsUtils.tosegwizard(file(bufferSegfile, "w"),
                                      segments.segmentlist([bufferSegment]))

            return onsource[best_comb], offsource[best_comb]

        num_ifos -= 1

    logging.warning("No suitable science segments available.")
    try:
        return None, offsource[best_comb]
    except UnboundLocalError:
        return None, min_seg

def save_veto_definer(cp, out_dir, tags=None):
    """ Retrieve the veto definer file and save it locally

    """
        tags = []
    veto_def_url = cp.get_opt_tags("workflow-segments",
                                 "segments-veto-definer-url", tags)
    veto_def_base_name = os.path.basename(veto_def_url)
    veto_def_new_path = os.path.abspath(os.path.join(out_dir,
                                        veto_def_base_name))
    # Don't need to do this if already done
    resolve_url(veto_def_url,out_dir)

    # and update location
    cp.set("workflow-segments", "segments-veto-definer-file", veto_def_new_path)
    return veto_def_new_path

def parse_cat_ini_opt(cat_str):
    """ Parse a cat str from the ini file into a list of sets """
    if cat_str == "":
        return []

    cat_groups = cat_str.split(',')
    cat_sets = []
    for group in cat_groups:
        group = group.strip()
        cat_sets += [set(c for c in group)]
    return cat_sets

def cat_to_veto_def_cat(val):
    """ Convert a category character to the corresponding value in the veto
    definer file.

    str : single character string
        The input category character

    Returns
    -------
    pipedown_str : str
        The pipedown equivalent notation that can be passed to programs
    that expect this definition.
    """
    if val == '1':
        return 1
    if val == '2':
        return 2
    if val == '3':
        return 4
    if val == 'H':
        return 3
    else:
        raise ValueError('Invalid Category Choice')

def file_needs_generating(file_path, cp, tags=None):
    """
    This job tests the file location and determines if the file should be
    generated now or if an error should be raised. This uses the
    generate_segment_files variable, global to this module, which is described
    above and in the documentation.

    file_path : path
        Location of file to check
        The associated ConfigParser from which the
        segments-generate-segment-files variable is returned.
        It is recommended for most applications to use the default option by
        leaving segments-generate-segment-files blank, which will regenerate
        all segment files at runtime. Only use this facility if you need it.
        Choices are
        * 'always' : DEFAULT: All files will be generated even if they already exist.
        * 'if_not_present': Files will be generated if they do not already exist. Pre-existing files will be read in and used.
        * 'error_on_duplicate': Files will be generated if they do not already exist. Pre-existing files will raise a failure.
        * 'never': Pre-existing files will be read in and used. If no file exists the code will fail.

    Returns
    --------
    int
        1 = Generate the file. 0 = File already exists, use it. Other cases
        will raise an error.
    """
        tags = []
                          "segments-generate-segment-files", tags):
        value = cp.get_opt_tags("workflow-segments",
                                   "segments-generate-segment-files", tags)
        generate_segment_files = value
    else:
        generate_segment_files = 'always'

    # Does the file exist
    if os.path.isfile(file_path):
        if generate_segment_files in ['if_not_present', 'never']:
            return 0
        elif generate_segment_files == 'always':
            err_msg = "File %s already exists. " %(file_path,)
            err_msg += "Regenerating and overwriting."
            return 1
        elif generate_segment_files == 'error_on_duplicate':
            err_msg += "Refusing to overwrite file and exiting."
        else:
            err_msg = 'Global variable generate_segment_files must be one of '
            err_msg += '"always", "if_not_present", "error_on_duplicate", '
            err_msg += '"never". Got %s.' %(generate_segment_files,)
    else:
        if generate_segment_files in ['always', 'if_not_present',
                                      'error_on_duplicate']:
            return 1
        elif generate_segment_files == 'never':
            err_msg = 'File %s does not exist. ' %(file_path,)
        else:

def get_segments_file(workflow, name, option_name, out_dir):
    """Get cumulative segments from option name syntax for each ifo.

    Use syntax of configparser string to define the resulting segment_file
    e.x. option_name = +up_flag1,+up_flag2,+up_flag3,-down_flag1,-down_flag2
    Each ifo may have a different string and is stored separately in the file.
    Flags which add time must precede flags which subtract time.

    name: string
        Name of the segment list being created
    option_name: str
        Name of option in the associated config parser to get the flag list

    returns
    --------
    seg_file: pycbc.workflow.SegFile
        SegFile intance that points to the segment xml file on disk.
    """
    from pycbc.dq import query_str
    start = workflow.analysis_time[0]
    end = workflow.analysis_time[1]

    # Check for veto definer file
    veto_definer = None
    if cp.has_option("workflow-segments", "segments-veto-definer-url"):
        veto_definer = save_veto_definer(workflow.cp, out_dir, [])

    # Check for provided server
    server = "https://segments.ligo.org"
    if cp.has_option("workflow-segments", "segments-database-url"):
        server = cp.get("workflow-segments",
                                 "segments-database-url")

    source = "any"
    if cp.has_option("workflow-segments", "segments-source"):
        source = cp.get("workflow-segments", "segments-source")
    if source == "file":
        local_file_path = \
            resolve_url(cp.get("workflow-segments", option_name+"-file"))
        pfn = os.path.join(out_dir, os.path.basename(local_file_path))
        shutil.move(local_file_path, pfn)
        return SegFile.from_segment_xml(pfn)

    segs = {}
        flag_str = cp.get_opt_tags("workflow-segments", option_name, [ifo])
        key = ifo + ':' + name
        segs[key] = query_str(ifo, flag_str, start, end,
                              source=source, server=server,
        logging.info("%s: got %s flags", ifo, option_name)

    return SegFile.from_segment_list_dict(name, segs,
                                          extension='.xml',
                                          directory=out_dir)

#
#
#

#
#
#                                   Preamble
#
#

"""
This module is responsible for setting up the splitting output files stage of
"""

import os
from pycbc.workflow.jobsetup import (PycbcSplitBankExecutable,
        PycbcSplitBankXmlExecutable, PycbcSplitInspinjExecutable)

def select_splitfilejob_instance(curr_exe):
    """
    This function returns an instance of the class that is appropriate for
    splitting an output file up within workflow (for e.g. splitbank).

        The name of the Executable that is being used.
    curr_section : string
        The name of the section storing options for this executble

    Returns
    --------
    exe class : sub-class of pycbc.workflow.core.Executable
        The class that holds the utility functions appropriate
        for the given Executable. This class **must** contain
        * exe_class.create_job()
        and the job returned by this **must** contain
    """
    if curr_exe == 'pycbc_hdf5_splitbank':
        exe_class = PycbcSplitBankExecutable
    elif curr_exe == 'pycbc_splitbank':
        exe_class = PycbcSplitBankXmlExecutable
    elif curr_exe == 'pycbc_split_inspinj':
        exe_class = PycbcSplitInspinjExecutable
    else:
        err_string = "No class exists for Executable %s" %(curr_exe,)
        raise NotImplementedError(err_string)

    return exe_class

def setup_splittable_workflow(workflow, input_tables, out_dir=None, tags=None):
    '''
    This function aims to be the gateway for code that is responsible for taking
    some input file containing some table, and splitting into multiple files
    containing different parts of that table. For now the only supported operation
    is using lalapps_splitbank to split a template bank xml file into multiple
    template bank xml files.

        The Workflow instance that the jobs will be added to.
    input_tables : pycbc.workflow.core.FileList
        The input files to be split up.
        The directory in which output will be written.

    Returns
    --------
    split_table_outs : pycbc.workflow.core.FileList
        The list of split up files as output from this job.
    '''
        tags = []
    logging.info("Entering split output files module.")
    splitMethod = workflow.cp.get_opt_tags("workflow-splittable",
                                           "splittable-method", tags)

    if splitMethod == "IN_WORKFLOW":
        # Scope here for choosing different options
        logging.info("Adding split output file jobs to workflow.")
        split_table_outs = setup_splittable_dax_generated(workflow,
                input_tables, out_dir, tags)
    elif splitMethod == "NOOP":
        # Probably better not to call the module at all, but this option will
        # return the input file list.
        split_table_outs = input_tables
    else:
        errMsg = "Splittable method not recognized. Must be one of "
        errMsg += "IN_WORKFLOW or NOOP."

    logging.info("Leaving split output files module.")
    return split_table_outs

def setup_splittable_dax_generated(workflow, input_tables, out_dir, tags):
    '''
    Function for setting up the splitting jobs as part of the workflow.

    Returns
    --------
    '''

    # Get values from ini file
    try:
        num_splits = cp.get_opt_tags("workflow-splittable",
                                     "splittable-num-banks", tags)
    except BaseException:
        inj_interval = int(cp.get_opt_tags("workflow-splittable",
                                           "splitinjtable-interval", tags))
        if cp.has_option_tags("em_bright_filter", "max-keep", tags) and \
                cp.has_option("workflow-injections", "em-bright-only"):
            num_injs = int(cp.get_opt_tags("em_bright_filter", "max-keep",
                                           tags))
        else:
            num_injs = int(cp.get_opt_tags("workflow-injections", "num-injs",
                                           tags))
        inj_tspace = float(abs(workflow.analysis_time)) / num_injs
        num_splits = int(inj_interval // inj_tspace) + 1

    split_exe_tag = cp.get_opt_tags("workflow-splittable",
                                    "splittable-exe-tag", tags)
    split_exe = os.path.basename(cp.get("executables", split_exe_tag))
    exe_class = select_splitfilejob_instance(split_exe)

    # Set up output structure
    out_file_groups = FileList([])

    curr_exe_job = exe_class(workflow.cp, split_exe_tag, num_splits,
                             out_dir=out_dir)

    for input in input_tables:
        node = curr_exe_job.create_node(input, tags=tags)
        out_file_groups += node.output_files
    return out_file_groups

#
#
#

#
#
#                                   Preamble
#
#

"""
This module is responsible for setting up the template bank stage of CBC
https://ldas-jobs.ligo.caltech.edu/~cbc/docs/pycbc/ahope/template_bank.html
"""

import os
from pycbc.workflow.core import FileList
from pycbc.workflow.jobsetup import select_tmpltbank_class, select_matchedfilter_class, sngl_ifo_job_setup

def setup_tmpltbank_workflow(workflow, science_segs, datafind_outs,
                             output_dir=None, psd_files=None, tags=None,
                             return_format=None):
    '''
    Setup template bank section of CBC workflow. This function is responsible
    for deciding which of the various template bank workflow generation
    utilities should be used.

        ifo.
        The file list containing predefined PSDs, if provided.

    Returns
    --------
        The FileList holding the details of all the template bank jobs.
    '''
        tags = []
    logging.info("Entering template bank generation module.")

    tmpltbankMethod = cp.get_opt_tags("workflow-tmpltbank", "tmpltbank-method",
                                      tags)

    # There can be a large number of different options here, for e.g. to set
    # up fixed bank, or maybe something else
    if tmpltbankMethod == "PREGENERATED_BANK":
        logging.info("Setting template bank from pre-generated bank(s).")
        tmplt_banks = setup_tmpltbank_pregenerated(workflow, tags=tags)
    # Else we assume template banks will be generated in the workflow
    elif tmpltbankMethod == "WORKFLOW_INDEPENDENT_IFOS":
        logging.info("Adding template bank jobs to workflow.")
        if cp.has_option_tags("workflow-tmpltbank",
            if not cp.has_option_tags("workflow-matchedfilter",
                errMsg = "If using tmpltbank-link-to-matchedfilter, you should "
                errMsg = "also use matchedfilter-link-to-tmpltbank."
            linkToMatchedfltr = True
        else:
            linkToMatchedfltr = False
            if not linkToMatchedfltr:
                errMsg += "tmpltbank-link-to-matchedfilter option is also set."
        else:
        tmplt_banks = setup_tmpltbank_dax_generated(workflow, science_segs,
                                         datafind_outs, output_dir, tags=tags,
                                         link_to_matchedfltr=linkToMatchedfltr,
                                         compatibility_mode=compatibility_mode,
                                         psd_files=psd_files)
    elif tmpltbankMethod == "WORKFLOW_INDEPENDENT_IFOS_NODATA":
        tmplt_banks = setup_tmpltbank_without_frames(workflow, output_dir,
                                         tags=tags, independent_ifos=True,
    elif tmpltbankMethod == "WORKFLOW_NO_IFO_VARIATION_NODATA":
                                         tags=tags, independent_ifos=False,
    else:
        errMsg = "Template bank method not recognized. Must be either "
        errMsg += "PREGENERATED_BANK, WORKFLOW_INDEPENDENT_IFOS "
        errMsg += "or WORKFLOW_INDEPENDENT_IFOS_NODATA."

    # Check the format of the input template bank file and return it in
    # the format requested as per return_format, provided a conversion
    # between the two specific formats has been implemented. Currently,
    # a conversion from xml.gz or xml to hdf is supported, but not vice
    # versa. If a return_format is not specified the function returns
    # the bank in the format as it was inputted.
    tmplt_bank_filename=tmplt_banks[0].name
    ext = tmplt_bank_filename.split('.', 1)[1]
    logging.info("Input bank is a %s file", ext)
    if return_format is None :
        tmplt_banks_return = tmplt_banks
    elif return_format in ('hdf', 'h5', 'hdf5'):
        if ext in ('hdf', 'h5', 'hdf5') or ext in ('xml.gz' , 'xml'):
            tmplt_banks_return = pycbc.workflow.convert_bank_to_hdf(workflow,
                                                        tmplt_banks, "bank")
    else :
        if ext == return_format:
            tmplt_banks_return = tmplt_banks
        else:
            raise NotImplementedError("{0} to {1} conversion is not "
                                      "supported.".format(ext, return_format))
    logging.info("Leaving template bank generation module.")
    return tmplt_banks_return

def setup_tmpltbank_dax_generated(workflow, science_segs, datafind_outs,
                                  output_dir, tags=None,
                                  link_to_matchedfltr=True,
                                  compatibility_mode=False,
                                  psd_files=None):
    '''
    Setup template bank jobs that are generated as part of the CBC workflow.
    This function will add numerous jobs to the CBC workflow using
    configuration options from the .ini file. The following executables are
    currently supported:

    * lalapps_tmpltbank
    * pycbc_geom_nonspin_bank

        ifo.
    link_to_matchedfltr : boolean, optional (default=True)
        If this option is given, the job valid_times will be altered so that
        there will be one inspiral file for every template bank and they will
        cover the same time span. Note that this option must also be given
        during matched-filter generation to be meaningful.
    psd_file : pycbc.workflow.core.FileList

    Returns
    --------
    '''
        tags = []

    tmplt_bank_exe = os.path.basename(cp.get('executables', 'tmpltbank'))
    exe_class = select_tmpltbank_class(tmplt_bank_exe)

    # The exe instance needs to know what data segments are analysed, what is

    if link_to_matchedfltr:
        tmpltbank_exe = os.path.basename(cp.get('executables', 'inspiral'))
        link_exe_instance = select_matchedfilter_class(tmpltbank_exe)
    else:

    tmplt_banks = FileList([])

    # Template banks are independent for different ifos, but might not be!
    # Begin with independent case and add after FIXME
        job_instance = exe_class(workflow.cp, 'tmpltbank', ifo=ifo,
        # Check for the write_psd flag
        if cp.has_option_tags("workflow-tmpltbank", "tmpltbank-write-psd-file", tags):
            job_instance.write_psd = True
        else:
            job_instance.write_psd = False

            link_job_instance = link_exe_instance(cp, 'inspiral', ifo=ifo,
                        out_dir=output_dir, tags=tags)
        else:
        sngl_ifo_job_setup(workflow, ifo, tmplt_banks, job_instance,
                           allow_overlap=True,
    return tmplt_banks

def setup_tmpltbank_without_frames(workflow, output_dir,
                                   tags=None, independent_ifos=False,
                                   psd_files=None):
    '''
    Setup CBC workflow to use a template bank (or banks) that are generated in
    the workflow, but do not use the data to estimate a PSD, and therefore do
    not vary over the duration of the workflow. This can either generate one
    bank that is valid for all ifos at all times, or multiple banks that are
    valid only for a single ifo at all times (one bank per ifo).

        The directory where the template bank outputs will be placed.
    independent_ifos : Boolean, optional (default=False)
        If given this will produce one template bank per ifo. If not given
        there will be on template bank to cover all ifos.

    Returns
    --------
        The FileList holding the details of the template bank(s).
    '''
        tags = []

    fullSegment = workflow.analysis_time

    tmplt_bank_exe = os.path.basename(cp.get('executables','tmpltbank'))
    # Can not use lalapps_template bank with this
    if tmplt_bank_exe == 'lalapps_tmpltbank':
        errMsg = "Lalapps_tmpltbank cannot be used to generate template banks "
        errMsg += "without using frames. Try another code."

    exe_instance = select_tmpltbank_class(tmplt_bank_exe)

    # Make the distinction between one bank for all ifos and one bank per ifo
    if independent_ifos:
        ifoList = [ifo for ifo in ifos]
    else:
        ifoList = [[ifo for ifo in ifos]]

    # Check for the write_psd flag
    if cp.has_option_tags("workflow-tmpltbank", "tmpltbank-write-psd-file", tags):
        exe_instance.write_psd = True
    else:
        exe_instance.write_psd = False

    for ifo in ifoList:
        job_instance = exe_instance(workflow.cp, 'tmpltbank', ifo=ifo,
                                               tags=tags,
                                               psd_files=psd_files)
        node = job_instance.create_nodata_node(fullSegment)
        tmplt_banks += node.output_files

def setup_tmpltbank_pregenerated(workflow, tags=None):
    '''
    Setup CBC workflow to use a pregenerated template bank.
    The bank given in cp.get('workflow','pregenerated-template-bank') will be used
    as the input file for all matched-filtering jobs. If this option is
    present, workflow will assume that it should be used and not generate
    template banks within the workflow.

    Returns
    --------
        The FileList holding the details of the template bank.
    '''
        tags = []
    # Currently this uses the *same* fixed bank for all ifos.
    # Maybe we want to add capability to analyse separate banks in all ifos?

    file_attrs = {'segs' : global_seg, 'tags' : tags}

    try:
        # First check if we have a bank for all ifos
        pre_gen_bank = cp.get_opt_tags('workflow-tmpltbank',
                                           'tmpltbank-pregenerated-bank', tags)
        curr_file = resolve_url_to_file(pre_gen_bank, attrs=file_attrs)
        tmplt_banks.append(curr_file)
        # Okay then I must have banks for each ifo
            try:
                pre_gen_bank = cp.get_opt_tags('workflow-tmpltbank',
                                'tmpltbank-pregenerated-bank-%s' % ifo.lower(),
                                tags)
                curr_file = resolve_url_to_file(pre_gen_bank, attrs=file_attrs)
                tmplt_banks.append(curr_file)

                err_msg = "Cannot find pregerated template bank in section "
                err_msg += "[workflow-tmpltbank] or any tagged sections. "
                if tags:
                    tagged_secs = " ".join("[workflow-tmpltbank-%s]" \
                                           %(ifo,) for ifo in workflow.ifos)
                    err_msg += "Tagged sections are %s. " %(tagged_secs,)
                err_msg += "I looked for 'tmpltbank-pregenerated-bank' option "
                err_msg += "and 'tmpltbank-pregenerated-bank-%s'." %(ifo,)
                raise ConfigParser.Error(err_msg)