import h5py
import numpy as np
import glob, os
from astropy.time import Time

import matplotlib as mpl
import matplotlib.pyplot as plt
fig_width_pt = 3*246.0  # Get this from LaTeX using \showthe\columnwidth
inches_per_pt = 1.0/72.27               # Convert pt to inch
golden_mean = (np.sqrt(5)-1.0)/2.0         # Aesthetic ratio
fig_width = fig_width_pt*inches_per_pt  # width in inches
fig_height = fig_width*golden_mean      # height in inches
fig_size =  [fig_width,fig_height]

params = { 'axes.labelsize': 24,
          'font.family': 'serif',
          'font.serif': 'Computer Modern Raman',
          'font.size': 24,
          'legend.fontsize': 20,
          'xtick.labelsize': 24,
          'ytick.labelsize': 24,
          'axes.grid' : True,
          'text.usetex': True,
          'savefig.dpi' : 100,
          'lines.markersize' : 14,
          'figure.figsize': fig_size}

mpl.rcParams.update(params)

f = h5py.File('/work/ahnitz/WWW/4ogc/4-ogc.hdf','r')

f.keys()

<KeysViewHDF5 ['H1_chisq', 'H1_end_time', 'H1_sg_chisq', 'H1_snr', 'L1_chisq', 'L1_end_time', 'L1_sg_chisq', 'L1_snr', 'V1_chisq', 'V1_end_time', 'V1_sg_chisq', 'V1_snr', 'ifar', 'mass1', 'mass2', 'name', 'obs', 'pastro', 'spin1z', 'spin2z', 'stat', 'time', 'trig']>

l = (f['mass1'][:] > 30) & (f['mass2'][:] > 30) & (f['ifar'][:] > 0.001)

len(l[l==True])

142

l.sum()

142

ltime = (f['time'][:] > Time('2020-03-01T00:00:00').gps) & (f['time'][:] < Time('2020-03-01T00:10:00').gps) & \
        (f['mass1'][:]>1) & (f['mass2'][:]>1) & (f['ifar'][:] != -1.0)
t = f['time'][ltime]
ifar = f['ifar'][ltime]
stat = f['stat'][ltime]

ifar

array([1.38838874e-07, 1.13568209e-07, 2.79479450e-06, 1.15876178e-07,
       2.25819672e-07, 2.42803710e-07, 2.26265073e-07, 1.62198917e-06,
       1.83968817e-07, 3.26010624e-07, 1.69516192e-07, 4.09782144e-07,
       3.35532404e-07, 2.00003944e-07, 1.35148749e-07, 1.61515213e-06,
       7.31208768e-06, 4.44287189e-07, 9.46597339e-08, 1.97551538e-07,
       1.31611235e-07, 2.79897217e-07, 2.69217860e-07, 5.46549270e-07,
       1.06851676e-07, 1.13217584e-07, 1.14992512e-07, 4.00251956e-07,
       5.19312546e-07, 7.94151319e-07, 2.51429388e-07, 1.34521156e-07,
       1.98019522e-07, 2.79659397e-07, 1.96063547e-07, 1.13607617e-07,
       3.27802610e-07, 5.00142848e-07, 1.48665927e-07, 9.20449587e-07,
       4.42060685e-07, 2.36107550e-06, 3.82617417e-07, 2.17893843e-07,
       1.62625051e-07, 2.87065462e-07, 3.38980577e-06, 2.26265073e-07,
       1.52676002e-06, 1.56878449e-06, 3.06851683e-06, 8.98532390e-07,
       1.66736811e-07, 7.82816734e-08, 2.96512755e-07, 1.69873602e-07,
       4.47354456e-07, 2.33570852e-07, 2.47047175e-07, 4.25741445e-07,
       1.13828947e-07, 1.09218256e-06, 4.62215733e-06, 1.97494999e-07,
       7.87051799e-08, 1.45548899e-07, 4.20128509e-07, 2.13240988e-07,
       4.46800617e-07, 1.70490762e-07, 2.54643394e-07, 3.61487144e-07,
       4.42031625e-07, 1.00034251e-06, 1.08267307e-07, 2.14296032e-05,
       2.04308296e-07, 3.35954257e-06, 5.55158876e-06, 1.58034252e-07,
       4.30753605e-07, 1.87262911e-07, 2.11781861e-07, 2.60921657e-07,
       2.11493636e-07, 5.01466859e-07, 1.99414734e-07, 4.04130590e-07,
       1.14397063e-07, 2.67668174e-07, 5.36874392e-07, 1.79002563e-07,
       2.04518284e-07, 1.62210294e-06, 1.12742223e-07, 1.61851637e-07,
       2.29218241e-07, 8.14854238e-07, 8.13702229e-08, 2.49512152e-07,
       7.66844400e-08, 1.09621564e-07, 1.65860812e-07, 1.27434991e-07,
       2.51891175e-07, 2.13415630e-07, 1.26444362e-07, 4.59095488e-07,
       2.97949160e-07, 1.93140465e-06, 2.14298089e-07, 1.14233704e-07,
       5.97752437e-07, 9.74199384e-06, 2.03093689e-07, 1.05686757e-07,
       9.39256995e-08, 1.49670618e-07])

stat

array([-1.80172157e+00, -8.35006332e+00, -2.19640994e+00, -2.80445933e+00,
       -7.65040398e+00,  9.54312742e-01, -2.36406493e+00,  4.55342102e+00,
       -8.31185913e+00,  9.22728479e-01, -8.60048294e+00,  2.10107636e+00,
       -9.80606198e-01, -3.03419471e-01, -5.46647453e+00,  3.80208635e+00,
       -5.72816432e-01, -3.35421562e-02, -4.76279354e+00,  4.44386065e-01,
       -9.64329815e+00, -1.60559106e+00, -7.14259529e+00,  2.66992164e+00,
       -3.39165282e+00, -1.04327259e+01, -7.98827267e+00, -6.11202812e+00,
        2.57248211e+00,  1.15640235e+00,  3.43392730e-01, -5.52077484e+00,
       -8.06386471e+00,  5.85694373e-01, -8.09740925e+00, -8.33695984e+00,
        1.63863266e+00,  1.78834915e+00, -5.35754144e-01,  1.43480957e+00,
       -5.87322426e+00,  5.11718369e+00,  1.25694644e+00, -7.75887060e+00,
       -3.98140979e+00, -6.96794415e+00,  4.91149330e+00, -2.36483955e+00,
        4.45956421e+00, -3.24177265e+00,  3.43685055e+00,  3.58050680e+00,
       -8.65982437e+00, -9.02950287e+00,  7.17808545e-01, -8.59252357e+00,
       -1.25859380e-02, -2.24052358e+00, -7.38776302e+00,  2.17977571e+00,
       -2.93651748e+00,  3.18008184e+00, -1.33210850e+00, -2.93708920e+00,
       -1.43397512e+01, -6.32758141e-01,  1.44535387e+00, -2.61301446e+00,
       -5.84795332e+00,  4.70994711e-02, -1.93116021e+00,  1.13953149e+00,
       -5.60382009e-02, -4.10260725e+00, -2.27826834e+00,  5.81032181e+00,
       -2.26939738e-01, -1.87656116e+00,  4.25472736e+00, -1.23426795e+00,
        2.20256782e+00, -8.25250816e+00, -2.64356160e+00,  4.29346621e-01,
       -7.85194540e+00,  2.36993968e-01, -8.04214573e+00,  1.36747158e+00,
       -8.14605808e+00, -7.15888739e+00,  2.63614845e+00, -7.30876982e-01,
       -2.78779602e+00,  3.80869102e+00, -1.04568920e+01, -1.54249132e-01,
       -2.31591988e+00,  3.40863204e+00, -1.36515484e+01,  3.26492190e-01,
       -1.50239277e+01, -1.00988092e+01, -4.52640653e-02, -9.79838085e+00,
       -1.96744537e+00, -6.82450533e-02, -9.83444500e+00,  5.11592627e-02,
        1.42856991e+00,  4.07903194e+00, -2.59316492e+00, -1.91210270e+00,
       -5.18588257e+00,  4.94936657e+00, -2.48483717e-01, -3.49107265e+00,
       -3.59482050e+00, -9.07752705e+00])

fig = plt.figure()
ms=5
ax = fig.add_subplot(211)
ax.plot(t,ifar,marker='o',ms=ms)
ax.set_yscale('log')
ax.set_ylabel('ifar')
bx = fig.add_subplot(212)
bx.plot(t,stat,marker='o',ms=ms)
bx.set_ylabel('stat')
fig.align_xlabels()

So where does negative stat come from?

Notes: ./copy/compile/

bbh.hdf, bns.hdf, and nsbh.hdf are template banks

{bbh, bns, nsbh}_{H1, L1}.hdf is single detector triggers

f = h5py.File('./copy/compile/o1o2o3a_coincs.hdf','r')

f.keys()

<KeysViewHDF5 ['H1_chisq', 'H1_end_time', 'H1_sg_chisq', 'H1_snr', 'L1_chisq', 'L1_end_time', 'L1_sg_chisq', 'L1_snr', 'V1_chisq', 'V1_end_time', 'V1_sg_chisq', 'V1_snr', 'ifar', 'mass1', 'mass2', 'spin1z', 'spin2z', 'stat']>

f = h5py.File('./copy/compile/bbh.hdf','r')

f.keys()

<KeysViewHDF5 ['approximant', 'f_lower', 'mass1', 'mass2', 'spin1z', 'spin2z']>

f = h5py.File('./copy/compile/bbh_H1_trigs.hdf','r')

f.keys()

<KeysViewHDF5 ['chisq', 'sg_chisq', 'snr', 'stat', 'tid', 'time']>

k = np.ones(len(f['chisq']),dtype=bool)

f['chisq'][k]

array([0.94310195, 1.23914811, 0.97312706, ..., 0.81201359, 0.90111299,
       0.82589907])

from pycbc.dq import query_flag
segs = {'H1':[], 'L1':[], 'V1':[]}
for ifo in ['H1', 'L1', 'V1']:
    for s, e in [(1126051217, 1137254417), (1164556817, 1187733618), (1238166018, 1253977218),  (1253977219, 1320363336)]:
        if ifo == 'V1' and s == 1126051217:
            continue
        segs[ifo] +=  query_flag(ifo, 'DATA', s, e, cache=True)

t=Time('1126051217',format='gps')

t.datetime64

numpy.datetime64('2015-09-12T00:00:36.000000000')

segs.keys()

dict_keys(['H1', 'L1', 'V1'])

f = h5py.File('./copy/compile/pastro_H1_bbh_single.hdf','r')

f.keys()

<KeysViewHDF5 ['pastro', 'time']>

f['pastro'][:]

array([0.        , 0.        , 0.        , 0.        , 0.65820356])

f = h5py.File('./copy/compile/pastro_L1_bbh_single.hdf','r')

f['pastro'][:]

array([0.52671383, 0.72537163, 0.72942365, 0.77138193, 0.78411081])

d = np.loadtxt('./copy/compile/coinc_pastro.txt', dtype=str)

d

array([['1266645879.40', '2020-02-25', '06:04:21.398193', '37.92',
        'H1L1', '1.00'],
       ['1266618172.40', '2020-02-24', '22:22:34.400146', '83.06',
        'H1L1V1', '1.00'],
       ['1267132237.66', '2020-03-01', '21:10:19.661133', '4.70', 'H1L1',
        '0.43'],
       ['1262353301.93', '2020-01-06', '13:41:23.925293', '5.41',
        'H1L1V1', '0.69'],
       ['1256999739.93', '2019-11-05', '14:35:21.930664', '16.68',
        'H1L1V1', '1.00'],
       ['1256772098.02', '2019-11-02', '23:21:20.023926', '3.25',
        'H1L1V1', '0.22'],
       ['1263744929.34', '2020-01-22', '16:15:11.340820', '3.22', 'H1L1',
        '0.27'],
       ['1257906499.42', '2019-11-16', '02:28:01.423828', '3.52',
        'H1L1V1', '0.22'],
       ['1260567236.48', '2019-12-16', '21:33:38.479248', '63.24',
        'H1V1', '1.00'],
       ['1260484270.33', '2019-12-15', '22:30:52.333740', '26.43',
        'H1L1V1', '1.00'],
       ['1267381378.62', '2020-03-04', '18:22:40.619141', '3.02', 'H1L1',
        '0.28'],
       ['1267433277.08', '2020-03-05', '08:47:39.077393', '4.86',
        'H1L1V1', '0.59'],
       ['1267522652.12', '2020-03-06', '09:37:14.125000', '4.08', 'H1L1',
        '0.51'],
       ['1258804397.63', '2019-11-26', '11:52:59.631104', '11.02',
        'H1L1', '1.00'],
       ['1258866165.55', '2019-11-27', '05:02:27.552246', '10.40',
        'H1L1V1', '0.99'],
       ['1259070047.20', '2019-11-29', '13:40:29.199219', '41.85',
        'H1L1', '1.00'],
       ['1261764316.40', '2019-12-30', '18:04:58.402100', '17.75',
        'H1L1V1', '1.00'],
       ['1257296855.21', '2019-11-09', '01:07:17.214844', '28.67',
        'H1L1', '1.00'],
       ['1263198258.97', '2020-01-16', '08:24:00.965088', '2.96',
        'H1L1V1', '0.21'],
       ['1268431094.16', '2020-03-16', '21:57:56.157715', '13.64',
        'H1L1V1', '1.00'],
       ['1268594035.14', '2020-03-18', '19:13:37.141113', '8.22',
        'H1L1V1', '0.97'],
       ['1265754805.00', '2020-02-14', '22:33:07.000732', '5.81',
        'H1L1V1', '0.72'],
       ['1265926102.89', '2020-02-16', '22:08:04.885254', '6.00',
        'H1L1V1', '0.78'],
       ['1265202095.95', '2020-02-08', '13:01:17.954753', '23.14',
        'H1L1V1', '1.00'],
       ['1264947442.84', '2020-02-05', '14:17:04.841553', '2.48', 'H1L1',
        '0.23'],
       ['1259514944.09', '2019-12-04', '17:15:26.089600', '69.06',
        'H1L1', '1.00'],
       ['1259492747.54', '2019-12-04', '11:05:29.542236', '9.77', 'H1L1',
        '0.99'],
       ['1259827432.84', '2019-12-08', '08:03:34.843750', '3.08',
        'H1L1V1', '0.23'],
       ['1264316116.42', '2020-01-29', '06:54:58.417480', '76.97',
        'H1V1', '1.00'],
       ['1264213229.90', '2020-01-28', '02:20:11.898682', '22.04',
        'H1L1', '1.00'],
       ['1264693411.56', '2020-02-02', '15:43:13.560791', '17.92',
        'H1L1V1', '1.00'],
       ['1264333383.11', '2020-01-29', '11:42:45.111328', '5.50',
        'H1L1V1', '0.53'],
       ['1261020955.12', '2019-12-22', '03:35:37.122314', '20.56',
        'H1L1', '1.00'],
       ['1261197166.15', '2019-12-24', '04:32:28.154297', '6.28',
        'H1L1V1', '0.87'],
       ['1258494582.97', '2019-11-22', '21:49:24.966553', '3.50',
        'H1L1V1', '0.24'],
       ['1267963151.38', '2020-03-11', '11:58:53.384603', '73.18',
        'H1L1V1', '1.00'],
       ['1267866122.76', '2020-03-10', '09:01:44.763672', '3.63', 'H1L1',
        '0.43'],
       ['1266140673.20', '2020-02-19', '09:44:15.197998', '13.22',
        'H1L1V1', '1.00'],
       ['1266238148.15', '2020-02-20', '12:48:50.151367', '4.11', 'H1L1',
        '0.43'],
       ['1265273710.17', '2020-02-09', '08:54:52.174072', '9.48',
        'H1L1V1', '0.99'],
       ['1265331100.74', '2020-02-10', '00:51:22.742432', '4.92',
        'H1L1V1', '0.74'],
       ['1177134832.19', '2017-04-25', '05:54:11.190918', '7.29', 'H1L1',
        '0.37'],
       ['1180922494.49', '2017-06-08', '02:01:53.488770', '55.50',
        'H1L1', '1.00'],
       ['1187529256.52', '2017-08-23', '13:14:35.516846', '38.80',
        'H1L1', '1.00'],
       ['1185152688.03', '2017-07-27', '01:05:07.033203', '17.33',
        'H1L1', '1.00'],
       ['1186302519.75', '2017-08-09', '08:28:58.750000', '41.27',
        'H1L1V1', '1.00'],
       ['1175295989.23', '2017-04-03', '23:06:48.231689', '8.01', 'H1L1',
        '0.72'],
       ['1182296561.37', '2017-06-23', '23:43:00.370117', '5.74', 'H1L1',
        '0.23'],
       ['1172680691.37', '2017-03-04', '16:38:30.365967', '8.22', 'H1L1',
        '0.74'],
       ['1135136350.65', '2015-12-26', '03:39:29.648926', '43.58',
        'H1L1', '1.00'],
       ['1126259462.42', '2015-09-14', '09:51:21.424561', '134.72',
        'H1L1', '1.00'],
       ['1128678900.45', '2015-10-12', '09:55:19.445068', '18.49',
        'H1L1', '1.00'],
       ['1128626886.60', '2015-10-11', '19:28:25.604492', '6.76', 'H1L1',
        '0.24'],
       ['1185389807.32', '2017-07-29', '18:57:06.323730', '13.07',
        'H1L1', '1.00'],
       ['1133380542.41', '2015-12-05', '19:56:01.409424', '6.05', 'H1L1',
        '0.20'],
       ['1186741861.53', '2017-08-14', '10:31:20.531006', '73.47',
        'H1L1V1', '1.00'],
       ['1187058327.08', '2017-08-18', '02:25:46.083740', '22.19',
        'H1L1V1', '1.00'],
       ['1183234821.62', '2017-07-04', '20:20:40.620117', '6.67', 'H1L1',
        '0.43'],
       ['1167559936.60', '2017-01-04', '10:12:35.598633', '46.82',
        'H1L1', '1.00'],
       ['1169069154.58', '2017-01-21', '21:26:13.575928', '19.02',
        'H1L1', '1.00'],
       ['1170079035.73', '2017-02-02', '13:57:34.732666', '9.25', 'H1L1',
        '0.86'],
       ['1252627040.69', '2019-09-15', '23:57:39.693359', '40.21',
        'H1L1V1', '1.00'],
       ['1252699636.90', '2019-09-16', '20:07:35.901367', '8.26',
        'H1L1V1', '0.90'],
       ['1242984073.79', '2019-05-27', '09:21:32.792969', '8.70', 'H1L1',
        '0.94'],
       ['1240292407.21', '2019-04-26', '05:40:26.212646', '4.53',
        'H1L1V1', '0.40'],
       ['1240423628.68', '2019-04-27', '18:07:27.675537', '5.13',
        'H1L1V1', '0.53'],
       ['1248242631.99', '2019-07-27', '06:04:10.986816', '33.78',
        'H1L1V1', '1.00'],
       ['1248331528.53', '2019-07-28', '06:45:47.531982', '43.88',
        'H1L1V1', '1.00'],
       ['1248112066.46', '2019-07-25', '17:48:05.464600', '9.15',
        'H1L1V1', '0.96'],
       ['1238782700.29', '2019-04-08', '18:18:39.286133', '57.54',
        'H1L1V1', '1.00'],
       ['1239082262.17', '2019-04-12', '05:31:21.166504', '71.02',
        'H1L1V1', '1.00'],
       ['1239198206.74', '2019-04-13', '13:43:45.735840', '13.67',
        'H1L1V1', '1.00'],
       ['1239168612.50', '2019-04-13', '05:30:31.499512', '11.29',
        'H1L1V1', '1.00'],
       ['1242459857.46', '2019-05-21', '07:44:36.462402', '122.82',
        'H1L1', '1.00'],
       ['1242315362.39', '2019-05-19', '15:36:21.389404', '47.79',
        'H1L1V1', '1.00'],
       ['1242442967.44', '2019-05-21', '03:03:06.442383', '22.34',
        'H1L1V1', '1.00'],
       ['1241816086.74', '2019-05-13', '20:55:05.742350', '25.27',
        'H1L1V1', '1.00'],
       ['1242107479.83', '2019-05-17', '05:51:38.826660', '14.47',
        'H1L1V1', '1.00'],
       ['1241852074.85', '2019-05-14', '06:54:53.847412', '7.34', 'H1L1',
        '0.82'],
       ['1242342822.07', '2019-05-19', '23:14:01.068359', '4.40',
        'H1L1V1', '0.26'],
       ['1245938000.49', '2019-06-30', '13:53:39.485352', '4.23', 'H1L1',
        '0.26'],
       ['1245955943.18', '2019-06-30', '18:52:42.178223', '37.04',
        'L1V1', '1.00'],
       ['1246048404.58', '2019-07-01', '20:33:43.580241', '18.86',
        'H1L1V1', '1.00'],
       ['1250398837.88', '2019-08-21', '05:00:56.875977', '5.11',
        'H1L1V1', '0.31'],
       ['1248834439.88', '2019-08-03', '02:27:38.877441', '13.37',
        'H1L1V1', '1.00'],
       ['1248617394.64', '2019-07-31', '14:10:13.635986', '8.16', 'H1L1',
        '0.92'],
       ['1248606001.71', '2019-07-31', '11:00:20.707031', '4.05',
        'H1L1V1', '0.29'],
       ['1249037690.78', '2019-08-05', '10:55:09.779053', '5.09', 'H1L1',
        '0.51'],
       ['1239917954.25', '2019-04-21', '21:39:33.254395', '17.72',
        'H1L1', '1.00'],
       ['1240128716.76', '2019-04-24', '08:12:15.762207', '3.81',
        'H1L1V1', '0.25'],
       ['1251009263.76', '2019-08-28', '06:34:42.755371', '73.87',
        'H1L1V1', '1.00'],
       ['1251010527.89', '2019-08-28', '06:55:46.887207', '23.35',
        'H1L1V1', '1.00'],
       ['1243533585.10', '2019-06-02', '18:00:04.098633', '34.02',
        'H1L1V1', '1.00'],
       ['1243220837.97', '2019-05-30', '03:07:36.968262', '5.01',
        'H1L1V1', '0.44'],
       ['1241719652.42', '2019-05-12', '18:07:51.420166', '36.96',
        'H1L1V1', '1.00'],
       ['1241397698.79', '2019-05-09', '00:41:57.786133', '4.29',
        'H1L1V1', '0.37'],
       ['1253489343.12', '2019-09-25', '23:29:22.125000', '15.05',
        'H1V1', '1.00'],
       ['1253885759.24', '2019-09-30', '13:36:18.241455', '16.53',
        'H1L1', '1.00'],
       ['1253755327.50', '2019-09-29', '01:22:26.502197', '10.44',
        'H1L1V1', '0.99'],
       ['1253509434.07', '2019-09-26', '05:04:13.073486', '8.07',
        'H1L1V1', '0.92'],
       ['1246527224.17', '2019-07-07', '09:34:03.169189', '43.60',
        'H1L1', '1.00'],
       ['1246487219.33', '2019-07-06', '22:27:18.327393', '26.21',
        'H1L1V1', '1.00'],
       ['1246272532.92', '2019-07-04', '10:49:11.918945', '5.04',
        'H1L1V1', '0.27'],
       ['1246519060.10', '2019-07-07', '07:17:59.101074', '5.23',
        'H1L1V1', '0.32'],
       ['1247616534.71', '2019-07-20', '00:09:13.705322', '21.35',
        'H1L1V1', '1.00'],
       ['1247608532.92', '2019-07-19', '21:55:51.920654', '7.98', 'H1L1',
        '0.92'],
       ['1238423132.99', '2019-04-04', '14:25:51.993164', '4.52', 'H1L1',
        '0.50'],
       ['1240944862.29', '2019-05-03', '18:54:41.291016', '32.45',
        'H1L1V1', '1.00'],
       ['1253326744.84', '2019-09-24', '02:19:23.844238', '31.25',
        'H1L1V1', '1.00'],
       ['1249340753.59', '2019-08-08', '23:06:12.590088', '5.17',
        'H1L1V1', '0.41']], dtype='<U15')

extract/coincs/extract.py --> coincs.hdf

MERGER.hdf ---> COMBINE.hdf

compile/compile.ipynb --> combine events (coinc and single), add pastro, adjust IFAR