import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats
from scipy.stats import norm
import numpy as np

LH_true = np.array([0.0243454 , 0.01137771, 0.01254166, 0.01764927, 0.01033814,
       0.02188591, 0.01433098, 0.02341697, 0.01063864, 0.01546347,
       0.02686151, 0.02025511, 0.01197237, 0.02294361, 0.01465379,
       0.0151744 , 0.01387005, 0.01617782, 0.00764115, 0.01585363,
       0.02129582, 0.02050885, 0.01058422, 0.00994643, 0.01816286,
       0.01130973, 0.01100134, 0.0147438 , 0.01585406, 0.00797675,
       0.02615668, 0.01925256, 0.01349633, 0.01097237, 0.02214721,
       0.01255305, 0.01814757, 0.01468691, 0.01447975, 0.01437719,
       0.01880171, 0.01347181, 0.00708943, 0.01589191, 0.00989473,
       0.01779402, 0.02564946, 0.01093551, 0.01478732, 0.0212522 ,
       0.00711471, 0.02169546, 0.01605215, 0.01323693, 0.00752848,
       0.019766  , 0.01428411, 0.01656917, 0.01463805, 0.00667371,
       0.01616888, 0.01166911, 0.0056077 , 0.02470745, 0.01036723,
       0.0126541 , 0.01747745, 0.0171829 , 0.01240231, 0.01147828,
       0.01886307, 0.02116652, 0.01692161, 0.01111378, 0.01393348,
       0.02386079, 0.02208303, 0.0199446 , 0.0107561 , 0.01627575,
       0.01685479, 0.0132076 , 0.01698396, 0.01858902, 0.00952094,
       0.01216046, 0.0106047 , 0.01459445, 0.01772976, 0.01722888,
       0.01728338, 0.01786537, 0.013072  , 0.01994857, 0.02338869,
       0.02684308, 0.02221262, 0.01947833, 0.01023174, 0.01598115,
       0.01896411, 0.0186295 , 0.0160543 , 0.01382898, 0.01204012,
       0.02464213, 0.01491632, 0.01788966, 0.01631663, 0.01475178,
       0.01406165, 0.01178407, 0.02474272, 0.01520064, 0.01239701,
       0.0152896 , 0.01348954, 0.01809647, 0.01170881, 0.02105506,
       0.01733557, 0.01744543, 0.02129121, 0.01120059, 0.01936956,
       0.02254451, 0.01549223, 0.02056385, 0.01295331, 0.01065183,
       0.01168009, 0.01060082, 0.00763025, 0.01442777, 0.02457093,
       0.00859534, 0.01106061, 0.01566492, 0.01324606, 0.01816762,
       0.02703854, 0.01756388, 0.01991268, 0.01629434, 0.01725145,
       0.01265852, 0.02766356, 0.02158618, 0.02161537, 0.00999176, 0.01473731, 0.01859166,
       0.01449057, 0.01835252, 0.0186778 , 0.01387449, 0.01903486,
       0.02722244, 0.01148044, 0.01982909, 0.026942  , 0.02122395,
       0.02773364, 0.02430927, 0.01051236, 0.01358496, 0.01102623,
       0.03416789, 0.03194469, 0.02149661, 0.01648806, 0.01531931,
       0.01690401, 0.01875775, 0.01430259, 0.01651443, 0.02838931,
       0.01727156, 0.02462665, 0.01555527, 0.01529346, 0.01392489,
       0.03032773, 0.02009171, 0.01412658, 0.01495382, 0.0152912 ,
       0.01597281, 0.01865346, 0.01833004, 0.01451648, 0.02242841,
       0.01782251, 0.01723732, 0.02525619, 0.01026545, 0.02427086,
       0.02305396, 0.04328358, 0.02482204, 0.01895393, 0.01382958,
       0.02659518, 0.01616063, 0.01551417, 0.00895099, 0.02512712,
       0.01415321, 0.0170071 , 0.0105686 , 0.01563909, 0.021591  ,
       0.01358423, 0.0184547 , 0.01539448, 0.00932675, 0.01997146,
       0.02333521, 0.01917388, 0.01850855, 0.02079451, 0.01310238,
       0.02184266, 0.01591911, 0.01651602, 0.01609568, 0.01704943,
       0.01193077, 0.02053374, 0.02039069, 0.02638249, 0.01671001,
       0.01905075, 0.01243356, 0.01343778, 0.01656691, 0.0095928 ,
       0.02071538, 0.02337493, 0.00827804, 0.01976005, 0.02027279,
       0.0247019 , 0.01459988, 0.01947134, 0.02537729, 0.00908717,
       0.02414107, 0.01676044, 0.01828911, 0.01069696, 0.01870233,
       0.02104198, 0.01302839, 0.0202646 , 0.01280324, 0.02970006,
       0.01539649, 0.01506436, 0.02363847, 0.01555497, 0.02231712,
       0.02302281, 0.00850748, 0.01601754, 0.01767472, 0.01077911,
       0.0081332 , 0.01378332, 0.01805357, 0.01497471, 0.02957416,
       0.01215248, 0.01825334, 0.0116856 , 0.021292  , 0.01611323,
       0.0144216 , 0.00901409, 0.02075969, 0.01812357, 0.01125501,
       0.01719236, 0.02411616, 0.02248682, 0.0132942 , 0.02110585,
       0.02063637, 0.01527086, 0.01527054, 0.01304916, 0.0187296 ])

LH_blank = np.array([0.08304965, 0.06080693, 0.0482525 , 0.0649201 , 0.0640573 ,
       0.06705837, 0.05588193, 0.06804985, 0.06204738, 0.07110258,
       0.06302665, 0.06502905, 0.05035468, 0.06022349, 0.0672238 ,
       0.06167749, 0.05564668, 0.04939812, 0.05372307, 0.05914843,
       0.06945211, 0.06564107, 0.05507835, 0.05505144, 0.05764116,
       0.07468537, 0.09165873, 0.05438113, 0.06730468, 0.05179002,
       0.06210032, 0.05342516, 0.06438767, 0.05800271, 0.05721558,
       0.05063468, 0.05135811, 0.06327395, 0.0535465 , 0.06835398,
       0.05815953, 0.0510736 , 0.0518777 , 0.0525936 , 0.04894333,
       0.04719285, 0.0595056 , 0.0646691 , 0.05812805, 0.04536958,
       0.04551008, 0.05569348, 0.0607035 , 0.05834383, 0.0599524 ,
       0.05684288, 0.07685856, 0.07334129, 0.05275547, 0.06272876,
       0.0759981 , 0.05757865, 0.05477774, 0.06868113, 0.06001401,
       0.05662394, 0.06233353, 0.06314249, 0.06525627, 0.04958797,
       0.05472175, 0.04701968, 0.05103793, 0.06741105, 0.06329287,
       0.06975287, 0.06096613, 0.06295818, 0.07347939, 0.06249537,
       0.07376806, 0.05273684, 0.06754793, 0.07171878, 0.06415552,
       0.04686758, 0.0644156 , 0.05213181, 0.06045513, 0.05235446,
       0.05073884, 0.05955901, 0.04698793, 0.07063938, 0.06507942,
       0.06418177, 0.0661636 , 0.07836832, 0.05613743, 0.06591954,
       0.05424148, 0.06357661, 0.06114524, 0.0742442 , 0.05157953,
       0.06200482, 0.06292991, 0.05647538, 0.05909437, 0.06021794,
       0.06420677, 0.06153891, 0.06777815, 0.06706225, 0.06150297,
       0.05009785, 0.06696403, 0.07399453, 0.05355416, 0.062324  ,
       0.06045424, 0.06034979, 0.05991676, 0.05596294, 0.05135072,
       0.0662274 , 0.06995949, 0.06120868, 0.06505551, 0.05986491,
       0.05456196, 0.05744792, 0.05685432, 0.05828814, 0.06398348,
       0.04472965, 0.04956053, 0.06066678, 0.05240257, 0.08326158,
       0.07985956, 0.06432525, 0.06154276, 0.05817112, 0.060151  ,
       0.05059023, 0.05489021, 0.05563025, 0.063263  , 0.06576202, 0.07686813, 0.06580925,
       0.0520939 , 0.06379999, 0.07863038, 0.0610713 , 0.05265974,
       0.04835468, 0.05268484, 0.07517789, 0.05306532, 0.06226278,
       0.06318275, 0.05354689, 0.06454231, 0.06309215, 0.05697054,
       0.07827121, 0.06857839, 0.06095889, 0.05661007, 0.05444159,
       0.05447736, 0.05697821, 0.07127665, 0.07957589, 0.07016962,
       0.07917787, 0.06106016, 0.06776004, 0.05779531, 0.07288225,
       0.07865921, 0.05486289, 0.07440147, 0.06326295, 0.05677992,
       0.05197074, 0.06196661, 0.05926011, 0.06985814, 0.06930489,
       0.07357475, 0.0540926 , 0.05544375, 0.05936779, 0.07614674,
       0.06522005, 0.07360504, 0.07443281, 0.05293929, 0.06339275,
       0.07029855, 0.07083263, 0.06782097, 0.05652063, 0.07244678,
       0.05733038, 0.05271803, 0.04808172, 0.07060203, 0.07378644,
       0.0630273 , 0.05762263, 0.07018321, 0.07420641, 0.05416345,
       0.06422581, 0.06260748, 0.06567329, 0.06028148, 0.05978885,
       0.04843193, 0.06057947, 0.07205037, 0.07889045, 0.07589037,
       0.06160799, 0.0640422 , 0.07622705, 0.06008831, 0.0779916 ,
       0.06576271, 0.06497154, 0.09299609, 0.05454212, 0.05018721,
       0.05485121, 0.06243918, 0.06107676, 0.07047842, 0.09407295,
       0.07071589, 0.06648009, 0.06374476, 0.06310968, 0.05093832,
       0.07328377, 0.05327595, 0.07889496, 0.04354965, 0.07494471,
       0.07227842, 0.06500141, 0.05200499, 0.06035391, 0.07445067,
       0.07164518, 0.06222149, 0.07251397, 0.05433986, 0.06180596,
       0.05355925, 0.06436578, 0.06308719, 0.09213091, 0.06305138,
       0.0592763 , 0.06326752, 0.06648095, 0.08136742, 0.07603733,
       0.052779  , 0.0694418 , 0.05053868, 0.06106663, 0.06493369,
       0.05378584, 0.05587382, 0.05808794, 0.0585373 , 0.0646637 ,
       0.0830994 , 0.07565213, 0.06382791, 0.06599428, 0.05706224,
       0.06382753, 0.06938497, 0.06560417, 0.0586922 , 0.06824748])

LHR = np.sqrt(len(LH_true))*(LH_true - LH_blank)
sns.set(color_codes=True)
LHR_dis = sns.distplot(LHR, fit=norm, kde=False);
# LHR_dis.set(xlim=(-.03, .04))
plt.title('Histgram of the likelihhod ratio based on 350 samples of random split')
plt.ylabel('count')
plt.show()

import scipy
## Null hypothesis is False, check about the type 2 error
Z_score = (LHR - 0.) / np.std(LHR)
# pval = scipy.stats.norm.sf(abs(Z_score)) #twosided
pval = 1. - scipy.stats.norm.sf(Z_score)
type_one_err = len(pval[pval<.05])/len(pval)
type_two_err = len(pval[pval>=.05])/len(pval)
print('Null hypothesis is False, and the type 2 error is %.3f' %type_two_err)
# Null hypothesis is False, and the type 2 error is 0.000

# if pval < .05:
#     print("reject null hypothesis")
# else:
#     print("not reject null hypothesis")


## case 2: Null hypothesis is True
LH_true = np.array([0.02110901, 0.02122322, 0.01396787, 0.01730401, 0.01411076,
       0.01147127, 0.01476736, 0.02086984, 0.02076641, 0.02118487,
       0.0177614 , 0.01391828, 0.01819567, 0.01983379, 0.01900922,
       0.01660589, 0.02638373, 0.01581072, 0.01679614, 0.00870597,
       0.01912305, 0.03012733, 0.02434628, 0.01668124, 0.01563341,
       0.02060881, 0.01741394, 0.016877  , 0.01676487, 0.02604208,
       0.01488976, 0.0113506 , 0.01645534, 0.01628211, 0.01901422,
       0.03348601, 0.02819555, 0.01448784, 0.01426146, 0.01229041,
       0.01776911, 0.01585287, 0.01699668, 0.0155901 , 0.01761474,
       0.0183158 , 0.0115107 , 0.01862218, 0.01389584, 0.02533186,
       0.01367654, 0.03678019, 0.0238745 , 0.01660677, 0.01425102,
       0.03379833, 0.01620863, 0.01921537, 0.01157019, 0.02402299,
       0.01929055, 0.02678166, 0.0101977 , 0.01549192, 0.03350013,
       0.01054586, 0.01412726, 0.01387186, 0.0111771 , 0.01859977,
       0.02302986, 0.02470228, 0.02126374, 0.02458561, 0.01173058,
       0.01784785, 0.01551921, 0.01646912, 0.01757362, 0.01756087,
       0.01311441, 0.02536714, 0.01954938, 0.03271354, 0.01972259,
       0.01686151, 0.01117707, 0.02425726, 0.0172456 , 0.01144799,
       0.01656203, 0.01861042, 0.00877684, 0.01750532, 0.02783596,
       0.02307991, 0.01453139, 0.02147229, 0.02002779, 0.01057966,
       0.02432578, 0.01992754, 0.01991193, 0.0141263 , 0.01524955,
       0.02300707, 0.01514282, 0.02518471, 0.00963853, 0.03426393,
       0.01675604, 0.01743034, 0.01469276, 0.01947351, 0.01526028,
       0.01762555, 0.00891232, 0.02097956, 0.01905084, 0.00979381,
       0.0066181 , 0.00978379, 0.01860343, 0.01706066, 0.02883298,
       0.00756637, 0.01953651, 0.007647  , 0.0186088 , 0.01848868,
       0.01443527, 0.00878512, 0.01954781, 0.01585722, 0.0131004 ,
       0.0144795 , 0.02170516, 0.01815112, 0.01631936, 0.0177237,
       0.03108736, 0.01698767, 0.01085187, 0.01430984, 0.00917604,
       0.02043364, 0.01690143, 0.02252791, 0.01007615, 0.01393212,
       0.01915265, 0.02661706, 0.01160692, 0.02675001, 0.0128878 ,
       0.0203684 , 0.01740005, 0.01529127, 0.00899668, 0.01061191,
       0.02161383, 0.02235791, 0.01313583, 0.01085924, 0.02507223,
       0.01386489, 0.00990224, 0.01647365, 0.01647599, 0.01145422,
       0.02601158, 0.01846643, 0.01647167, 0.01444472, 0.01790716,
       0.01527652, 0.01537391, 0.01585855, 0.01594076, 0.0165197 ,
       0.02267513, 0.01934798, 0.01243319, 0.02636654, 0.01828716,
       0.0244514 , 0.02933638, 0.01137558, 0.01383904, 0.02031605,
       0.01083434, 0.01371933, 0.01395322, 0.01220539, 0.00850149,
       0.02553756, 0.01351714, 0.02221336, 0.01828487, 0.02053147,
       0.01477501, 0.02058134, 0.00906027, 0.02490342, 0.01271939,
       0.00807848, 0.01641834, 0.0207138 , 0.02204211, 0.01578005,
       0.01701663, 0.02059307, 0.01972185, 0.01892898, 0.02180095,
       0.023869  , 0.02172857, 0.02222568, 0.01404804, 0.02636908,
       0.01802723, 0.00966441, 0.02361758, 0.03520979, 0.01109582,
       0.0164265 , 0.0118379 , 0.01250536, 0.01904816, 0.01792719,
       0.01748465, 0.01826239, 0.01634527, 0.02900813, 0.02630445,
       0.03076924, 0.02533985, 0.02499371, 0.01310629, 0.01645636,
       0.01961468, 0.01502688, 0.02303176, 0.01766333, 0.0114605 ,
       0.02745647, 0.01872278, 0.01977256, 0.02098619, 0.01711142,
       0.01323196, 0.01504697, 0.02883058, 0.01798728, 0.01454334,
       0.01623504, 0.01980402, 0.02182559, 0.01137838, 0.02130051,
       0.02497323, 0.02101561, 0.02486651, 0.01132993, 0.02478787,
       0.02687415, 0.02025087, 0.02164845, 0.01171314, 0.01382717,
       0.01015512, 0.00758542, 0.01226884, 0.01533492, 0.02520869,
       0.01360755, 0.01219129, 0.01581971, 0.01852451, 0.01935774])


LH_blank = np.array([0.01173907, 0.01671609, 0.01328134, 0.01572196, 0.01246392,
       0.01419775, 0.0134874 , 0.016825  , 0.01462631, 0.02101087,
       0.02238432, 0.01210101, 0.01685883, 0.02453879, 0.02117584,
       0.02144984, 0.0277172 , 0.01713768, 0.0132939 , 0.01458024,
       0.01713077, 0.03960168, 0.03018391, 0.01446051, 0.01645248,
       0.0197135 , 0.01547749, 0.01826562, 0.01551218, 0.01756696,
       0.01554278, 0.01372506, 0.01177651, 0.02210335, 0.01999756,
       0.0280613 , 0.02357611, 0.01642352, 0.01696699, 0.01592029,
       0.01214593, 0.01973377, 0.02123124, 0.01238809, 0.01789073,
       0.02119333, 0.01860612, 0.02213315, 0.01228433, 0.02298727,
       0.01432964, 0.02398576, 0.01956945, 0.01524143, 0.0128571 ,
       0.03279978, 0.01885539, 0.0147369 , 0.00815016, 0.02095101,
       0.01329874, 0.01876593, 0.0144998 , 0.01541098, 0.0173886 ,
       0.01217361, 0.0139112 , 0.02124608, 0.01244899, 0.01476102,
       0.02306091, 0.02178363, 0.01883102, 0.02067787, 0.00737882,
       0.01997731, 0.023053  , 0.01379017, 0.01844824, 0.01365592,
       0.01515464, 0.02358586, 0.01924864, 0.02191645, 0.01800268,
       0.01669202, 0.01030453, 0.01598953, 0.01360406, 0.01411836,
       0.02141641, 0.01710006, 0.01036399, 0.02510254, 0.02246132,
       0.02592154, 0.0169996 , 0.01238996, 0.02170249, 0.01004694,
       0.02691189, 0.01735904, 0.01664837, 0.01493991, 0.02142278,
       0.02224971, 0.00887164, 0.02033879, 0.01309672, 0.02232294,
       0.01295965, 0.01194146, 0.02029571, 0.01956901, 0.01791371,
       0.0166137 , 0.00688678, 0.01394506, 0.01621512, 0.0097554 ,
       0.00871196, 0.01200733, 0.01754177, 0.01928188, 0.02385043,
       0.00597239, 0.02510715, 0.0069319 , 0.01545334, 0.0197634 ,
       0.01447616, 0.00924064, 0.01876592, 0.01732293, 0.01062856,
       0.01772209, 0.0210629 , 0.02200535, 0.01312718, 0.01944589,
       0.02849782, 0.0162253 , 0.02172259, 0.01638373, 0.01275356,
       0.01849395, 0.01472295, 0.02478584, 0.00743195, 0.01790588,
       0.02104423, 0.02402892, 0.0139067 , 0.02838246, 0.01658621,
       0.01698267, 0.02204968, 0.02471447, 0.00981282, 0.01046402,
       0.01931858, 0.01962362, 0.01175898, 0.00968246, 0.02202152,
       0.0139466 , 0.00950151, 0.01332848, 0.02150309, 0.00761243,
       0.02874283, 0.0177779 , 0.01488973, 0.0131313 , 0.01732361,
       0.01901901, 0.01704625, 0.01961628, 0.02089721, 0.01272492,
       0.01953329, 0.02332706, 0.0077782 , 0.01716139, 0.01060251,
       0.0164972 , 0.02484345, 0.01491021, 0.01309185, 0.02320076,
       0.00741879, 0.01818582, 0.01494764, 0.01232491, 0.01836978,
       0.02028832, 0.0144585 , 0.02013793, 0.01553431, 0.00707197,
       0.01522741, 0.02015876, 0.00797567, 0.0296307 , 0.01210931,
       0.00878076, 0.02088972, 0.019505  , 0.01660328, 0.01270293,
       0.01714694, 0.01533946, 0.02569759, 0.01348157, 0.0122238 ,
       0.02340313, 0.02108743, 0.01635429, 0.01273061, 0.02283466,
       0.01584261, 0.01024013, 0.02483294, 0.01599055, 0.01069407,
       0.01309738, 0.01232864, 0.01228248, 0.01873563, 0.01939459,
       0.02198039, 0.02110797, 0.01438732, 0.02416476, 0.02264175,
       0.02959994, 0.01859395, 0.01819473, 0.01630725, 0.01907007,
       0.0198217 , 0.01753156, 0.0189396 , 0.01832184, 0.01110244,
       0.02966059, 0.01404671, 0.02029747, 0.02349346, 0.01830204,
       0.01667922, 0.01225811, 0.02715852, 0.0179516 , 0.01506789,
       0.01316973, 0.01745994, 0.02262946, 0.01228198, 0.02311541,
       0.01971733, 0.01802267, 0.02045813, 0.01028526, 0.02642172,
       0.02648379, 0.01634848, 0.01929136, 0.01428894, 0.01400413,
       0.02001111, 0.00586254, 0.00991013, 0.01430796, 0.02217456,
       0.00907957, 0.01388092, 0.0193133 , 0.00873184, 0.01729049])

LHR = np.sqrt(len(LH_true))*(LH_true - LH_blank)
sns.set(color_codes=True)
LHR_dis = sns.distplot(LHR, fit=norm, kde=False);
# LHR_dis.set(xlim=(-.03, .04))
plt.title('Histgram of the likelihhod ratio based on 350 samples of random split')
plt.ylabel('count')
plt.show()

import scipy
Z_score = (LHR - 0.) / np.std(LHR)
pval = 1. - scipy.stats.norm.sf(Z_score) #twosided
## Null hypothesis is True, compute the type 2 error.
type_one_err = len(pval[pval<.05])/len(pval)
type_two_err = len(pval[pval>=.05])/len(pval)
print('Null hypothesis is True, and the type 1 error is %.3f' %type_one_err)
# Null hypothesis is True, and the type 1 error is 0.064

## test 3: Null hypothesis is true.
LH_true = np.array([0.03530239, 0.01465264, 0.00907827, 0.01716359, 0.00885248,
       0.02304688, 0.01450974, 0.02139546, 0.00517216, 0.01897794,
       0.02495936, 0.02322404, 0.01447472, 0.01775757, 0.01978167,
       0.01705322, 0.01664152, 0.02354305, 0.00758391, 0.01419944,
       0.01914344, 0.01598736, 0.01481371, 0.01119122, 0.02141743,
       0.01890822, 0.01060044, 0.01320479, 0.02125933, 0.00770559,
       0.03187721, 0.02149893, 0.01378444, 0.02296793, 0.02285571,
       0.01379398, 0.02106054, 0.01763897, 0.01979187, 0.01482453,
       0.0210673 , 0.01993326, 0.00930613, 0.02084823, 0.01283187,
       0.02631209, 0.02266766, 0.01576612, 0.01437757, 0.02004227,
       0.01238926, 0.02278669, 0.01368972, 0.01372949, 0.00777934,
       0.02609723, 0.01573571, 0.02211801, 0.01423231, 0.01167674,
       0.01561831, 0.01667569, 0.00828553, 0.0238652 , 0.01439228,
       0.00829232, 0.0190263 , 0.01753628, 0.01914635, 0.01571384,
       0.01390072, 0.01506484, 0.02411429, 0.01531119, 0.0127624 ,
       0.02753928, 0.02176792, 0.02407449, 0.01462081, 0.02095964,
       0.01914368, 0.00969721, 0.01901971, 0.02050135, 0.01022556,
       0.0179642 , 0.01356226, 0.01475663, 0.01795925, 0.01809724,
       0.0163185 , 0.01577545, 0.01312407, 0.02373167, 0.02849648,
       0.03500406, 0.02439517, 0.02693511, 0.01052537, 0.01356277,
       0.01725365, 0.02044969, 0.01911787, 0.01845502, 0.01676364,
       0.02882498, 0.01644795, 0.01722672, 0.01969678, 0.01798604,
       0.01731728, 0.00969171, 0.02547352, 0.01592522, 0.01342463,
       0.01294406, 0.0206435 , 0.02087885, 0.0127151 , 0.02780009,
       0.02504913, 0.02195319, 0.02879592, 0.01039277, 0.02174954,
       0.02956753, 0.01837184, 0.02129254, 0.01199637, 0.01752793,
       0.0121228 , 0.00762677, 0.01519742, 0.01291183, 0.03133999,
       0.00696626, 0.01236547, 0.01230593, 0.01200311, 0.0215418 ,
       0.01646293, 0.02534558, 0.01649229, 0.01814482, 0.01966562,
       0.01758827, 0.01276869, 0.01982127, 0.02004105, 0.02050831,
       0.02473435, 0.0134885 , 0.01655339, 0.01704345, 0.01889704,
       0.0166512 , 0.02473536, 0.01387404, 0.01299484, 0.01180163,
       0.01705197, 0.02248541, 0.02628241, 0.01699344, 0.01835425,
       0.01786291, 0.01791776, 0.01538165, 0.01600117, 0.02300823,
       0.01091777, 0.01383207, 0.01381913, 0.01438094, 0.01481894,
       0.02926018, 0.01799479, 0.0194584 , 0.01754149, 0.0112764 ,
       0.01614402, 0.0178998 , 0.01618423, 0.01724838, 0.01763935,
       0.01892249, 0.02133302, 0.01971874, 0.01122411, 0.02564554,
       0.01545839, 0.03690646, 0.01945257, 0.01516679, 0.00972061,
       0.02435466, 0.02044011, 0.01710021, 0.01194377, 0.02901902,
       0.01077983, 0.0178732 , 0.01299096, 0.01470877, 0.01883627,
       0.01075928, 0.01358261, 0.01933536, 0.01303887, 0.0155286 ,
       0.02637749, 0.02806207, 0.02319586, 0.01609563, 0.01048064,
       0.01562891, 0.01226184, 0.01382432, 0.02440327, 0.01565143,
       0.00959443, 0.02160321, 0.0247046 , 0.0228768 , 0.01660699,
       0.01673685, 0.01300497, 0.02519848, 0.02097665, 0.01117389,
       0.0131581 , 0.01745152, 0.00755802, 0.01574522, 0.01754735,
       0.02554392, 0.01697002, 0.02224931, 0.018328  , 0.00972998,
       0.01910048, 0.01620759, 0.01808782, 0.01461288, 0.01355871,
       0.02128513, 0.01743146, 0.01974954, 0.0078469 , 0.02251644,
       0.01577887, 0.01543592, 0.01788883, 0.0156647 , 0.01684469,
       0.0182239 , 0.01340232, 0.01234404, 0.02046655, 0.01033793,
       0.0089193 , 0.01603845, 0.01495512, 0.02689486, 0.03213946,
       0.00580244, 0.02043279, 0.00887945, 0.0176937 , 0.01948263,
       0.01745038, 0.00693908, 0.02067552, 0.01421092, 0.01193196,
       0.0174413 , 0.01778943, 0.02476338, 0.01679437, 0.02396268])

LH_blank = np.array([0.02354782, 0.00998313, 0.01101701, 0.01716404, 0.0162417 ,
       0.03004563, 0.0130768 , 0.02422732, 0.00984059, 0.01628238,
       0.02461652, 0.02106476, 0.01376611, 0.02551365, 0.01765434,
       0.02100639, 0.02109534, 0.01836961, 0.01086823, 0.0091393 ,
       0.02021854, 0.02124841, 0.01012425, 0.01337569, 0.0271158 ,
       0.01617562, 0.01712195, 0.01453155, 0.01951992, 0.01079534,
       0.02636386, 0.01754987, 0.02119793, 0.01188576, 0.02017739,
       0.01765511, 0.01689797, 0.01878546, 0.01390778, 0.01436565,
       0.02386363, 0.01014929, 0.00784966, 0.01692514, 0.00929854,
       0.02386939, 0.02505442, 0.01314343, 0.01157001, 0.01979486,
       0.00985758, 0.01551164, 0.01731174, 0.01560519, 0.00836956,
       0.02117163, 0.01545744, 0.02142206, 0.01505816, 0.00665808,
       0.01640182, 0.01770428, 0.00603641, 0.01768817, 0.01284768,
       0.00868248, 0.01346026, 0.01945827, 0.01490585, 0.01445105,
       0.01330155, 0.02041911, 0.01926657, 0.01499021, 0.0124376 ,
       0.02339724, 0.01948927, 0.02251236, 0.01778095, 0.02462561,
       0.01346754, 0.01478335, 0.02401428, 0.02304888, 0.00844672,
       0.01336142, 0.01206983, 0.01455824, 0.0170895 , 0.01365552,
       0.02106254, 0.01736773, 0.011879  , 0.02039843, 0.02951163,
       0.02981202, 0.02400628, 0.02130278, 0.01207581, 0.01549557,
       0.02346974, 0.01746005, 0.01865692, 0.02194032, 0.01386908,
       0.02428473, 0.01447441, 0.01985095, 0.02147746, 0.02079464,
       0.01616592, 0.01066681, 0.02609863, 0.0183245 , 0.01182937,
       0.0162606 , 0.01616773, 0.02141142, 0.01354127, 0.02583899,
       0.02322992, 0.02529407, 0.02625163, 0.00994472, 0.01971464,
       0.02839851, 0.01408134, 0.01685431, 0.01258595, 0.01172132,
       0.01385251, 0.00819719, 0.00813253, 0.0192377 , 0.02905186,
       0.01081188, 0.01529155, 0.01325302, 0.01339183, 0.02150603,
       0.02212793, 0.02318146, 0.01140128, 0.01348471, 0.01180734,
       0.01308139, 0.01592545, 0.01630013, 0.01618704, 0.01858067,
       0.01990996, 0.0123917 , 0.01784735, 0.01803621, 0.02106338,
       0.02243103, 0.02614777, 0.01300051, 0.01246187, 0.01484621,
       0.01614773, 0.02453341, 0.02448535, 0.01367303, 0.0151378 ,
       0.02154124, 0.01849798, 0.02786709, 0.01591004, 0.01921536,
       0.01526651, 0.01476604, 0.01450387, 0.01528149, 0.01915243,
       0.02511228, 0.02279655, 0.02146604, 0.02125344, 0.01454827,
       0.01913756, 0.01422387, 0.02232206, 0.01394871, 0.01657717,
       0.02000843, 0.01260112, 0.02005666, 0.01422388, 0.02228687,
       0.0148398 , 0.02513854, 0.02156465, 0.02223945, 0.01473202,
       0.02639303, 0.01876766, 0.01741004, 0.00960972, 0.02110572,
       0.01385373, 0.01788645, 0.01194458, 0.0141506 , 0.02292721,
       0.01424513, 0.0127423 , 0.01799808, 0.0106818 , 0.01542924,
       0.01939736, 0.02570669, 0.02015132, 0.01666969, 0.01306803,
       0.01746445, 0.02808474, 0.01707265, 0.02419964, 0.01688942,
       0.00867974, 0.01682128, 0.02442153, 0.02456922, 0.01538537,
       0.01503679, 0.01399307, 0.01651568, 0.0168345 , 0.01424225,
       0.01387298, 0.01210147, 0.00912979, 0.01761029, 0.02150022,
       0.02694846, 0.01226073, 0.0142799 , 0.02118374, 0.00933003,
       0.02706166, 0.01426454, 0.01968939, 0.01374148, 0.01852066,
       0.01928957, 0.01186029, 0.02034095, 0.00894612, 0.01789218,
       0.01746261, 0.01357101, 0.01740885, 0.01685188, 0.01981959,
       0.0181081 , 0.01303871, 0.01056494, 0.01853084, 0.01021854,
       0.0062674 , 0.01389028, 0.01759902, 0.03160899, 0.02299682,
       0.00522207, 0.02453184, 0.00864558, 0.02275185, 0.02238369,
       0.01646188, 0.01536592, 0.02655552, 0.0147445 , 0.011268  ,
       0.01936394, 0.01844726, 0.01914403, 0.01171202, 0.02905923])


LHR = np.sqrt(len(LH_true))*(LH_true - LH_blank)
sns.set(color_codes=True)
LHR_dis = sns.distplot(LHR, fit=norm, kde=False);
# LHR_dis.set(xlim=(-.03, .04))
plt.title('Histgram of the likelihhod ratio based on 350 samples of random split')
plt.ylabel('count')
plt.show()

import scipy
## reject H0 if Z_score > 1.645
Z_score = (LHR - 0.) / np.std(LHR)
pval = 1. - scipy.stats.norm.sf(Z_score) #twosided
## Null hypothesis is True, compute the type 2 error.
type_one_err = len(pval[pval<.05])/len(pval)
type_two_err = len(pval[pval>=.05])/len(pval)
print('Null hypothesis is True, and the type 1 error is %.3f' %type_one_err)
# Null hypothesis is True, and the type 1 error is 0.054


## test 4: Null hypothesis is true.
LH_true = np.array([0.0289909 , 0.00996564, 0.01123361, 0.01290034, 0.01207044,
       0.01962606, 0.01488437, 0.02852117, 0.00821628, 0.01809762,
       0.03042342, 0.02449683, 0.01144128, 0.01693983, 0.02584457,
       0.01695727, 0.01528515, 0.0164863 , 0.00806371, 0.01366124,
       0.02198016, 0.02355744, 0.01350532, 0.01552365, 0.02278013,
       0.01387299, 0.00962465, 0.01561164, 0.01612276, 0.00883953,
       0.0240904 , 0.01796866, 0.01394063, 0.01449452, 0.02113872,
       0.01742177, 0.01930521, 0.01936398, 0.01258035, 0.0143397 ,
       0.02087961, 0.01817188, 0.00843492, 0.01770836, 0.01159862,
       0.02341103, 0.02388606, 0.0129844 , 0.01505446, 0.01725272,
       0.01062052, 0.01971357, 0.01611996, 0.01632485, 0.00876658,
       0.02040592, 0.0198433 , 0.0197553 , 0.01796314, 0.00759999,
       0.01476131, 0.01563961, 0.00649608, 0.0274707 , 0.01739049,
       0.00867047, 0.01644101, 0.01690963, 0.01731512, 0.01558203,
       0.01136403, 0.01506615, 0.0208101 , 0.01249859, 0.01245506,
       0.02465407, 0.02467079, 0.02376276, 0.01518803, 0.02281447,
       0.01408916, 0.01273823, 0.02391632, 0.01908522, 0.00706052,
       0.01081106, 0.00995431, 0.01921426, 0.01768981, 0.01837688,
       0.01754725, 0.01426835, 0.0132183 , 0.0262989 , 0.03154348,
       0.03115247, 0.02010987, 0.02319532, 0.01583593, 0.01898613,
       0.01981151, 0.01827441, 0.01806215, 0.02052882, 0.01512827,
       0.02891164, 0.02527411, 0.01522687, 0.02390867, 0.02024227,
       0.0166402 , 0.01124451, 0.02685761, 0.01869404, 0.01547149,
       0.01786958, 0.02155147, 0.02190259, 0.01612235, 0.02326884,
       0.02122678, 0.02408497, 0.02825696, 0.00865343, 0.01815083,
       0.03174433, 0.01735963, 0.02428563, 0.01287843, 0.01726718,
       0.0151704 , 0.009198  , 0.01463456, 0.01211291, 0.02160178,
       0.01086677, 0.00938859, 0.01563826, 0.00991477, 0.01837406,
       0.01671532, 0.01589329, 0.01426812, 0.01881311, 0.0161734 ,
       0.01425187, 0.01679728, 0.02438927, 0.01969577, 0.01768218,
       0.0184275 , 0.0116609 , 0.02373066, 0.01984298, 0.01791676,
       0.02490733, 0.02447246, 0.01420444, 0.01428614, 0.01844663,
       0.02129448, 0.02881367, 0.01914889, 0.01761685, 0.02309182,
       0.02224094, 0.02176841, 0.01381737, 0.01391518, 0.02177258,
       0.01645317, 0.01658206, 0.0145876 , 0.01278772, 0.01537471,
       0.02695636, 0.02153564, 0.01359723, 0.01901507, 0.01270971,
       0.01761031, 0.01740305, 0.01578889, 0.01333962, 0.02471446,
       0.01926222, 0.01622406, 0.01757346, 0.0106165 , 0.02050819,
       0.01262682, 0.03337195, 0.02144978, 0.02231261, 0.01474247,
       0.03102731, 0.01722711, 0.02041766, 0.01251339, 0.02524772,
       0.01516908, 0.02839136, 0.0131279 , 0.01529364, 0.01490431,
       0.00984593, 0.01484787, 0.02193208, 0.01080478, 0.01408173,
       0.02008057, 0.02228339, 0.02141879, 0.01559271, 0.00901815,
       0.01876958, 0.01726154, 0.02493751, 0.02270226, 0.01652921,
       0.01102679, 0.02779711, 0.02756574, 0.02438172, 0.01748243,
       0.01504819, 0.01449042, 0.01447473, 0.01937437, 0.01357669,
       0.0132955 , 0.01601931, 0.00860951, 0.01683842, 0.02165327,
       0.03335896, 0.01815091, 0.01908513, 0.01949186, 0.01247295,
       0.02377122, 0.01723096, 0.02142523, 0.01113056, 0.01846689,
       0.0292811 , 0.01817185, 0.02162947, 0.01077532, 0.02913011,
       0.01377394, 0.0186011 , 0.01965088, 0.02367611, 0.01793326,
       0.01621748, 0.00847194, 0.02164113, 0.02080823, 0.01034334,
       0.00795586, 0.01626293, 0.0183487 , 0.01876439, 0.03082857,
       0.00873457, 0.01988891, 0.00938579, 0.0191887 , 0.01701396,
       0.01444853, 0.01266615, 0.01902117, 0.01673189, 0.01123494,
       0.0161766 , 0.02273498, 0.01950855, 0.01731316, 0.03018586])

LH_blank = np.array([0.02754888, 0.00945395, 0.00847625, 0.01624959, 0.01023416,
       0.02221447, 0.01927047, 0.02489335, 0.00751704, 0.01335426,
       0.02821649, 0.02363456, 0.01019367, 0.01616856, 0.02439001,
       0.01856291, 0.016564  , 0.01706684, 0.00708995, 0.01139343,
       0.02207098, 0.025556  , 0.01381212, 0.01207094, 0.0225565 ,
       0.01129522, 0.015519  , 0.01426059, 0.02005222, 0.00788713,
       0.02477374, 0.02046337, 0.0167452 , 0.0110868 , 0.0227084 ,
       0.01788108, 0.02001117, 0.0161904 , 0.01602249, 0.01134937,
       0.02195418, 0.01566902, 0.01169857, 0.01496544, 0.01537   ,
       0.0192298 , 0.02683562, 0.01198347, 0.01240656, 0.01929848,
       0.00968842, 0.01546787, 0.01662707, 0.01461375, 0.00841651,
       0.02033528, 0.02261378, 0.01675596, 0.01667228, 0.01198357,
       0.01449488, 0.01781319, 0.0072589 , 0.0269766 , 0.01420894,
       0.00813927, 0.02308196, 0.01573863, 0.01360331, 0.01214438,
       0.016219  , 0.01722435, 0.01838013, 0.02018908, 0.01124245,
       0.02250284, 0.02184906, 0.02355959, 0.01516464, 0.0245342 ,
       0.01796559, 0.01369538, 0.01911424, 0.02158883, 0.01142766,
       0.0147999 , 0.01243394, 0.01052699, 0.02428577, 0.01458551,
       0.0166105 , 0.01928042, 0.01302348, 0.02329473, 0.0334364 ,
       0.03039343, 0.0219527 , 0.02407545, 0.01509779, 0.01413944,
       0.01720437, 0.01662247, 0.02419186, 0.01486672, 0.01178108,
       0.02479227, 0.01720325, 0.02299271, 0.01855309, 0.02120519,
       0.01648258, 0.01516635, 0.02405041, 0.01954971, 0.01470832,
       0.0137072 , 0.02247973, 0.0246729 , 0.0118543 , 0.02124191,
       0.02921759, 0.02552707, 0.02498186, 0.00851958, 0.01540462,
       0.02610336, 0.01278312, 0.02860654, 0.01133789, 0.01177596,
       0.0134692 , 0.00931353, 0.01141157, 0.01423861, 0.02398663,
       0.00774   , 0.01451289, 0.01190705, 0.01674905, 0.02722802,
       0.01605798, 0.02064635, 0.01490526, 0.01558372, 0.0168699 ,
       0.0169924 , 0.01766041, 0.01613829, 0.01845394, 0.02466066,
       0.01644896, 0.01875719, 0.0242744 , 0.02002758, 0.01986131,
       0.01941583, 0.0276761 , 0.01809065, 0.02231477, 0.00979567,
       0.02938028, 0.0276381 , 0.02250664, 0.01398334, 0.01718513,
       0.01903054, 0.01817006, 0.01701543, 0.0158068 , 0.02190274,
       0.01277664, 0.01423951, 0.01743211, 0.0120163 , 0.02162563,
       0.028682  , 0.02657547, 0.01657655, 0.01958296, 0.02046204,
       0.01404258, 0.01885593, 0.02116319, 0.01600114, 0.01853958,
       0.02945167, 0.01417332, 0.01510813, 0.0161871 , 0.02191602,
       0.02481774, 0.02465937, 0.0291642 , 0.01718868, 0.01126912,
       0.0283575 , 0.01704062, 0.02041574, 0.01100693, 0.02141859,
       0.01415878, 0.0187956 , 0.01610687, 0.01315301, 0.01619489,
       0.01298056, 0.01316301, 0.01471072, 0.01378045, 0.02173893,
       0.02518685, 0.02154309, 0.01792016, 0.01603059, 0.01246198,
       0.01791274, 0.01774446, 0.01640391, 0.03453085, 0.01464772,
       0.01059962, 0.0291205 , 0.02558237, 0.01803485, 0.01608675,
       0.01915512, 0.01302254, 0.02284217, 0.01252803, 0.02460786,
       0.02130296, 0.01563684, 0.01044095, 0.02110147, 0.02397927,
       0.02131267, 0.0221873 , 0.02260934, 0.02446898, 0.01082514,
       0.02152123, 0.02074579, 0.02024373, 0.01477959, 0.01566314,
       0.02363139, 0.01827771, 0.01908971, 0.01203373, 0.0248739 ,
       0.01465098, 0.01531631, 0.01666149, 0.01572107, 0.02249708,
       0.01888601, 0.00686697, 0.02631018, 0.02402434, 0.01085107,
       0.0074041 , 0.00957816, 0.02026349, 0.01630594, 0.02764961,
       0.00541836, 0.01868962, 0.01099102, 0.02179069, 0.01917533,
       0.02084621, 0.00877173, 0.03045157, 0.01412261, 0.01361984,
       0.02524971, 0.02507405, 0.01984367, 0.01100482, 0.02282196])

LHR = np.sqrt(len(LH_true))*(LH_true - LH_blank)
# LHR = (LH_true - LH_blank)
sns.set(color_codes=True)
LHR_dis = sns.distplot(LHR, fit=norm, kde=False);
# LHR_dis.set(xlim=(-.03, .04))
plt.title('Histgram of the likelihhod ratio based on 350 samples of random split')
plt.ylabel('count')
plt.show()

sns.distplot(LH_true, fit=norm, kde=False);
sns.distplot(LH_blank, fit=norm, kde=False);
# plt.set(xlim=(None, .04))
plt.title('Histgram of two distributions of likelihhod based on 350 samples of random split')
plt.ylabel('count')
plt.show()

Z_score = (LHR - 0.) / np.std(LHR)
pval = 1 - scipy.stats.norm.sf(Z_score) #twosided
## Null hypothesis is True, compute the type 2 error.
type_one_err = len(pval[pval<.05])/len(pval)
type_two_err = len(pval[pval>=.05])/len(pval)
print('Null hypothesis is True, and the type 1 error is %.3f' %type_one_err)
# Null hypothesis is True, and the type 1 error is 0.054
