Pcf 634 new stats analysis (mozilla#8944)

* rebased in master * update serializer, handle review comments * update serializer to remove duplicates * update serializer, results * remove flatten * fix serializer * remove unused serializer * add test and fix code to handle errors and exceptions * smooth out minor issues * fix spacing * handle undeclared variable * minor stats grouping * fix variables * fix more changes need reasignment * update functions * update variables names to not be confusing * clean up seralizer, fixed bugs * fix minor text interpretation cleanup * fix endpoint shape and handle nan values * minor restructure * remove print statements * swap back comments * remove redundant values * add ks_p value * use python 3.10 * Delete -u * remove google proxy file * add google cloud proxy file to gitignore * fixing dev.txt compile issue * add the same scipy and numpy as in current treeherder common.txt * remove specifications * Update dev.txt remove changes to numpy and scipy * Update dev.txt Remove kdepy and cliffs-delta from dev * Update dev.in Remove kdepy and cliffsdelta from dev

Author: misspran

.gitignore

@@ -55,3 +55,6 @@ docs.egg-info
 mcp-servers/
 .claude
 CLAUDE.md
+
+#Cloud SQL Auth Proxy
+cloud-sql-proxy

requirements/common.in

@@ -52,3 +52,7 @@ inflection==0.5.1
 
 # Change detection tooling
 mozdetect==0.0.9
+
+# Statistical methods for performance analysis
+cliffs-delta==1.0.0
+KDEpy==1.1.12

requirements/common.txt

@@ -377,6 +377,10 @@ click-repl==0.3.0 \
  --hash=sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9 \
  --hash=sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812
  # via celery
+cliffs-delta==1.0.0 \
+ --hash=sha256:2ec087e8254861204aab79e222b2e863a79f3d6f35d69e14f8b188fb6c5b048f \
+ --hash=sha256:909595ceaf051390fc82e662fdefb5ccca2cd3f49bb23531d903a53ae9f07e43
+ # via -r requirements/common.in
 crashtest==0.4.1 \
  --hash=sha256:80d7b1f316ebfbd429f648076d6275c877ba30ba48979de4191714a75266f0ce \
  --hash=sha256:8d23eac5fa660409f57472e3851dab7ac18aba459a8d19cbbba86d3d5aecd2a5
@@ -692,6 +696,45 @@ jsonschema-specifications==2023.12.1 \
  --hash=sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc \
  --hash=sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c
  # via jsonschema
+kdepy==1.1.12 \
+ --hash=sha256:0834ff21a74fbd85ee748a8e9ff0edc886e0dd2c9a7011d4f69a943029a61e3d \
+ --hash=sha256:0fe3784dd2bfaa4d6278f2fcc5e0140dae0b643f0ad9b15c5bed35d3e368c7a4 \
+ --hash=sha256:1089bbdc007c0495556d6409f7c9235abfc79071c7147781ca6f644aad4e5eb4 \
+ --hash=sha256:2b00d31897ce5c88362149a5651048e594a5c0f7db7b85f0ace7767604bf004f \
+ --hash=sha256:34ba64f462127e2dc6a1de9ae4a742785cc97acf9fc9922a1378e3087e19c494 \
+ --hash=sha256:35bdfc7f046014b6bdf83d6d14b2adbd8f08c1d3b83d63a91d74970dc8085972 \
+ --hash=sha256:3bd8bff8b5d18ccc253c22dcfa6ab1aa8098df62fbb8899fa6d25579f0ff0b8f \
+ --hash=sha256:3d9a4ffcf155f4fa7e45e3ff33a8436e16a8e405fa8bd610803a697dc0dd6392 \
+ --hash=sha256:4f7deb921bbe944247bf8c4d2b2078dd67be8130d3be4a32f947766daed60095 \
+ --hash=sha256:562d1d7a63e61209a9c2852e0de67090abe90def2e47deaf7d4bd935d36ba67b \
+ --hash=sha256:64a837b0c552d681ef7bdfa5b0e18bbbc66e3d5dac7ad7fc9919c881e0f16498 \
+ --hash=sha256:6e22ea5a38278e86be531e87da07026a7aaa0450fdf7b8f6e8813ef9c04e5de0 \
+ --hash=sha256:6e5d59f51dd1797b7729e09993986f2920f45e62609c3704c5946aa5f05171f1 \
+ --hash=sha256:78f319dbaafe44d8583e5f1ff3dcff14124d9718ff5bdf5b38128180fbbe4c06 \
+ --hash=sha256:878cfcfefa32dfd464b0a87b2ecc90239ade075528285ba8893b3c559dc6e132 \
+ --hash=sha256:8f26a6a8deeed57a0f52a35fbf0033acedb5935c3a9ae028209a3758bc45d86e \
+ --hash=sha256:8fd2a55b71e07aac63b8c8614a8f3e44268eabb9d4282e7bb78a11e840b20268 \
+ --hash=sha256:94b555ec63d953ea3fb3b5a00fe7e7efadd7881cc6fe817551df913d38796de2 \
+ --hash=sha256:9e5115b65531d36ab48b9e9dd952221bb7789999f95c7b44564367f885f5ade5 \
+ --hash=sha256:aaa900fbf3fe66bb30a6e1069bd90940f00a02abc15fd6217b63df2fd07204a5 \
+ --hash=sha256:aea3587396e458b53c63c8a070505bcd7fd7e60756f99d3390ee344e22d08f41 \
+ --hash=sha256:b0cf62dc5a255075a5813965a58ac16b8b1144d538760bccf06ac529720db61e \
+ --hash=sha256:b2f04a95a6d706ab6fcfba104c49b97e4b6ade40212bcface291e4ab6747eed0 \
+ --hash=sha256:c5fdc115c212df1a622997a9d8ff9c1bae8179562cc8f2e9710adced96abfd20 \
+ --hash=sha256:c7c5b091e3034d026bcf718a5efecbe2a1eee5100d1328621387d7fadb164dd8 \
+ --hash=sha256:d3839f95cfd45084901a0d9c98b4ac83a5eb57449a6be7004d422cf29d9eedff \
+ --hash=sha256:df4335cac2eb387265ec93911a1fcf6520d5a6d2a7762cb00e134e6c3beae18d \
+ --hash=sha256:e0c5a22798499154e1aef0378fbeb04f8832dd74d05246fcdbb5438e3c4424e1 \
+ --hash=sha256:e1928bf44c9ee7f78239461d5670e8515f5d4a201eff5622547967c7c20fa82d \
+ --hash=sha256:e62facb46ea6d6eff92a0f8ad054bdf4b5a84ec66444b5cae9d5af1f828b07b7 \
+ --hash=sha256:e9ffa424aa96988a9b7f0154168c17cf64cba46ad691a556e77e51cae7992864 \
+ --hash=sha256:eb3a62abc5a982f5a30ec0a3560e2f9cbd3d19bc4f721b2ebbde06949d52af61 \
+ --hash=sha256:f3fccc80e33e291752d2cee7686628c19f066806b9b3f62d7d007c09c252c8fc \
+ --hash=sha256:f64b740a61f60076630cfcdd1ff011c70d1ce1a7ceab05b14763d95163140e46 \
+ --hash=sha256:f6a98453558eb39fe093601dea19769ebb13e371e11a212be516b27b4f059a55 \
+ --hash=sha256:ff27b711d3e914da68a80bdd8044d7992a9a48be090378be246f6b39d91b6e46 \
+ --hash=sha256:ff6a4db7b40a63b9accde48ebfd25e664138f37eda56ad1be34fe37f60ce3d84
+ # via -r requirements/common.in
 kombu==5.5.2 \
  --hash=sha256:2dd27ec84fd843a4e0a7187424313f87514b344812cb98c25daddafbb6a7ff0e \
  --hash=sha256:40f3674ed19603b8a771b6c74de126dbf8879755a0337caac6602faa82d539cd
@@ -1048,6 +1091,7 @@ numpy==1.26.4 \
  --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f
  # via
  # db-dtypes
+ # kdepy
  # moz-measure-noise
  # pandas
  # scipy
@@ -1781,6 +1825,7 @@ scipy==1.15.1 \
  --hash=sha256:f735bc41bd1c792c96bc426dece66c8723283695f02df61dcc4d0a707a42fc54 \
  --hash=sha256:f82fcf4e5b377f819542fbc8541f7b5fbcf1c0017d0df0bc22c781bf60abc4d8
  # via
+ # kdepy
  # moz-measure-noise
  # mozdetect
 simplejson==3.20.2 \

tests/perf/test_stats.py

@@ -0,0 +1,23 @@
+# from treeherder.model.models import Push
+from treeherder.perf.stats import interpret_silverman_kde
+
+# p-value threshold to use throughout
+PVALUE_THRESHOLD = 0.05
+
+# def test_split_per_mode():
+
+
+def test_interpret_silverman_kde():
+ mock_base = [2.74]
+ mock_new = [2.65]
+ lower_is_better = False
+ (
+ silverman_kde,
+ is_regression,
+ is_improvement,
+ more_runs_are_needed,
+ warning_msgs,
+ performance_intepretation,
+ ) = interpret_silverman_kde(mock_base, mock_new, lower_is_better)
+ assert silverman_kde["bandwidth"] == "Silverman"
+ assert warning_msgs == []

tests/webapp/api/test_perfcompare_api.py

@@ -1044,3 +1044,114 @@ def get_expected(
  new_sig.has_subtests if new_sig else False
  )
  return response
+
+
+def test_perfcompare_results_with_mann_witney_u_against_no_base(
+ client,
+ create_signature,
+ create_perf_datum,
+ test_perf_signature,
+ test_repository,
+ try_repository,
+ eleven_jobs_stored,
+ test_perfcomp_push,
+ test_perfcomp_push_2,
+ test_linux_platform,
+ test_option_collection,
+):
+ perf_jobs = Job.objects.filter(pk__in=range(1, 11)).order_by("push__time").all()
+
+ test_perfcomp_push.time = FOUR_DAYS_AGO
+ test_perfcomp_push.repository = try_repository
+ test_perfcomp_push.save()
+ test_perfcomp_push_2.time = datetime.datetime.now()
+ test_perfcomp_push_2.save()
+
+ suite = "a11yr"
+ test = "dhtml.html"
+ extra_options = "e10s fission stylo webrender"
+ measurement_unit = "ms"
+ base_application = "firefox"
+ new_application = "firefox"
+
+ base_sig = create_signature(
+ signature_hash=(20 * "t1"),
+ extra_options=extra_options,
+ platform=test_linux_platform,
+ measurement_unit=measurement_unit,
+ suite=suite,
+ test=test,
+ test_perf_signature=test_perf_signature,
+ repository=try_repository,
+ application=base_application,
+ )
+
+ base_perf_data_values = [32.4]
+ new_perf_data_values = [40.2]
+
+ job = perf_jobs[0]
+ job.push = test_perfcomp_push
+ job.save()
+ perf_datum = PerformanceDatum.objects.create(
+ value=base_perf_data_values[0],
+ push_timestamp=job.push.time,
+ job=job,
+ push=job.push,
+ repository=try_repository,
+ signature=base_sig,
+ )
+ perf_datum.push.time = job.push.time
+ perf_datum.push.save()
+
+ new_sig = create_signature(
+ signature_hash=(20 * "t2"),
+ extra_options=extra_options,
+ platform=test_linux_platform,
+ measurement_unit=measurement_unit,
+ suite=suite,
+ test=test,
+ test_perf_signature=test_perf_signature,
+ repository=test_repository,
+ application=new_application,
+ )
+
+ job = perf_jobs[1]
+ job.push = test_perfcomp_push_2
+ job.save()
+ perf_datum = PerformanceDatum.objects.create(
+ value=new_perf_data_values[0],
+ push_timestamp=job.push.time,
+ job=job,
+ push=job.push,
+ repository=job.repository,
+ signature=new_sig,
+ )
+ perf_datum.push.time = job.push.time
+ perf_datum.push.save()
+
+ response = get_expected(
+ base_sig,
+ new_sig,
+ extra_options,
+ test_option_collection,
+ new_perf_data_values,
+ base_perf_data_values,
+ )
+
+ query_params = (
+ "?base_repository={}&new_repository={}&new_revision={}&framework={"
+ "}&interval={}&no_subtests=true&test_version={}".format(
+ try_repository.name,
+ test_repository.name,
+ test_perfcomp_push_2.revision,
+ test_perf_signature.framework_id,
+ 604800, # seven days in milliseconds
+ "mann-whitney-u",
+ )
+ )
+
+ response = client.get(reverse("perfcompare-results") + query_params)
+
+ assert response.status_code == 200
+ assert response.json()[0]["base_parent_signature"] is None
+ assert response.json()[0]["new_parent_signature"] is None