SchemathesisでOpenAPIスキーマからプロパティベーステスト(PBT)を行ってみる

uv add fastapi "uvicorn[standard]" schemathesis pytest 

import uuid from typing import Optional from fastapi import Depends, FastAPI, Header, HTTPException from pydantic import BaseModel, Field app = FastAPI(title="Booking API", version="1.0.0") BOOKINGS: dict[str, dict] = {} def verify_token(authorization: Optional[str] = Header(None)) -> bool: if not authorization or authorization != "Bearer secret-token": raise HTTPException(status_code=401, detail="Invalid or missing token") return True class BookingRequest(BaseModel): guest_name: str = Field(min_length=2, max_length=100) room_type: str nights: int = Field(gt=0, le=365) class BookingResponse(BaseModel): booking_id: str guest_name: str room_type: str nights: int status: str price_per_night: float total_price: float @app.post("/bookings", response_model=BookingResponse, responses={400: {"description": "Invalid booking"}}) # type: ignore[misc] def create_booking(booking: BookingRequest, _: bool = Depends(verify_token)) -> BookingResponse: # Calculate price based on room type room_prices = {"standard": 99.99, "deluxe": 149.99, "suite": 299.99} price_per_night = room_prices[booking.room_type] total_price = price_per_night * booking.nights booking_id = str(uuid.uuid4()) booking_data = { "booking_id": booking_id, "guest_name": booking.guest_name, "room_type": booking.room_type, "nights": booking.nights, "status": "confirmed", "price_per_night": price_per_night, "total_price": total_price, } BOOKINGS[booking_id] = booking_data return BookingResponse(**booking_data) @app.get( # type: ignore[misc] "/bookings/{booking_id}", response_model=BookingResponse, responses={404: {"description": "Booking not found"}} ) def get_booking(booking_id: str, _: bool = Depends(verify_token)) -> BookingResponse: if booking_id not in BOOKINGS: raise HTTPException(status_code=404, detail="Booking not found") return BookingResponse(**BOOKINGS[booking_id]) @app.get("/health") # type: ignore[misc] def health_check() -> dict: return {"status": "healthy"} 

uvicorn app:app --reload INFO: Will watch for changes in these directories: ['/tmp/pytest-schemathesis'] INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) INFO: Started reloader process [79391] using WatchFiles INFO: Started server process [79393] INFO: Waiting for application startup. INFO: Application startup complete. 

import schemathesis from hypothesis import settings # from_url を使って、OpenAPIスキーマの場所を指定する schema = schemathesis.openapi.from_url("http://127.0.0.1:8000/openapi.json") # @schema.parametrize() デコレータをテスト関数につけるだけでSchemathesisがスキーマ内の各エンドポイントに対してテストケースを自動生成する @schema.parametrize() @settings(max_examples=500) def test_api(case): """ APIがスキーマ定義に準拠しているかをテストする。 Args: case: Schemathesisが生成した個々のテストケース。 リクエストのメソッド、パス、ヘッダー、ボディなどが含まれる。 """ # case.call_and_validate() は、以下の処理をまとめて実行してくれる便利なメソッド case.call_and_validate(headers={"Authorization": "Bearer secret-token"}) 

headers={"Authorization": "Bearer secret-token"} 

$ pytest test_api.py -v Test session starts (platform: darwin, Python 3.11.4, pytest 8.4.2, pytest-sugar 1.1.1) ... ―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― test_api[POST /bookings] ――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― + Exception Group Traceback (most recent call last): | File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py", line 226, in test_api | def test_wrapper(*args: Any, **kwargs: Any) -> Any: | ^^^ | File "/tmp/venv/lib/python3.11/site-packages/hypothesis/core.py", line 1988, in wrapped_test | _raise_to_user(errors, state.settings, [], " in explicit examples") | File "/tmp/venv/lib/python3.11/site-packages/hypothesis/core.py", line 1582, in _raise_to_user | raise the_error_hypothesis_found | File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py", line 228, in test_wrapper | return test_function(*args, **kwargs) | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | File "/tmp/pytest-schemathesis/test_api.py", line 20, in test_api | case.call_and_validate(headers={"Authorization": "Bearer secret-token"}) | File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/case.py", line 476, in call_and_validate | self.validate_response( | File "/tmp/venv/lib/python3.11/site-packages/schemathesis/generation/case.py", line 450, in validate_response | raise FailureGroup(_failures, message) from None | schemathesis.core.failures.FailureGroup: Schemathesis found 2 distinct failures | | - Server error | | - Undocumented HTTP status code | | Received: 500 | Documented: 200, 400, 422 | | [500] Internal Server Error: | | `Internal Server Error` | | Reproduce with: | | curl -X POST -H 'Authorization: [Filtered]' -H 'Content-Type: application/json' -d '{"guest_name": "00", "nights": 1, "room_type": ""}' http://127.0.0.1:8000/bookings | | (2 sub-exceptions) +-+---------------- 1 ---------------- | schemathesis.core.failures.ServerError: Server error +---------------- 2 ---------------- | schemathesis.openapi.checks.UndefinedStatusCode: Undocumented HTTP status code | | Received: 500 | Documented: 200, 400, 422 +------------------------------------ ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[POST /bookings] ⨯ 33% ███▍ ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /bookings/{booking_id}] ✓ 67% ██████▋ ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /health] ✓ 100% ██████████ ===================================================================================================== short test summary info ===================================================================================================== FAILED test_api.py::test_api[POST /bookings] Results (12.78s): 2 passed 1 failed - ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py:225 test_api[POST /bookings] 

―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― test_api[POST /bookings] ――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― ... | - Server error | | - Undocumented HTTP status code | | Received: 500 | Documented: 200, 400, 422 | | [500] Internal Server Error: | | `Internal Server Error` | | Reproduce with: | | curl -X POST -H 'Authorization: [Filtered]' -H 'Content-Type: application/json' -d '{"guest_name": "00", "nights": 1, "room_type": ""}' http://127.0.0.1:8000/bookings ... 

$ curl -X POST -H 'Authorization: [Filtered]' -H 'Content-Type: application/json' -d '{"guest_name": "00", "nights": 1, "room_type": ""}' http://127.0.0.1:8000/bookings {"detail":"Internal Server Error"}% 

room_prices = {"standard": 99.99, "deluxe": 149.99, "suite": 299.99} price_per_night = room_prices[booking.room_type] 

$ diff --git a/pytest-schemathesis/app.py b/pytest-schemathesis/app.py --- a/pytest-schemathesis/app.py(revision 12231fe1bfc5ed8caf1f5d6430210d8c50cdc0aa) +++ b/pytest-schemathesis/app.py(date 1762762281090) @@ -1,4 +1,5 @@ import uuid +from enum import Enum from typing import Optional from fastapi import Depends, FastAPI, Header, HTTPException @@ -15,9 +16,15 @@ return True +class RoomType(str, Enum): + standard = "standard" + deluxe = "deluxe" + suite = "suite" + + class BookingRequest(BaseModel): guest_name: str = Field(min_length=2, max_length=100) - room_type: str + room_type: RoomType nights: int = Field(gt=0, le=365) 

$ pytest test_api.py -v Test session starts (platform: darwin, Python 3.11.4, pytest 8.4.2, pytest-sugar 1.1.1) ... collected 3 items ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[POST /bookings] ✓ 33% ███▍ ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /bookings/{booking_id}] ✓ 67% ██████▋ ../../venv/lib/python3.11/site-packages/schemathesis/generation/hypothesis/builder.py::test_api[GET /health] ✓ 100% ██████████ Results (20.08s): 3 passed