Fix semaphore for producer ducktape tests + clean up files that should've been removed (#2081)

* update * use warning for producer validate * remove unnecessary assert

Author: fangnx

.semaphore/semaphore.yml

@@ -287,7 +287,7 @@ blocks:
  - name: ARCH
  value: x64
  - name: BENCHMARK_BOUNDS_CONFIG
- value: tests/ducktape/benchmark_bounds.json
+ value: tests/ducktape/producer_benchmark_bounds.json
  - name: BENCHMARK_ENVIRONMENT
  value: ci
  prologue:
@@ -299,80 +299,80 @@ blocks:
  # Setup Python environment
  - sem-version python 3.9
  - python3 -m venv _venv && source _venv/bin/activate
- 
+
  # Install ducktape framework and additional dependencies
  - pip install ducktape psutil
- 
+
  # Install existing test requirements
  - pip install -r requirements/requirements-tests-install.txt
- 
+
  # Build and install confluent-kafka from source
  - lib_dir=dest/runtimes/$OS_NAME-$ARCH/native
  - tools/wheels/install-librdkafka.sh "${LIBRDKAFKA_VERSION#v}" dest
  - export CFLAGS="$CFLAGS -I${PWD}/dest/build/native/include"
  - export LDFLAGS="$LDFLAGS -L${PWD}/${lib_dir}"
  - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/$lib_dir"
  - python3 -m pip install -e .
- 
+
  # Store project root for reliable navigation
  - PROJECT_ROOT="${PWD}"
- 
+
  # Start Kafka cluster and Schema Registry using dedicated ducktape compose file (KRaft mode)
  - cd "${PROJECT_ROOT}/tests/docker"
  - docker-compose -f docker-compose.ducktape.yml up -d kafka schema-registry
- 
+
  # Debug: Check container status and logs
  - echo "=== Container Status ==="
  - docker-compose -f docker-compose.ducktape.yml ps
  - echo "=== Kafka Logs ==="
  - docker-compose -f docker-compose.ducktape.yml logs kafka | tail -50
- 
+
  # Wait for Kafka to be ready (using PLAINTEXT listener for external access)
  - |
  timeout 1800 bash -c '
  counter=0
- until docker-compose -f docker-compose.ducktape.yml exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; do 
+ until docker-compose -f docker-compose.ducktape.yml exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; do
  echo "Waiting for Kafka... (attempt $((counter+1)))"
- 
+
  # Show logs every 4th attempt (every 20 seconds)
  if [ $((counter % 4)) -eq 0 ] && [ $counter -gt 0 ]; then
  echo "=== Recent Kafka Logs ==="
  docker-compose -f docker-compose.ducktape.yml logs --tail=10 kafka
  echo "=== Container Status ==="
  docker-compose -f docker-compose.ducktape.yml ps kafka
  fi
- 
+
  counter=$((counter+1))
  sleep 5
  done
  '
  - echo "Kafka cluster is ready!"
- 
+
  # Wait for Schema Registry to be ready
  - echo "=== Waiting for Schema Registry ==="
  - |
  timeout 300 bash -c '
  counter=0
- until curl -f http://localhost:8081/subjects >/dev/null 2>&1; do 
+ until curl -f http://localhost:8081/subjects >/dev/null 2>&1; do
  echo "Waiting for Schema Registry... (attempt $((counter+1)))"
- 
+
  # Show logs every 3rd attempt (every 15 seconds)
  if [ $((counter % 3)) -eq 0 ] && [ $counter -gt 0 ]; then
  echo "=== Recent Schema Registry Logs ==="
  docker-compose -f docker-compose.ducktape.yml logs --tail=10 schema-registry
  echo "=== Schema Registry Container Status ==="
  docker-compose -f docker-compose.ducktape.yml ps schema-registry
  fi
- 
+
  counter=$((counter+1))
  sleep 5
  done
  '
  - echo "Schema Registry is ready!"
- 
+
  # Run standard ducktape tests with CI bounds
  - cd "${PROJECT_ROOT}" && PYTHONPATH="${PROJECT_ROOT}" python tests/ducktape/run_ducktape_test.py
- 
+
  # Cleanup
  - cd "${PROJECT_ROOT}/tests/docker" && docker-compose -f docker-compose.ducktape.yml down -v || true
  - name: "Packaging"