Move selected e2e tests to scorecard tests

Signed-off-by: Andreas Gerstmayr <agerstmayr@redhat.com>

.github/workflows/scorecard.yaml

@@ -32,7 +32,7 @@ jobs:
       - name: "prepare cluster"
         env:
           KUBE_VERSION: ${{ matrix.kube-version }}
-        run: make prepare-scorecard-tests KUBE_VERSION=$KUBE_VERSION VERSION=e2e
+        run: make prepare-e2e KUBE_VERSION=$KUBE_VERSION VERSION=e2e
 
       - name: "run scorecard tests"
         run: make scorecard-tests

Makefile

@@ -64,6 +64,23 @@ OPERATOR_SDK_VERSION ?= 1.27.0
 
 CERTMANAGER_VERSION ?= 1.10.0
 
+SCORECARD_KUTTL_TESTS := \
+	ingress \
+	instrumentation-dotnet \
+	instrumentation-dotnet-multicontainer \
+	instrumentation-java \
+	instrumentation-java-multicontainer \
+	instrumentation-nodejs \
+	instrumentation-nodejs-multicontainer \
+	instrumentation-python \
+	instrumentation-python-multicontainer \
+	smoke-pod-annotations \
+	smoke-restarting-deployment \
+	smoke-sidecar \
+	smoke-simplest \
+	smoke-statefulset \
+	statefulset-features \
+
 ifndef ignore-not-found
   ignore-not-found = false
 endif
@@ -189,13 +206,10 @@ e2e-log-operator:
 prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server install-openshift-routes load-image-all deploy
 	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
 
-.PHONY: prepare-scorecard-tests
-prepare-scorecard-tests: prepare-e2e
-	kubectl apply -f tests/scorecard/rbac.yaml
-
 .PHONY: scorecard-tests
-scorecard-tests: operator-sdk
-	$(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1)
+scorecard-tests: operator-sdk bundle-scorecard-kuttl-tests
+	kubectl apply -f tests/scorecard/rbac.yaml
+	$(OPERATOR_SDK) scorecard -w=15m bundle || (echo "scorecard test failed" && exit 1)
 
 
 # Build the container image, used only for local dev purposes
@@ -381,6 +395,11 @@ bundle: kustomize operator-sdk manifests set-image-controller
 	$(OPERATOR_SDK) bundle validate ./bundle
 	./hack/ignore-createdAt-bundle.sh
 
+# Copy selected e2e tests to the kuttl scorecard tests of the bundle
+.PHONY: bundle-scorecard-kuttl-tests
+bundle-scorecard-kuttl-tests:
+	for t in $(SCORECARD_KUTTL_TESTS); do cp -r tests/e2e/$$t bundle/tests/scorecard/kuttl; done
+
 # Build the bundle image, used only for local dev purposes
 .PHONY: bundle-build
 bundle-build:

bundle/tests/scorecard/config.yaml

@@ -46,10 +46,9 @@ stages:
     storage:
       spec:
         mountPath: {}
-  - image: quay.io/operator-framework/scorecard-test-kuttl:v2.0.0
+  - image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083
     labels:
-      suite: kuttlsuite
-      test: kuttltest1
+      suite: kuttl
     storage:
       spec:
         mountPath: {}

bundle/tests/scorecard/kuttl/kuttl-test.yaml

@@ -1,4 +1,7 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
-timeout: 150
+parallel: 1
 startControlPlane: false
+timeout: 120
+suppress:
+  - events

config/scorecard/bases/config.yaml

@@ -4,5 +4,5 @@ metadata:
   name: config
 serviceaccount: scorecard-admin
 stages:
-  - parallel: true
-    tests: []
+- parallel: true
+  tests: []

config/scorecard/kustomization.yaml

@@ -13,7 +13,7 @@ patchesJson6902:
     version: v1alpha3
     kind: Configuration
     name: config
-- path: patches/smoke.config.yaml
+- path: patches/kuttl.config.yaml
   target:
     group: scorecard.operatorframework.io
     version: v1alpha3

config/scorecard/patches/kuttl.config.yaml

@@ -0,0 +1,6 @@
+- op: add
+  path: /stages/0/tests/-
+  value:
+    image: docker.io/andreasgerstmayr/scorecard-test-kuttl:1681909083
+    labels:
+      suite: kuttl