Implement Minder TestKit (#4762)

* Implement Minder TestKit

TestKit is an implementation of several interfaces that allows for
easier testing. The intent is to be able to test rule types locally via
a programmatic framework, as opposed to relying on an integration test.

Signed-off-by: Juan Antonio Osorio <ozz@stacklok.com>

* Add REST ingester support

Signed-off-by: Juan Antonio Osorio <ozz@stacklok.com>

* Address feedback

Signed-off-by: Juan Antonio Osorio <ozz@stacklok.com>

* Change signature of rule parameters validation

It now takes a map, since that's easier to use.

Signed-off-by: Juan Antonio Osorio <ozz@stacklok.com>

---------

Signed-off-by: Juan Antonio Osorio <ozz@stacklok.com>

cmd/dev/app/rule_type/rttst.go

@@ -24,7 +24,6 @@ import (
 	"github.com/mindersec/minder/internal/engine/errors"
 	"github.com/mindersec/minder/internal/engine/eval/rego"
 	engif "github.com/mindersec/minder/internal/engine/interfaces"
-	"github.com/mindersec/minder/internal/engine/rtengine"
 	"github.com/mindersec/minder/internal/engine/selectors"
 	entModels "github.com/mindersec/minder/internal/entities/models"
 	entProps "github.com/mindersec/minder/internal/entities/properties"
@@ -41,6 +40,7 @@ import (
 	"github.com/mindersec/minder/internal/providers/telemetry"
 	"github.com/mindersec/minder/internal/util/jsonyaml"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/rtengine"
 	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -149,7 +149,7 @@ func testCmdRun(cmd *cobra.Command, _ []string) error {
 	off := "off"
 	profile.Alert = &off
 
-	rules, err := rtengine.GetRulesFromProfileOfType(profile, ruletype)
+	rules, err := profiles.GetRulesFromProfileOfType(profile, ruletype)
 	if err != nil {
 		return fmt.Errorf("error getting relevant fragment: %w", err)
 	}
@@ -230,7 +230,7 @@ func runEvaluationForRules(
 		}
 		cmd.Printf("Profile valid according to the JSON schema!\n")
 
-		if err := val.ValidateParamsAgainstSchema(frag.GetParams()); err != nil {
+		if err := val.ValidateParamsAgainstSchema(frag.GetParams().AsMap()); err != nil {
 			return fmt.Errorf("error validating params against schema: %w", err)
 		}
 
@@ -300,7 +300,7 @@ func selectAndEval(
 
 	var evalErr error
 	if selected {
-		evalErr = eng.Eval(ctx, inf, evalStatus)
+		evalErr = eng.Eval(ctx, inf.Entity, evalStatus.GetRule().Def, evalStatus.GetRule().Params, evalStatus)
 	} else {
 		evalErr = errors.NewErrEvaluationSkipped("entity not selected by selector %s", matchedSelector)
 	}

internal/engine/actions/remediate/pull_request/pull_request.go

@@ -29,6 +29,7 @@ import (
 	"github.com/mindersec/minder/internal/profiles/models"
 	"github.com/mindersec/minder/internal/util"
 	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	engifv1 "github.com/mindersec/minder/pkg/engine/v1/interfaces"
 	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -74,7 +75,7 @@ type Remediator struct {
 }
 
 type paramsPR struct {
-	ingested   *interfaces.Result
+	ingested   *engifv1.Result
 	repo       *pb.Repository
 	title      string
 	modifier   fsModifier

internal/engine/actions/remediate/pull_request/pull_request_test.go

@@ -37,6 +37,7 @@ import (
 	"github.com/mindersec/minder/internal/providers/ratecache"
 	"github.com/mindersec/minder/internal/providers/telemetry"
 	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	interfaces2 "github.com/mindersec/minder/pkg/engine/v1/interfaces"
 	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -613,7 +614,7 @@ func TestPullRequestRemediate(t *testing.T) {
 			require.NoError(t, err, "unexpected error creating test worktree")
 
 			evalParams.SetIngestResult(
-				&interfaces.Result{
+				&interfaces2.Result{
 					Fs:     testWt.Filesystem,
 					Storer: testrepo.Storer,
 				})

internal/engine/eval/eval.go

@@ -15,9 +15,9 @@ import (
 	"github.com/mindersec/minder/internal/engine/eval/rego"
 	"github.com/mindersec/minder/internal/engine/eval/trusty"
 	"github.com/mindersec/minder/internal/engine/eval/vulncheck"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	eoptions "github.com/mindersec/minder/internal/engine/options"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 	provinfv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -27,7 +27,7 @@ func NewRuleEvaluator(
 	ruletype *minderv1.RuleType,
 	provider provinfv1.Provider,
 	opts ...eoptions.Option,
-) (engif.Evaluator, error) {
+) (interfaces.Evaluator, error) {
 	e := ruletype.Def.GetEval()
 	if e == nil {
 		return nil, fmt.Errorf("rule type missing eval configuration")

internal/engine/eval/homoglyphs/application/homoglyphs_service.go

@@ -13,10 +13,10 @@ import (
 
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/communication"
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/domain"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	eoptions "github.com/mindersec/minder/internal/engine/options"
 	pbinternal "github.com/mindersec/minder/internal/proto"
 	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -34,7 +34,7 @@ func NewHomoglyphsEvaluator(
 	reh *pb.RuleType_Definition_Eval_Homoglyphs,
 	ghClient provifv1.GitHub,
 	opts ...eoptions.Option,
-) (engif.Evaluator, error) {
+) (interfaces.Evaluator, error) {
 	if ghClient == nil {
 		return nil, fmt.Errorf("provider builder is nil")
 	}
@@ -59,7 +59,7 @@ func NewHomoglyphsEvaluator(
 func evaluateHomoglyphs(
 	ctx context.Context,
 	processor domain.HomoglyphProcessor,
-	res *engif.Result,
+	res *interfaces.Result,
 	reviewHandler *communication.GhReviewPrHandler,
 ) ([]*domain.Violation, error) {
 	// create an empty list of violations

internal/engine/eval/homoglyphs/application/invisible_characters_eval.go

@@ -12,8 +12,8 @@ import (
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/communication"
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/domain"
 	"github.com/mindersec/minder/internal/engine/eval/templates"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	eoptions "github.com/mindersec/minder/internal/engine/options"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -48,7 +48,7 @@ func (ice *InvisibleCharactersEvaluator) Eval(
 	ctx context.Context,
 	_ map[string]any,
 	_ protoreflect.ProtoMessage,
-	res *engif.Result,
+	res *interfaces.Result,
 ) error {
 	violations, err := evaluateHomoglyphs(ctx, ice.processor, res, ice.reviewHandler)
 	if err != nil {

internal/engine/eval/homoglyphs/application/mixed_scripts_eval.go

@@ -13,8 +13,8 @@ import (
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/communication"
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/domain"
 	"github.com/mindersec/minder/internal/engine/eval/templates"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	eoptions "github.com/mindersec/minder/internal/engine/options"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 	provifv1 "github.com/mindersec/minder/pkg/providers/v1"
 )
 
@@ -54,7 +54,7 @@ func (mse *MixedScriptsEvaluator) Eval(
 	ctx context.Context,
 	_ map[string]any,
 	_ protoreflect.ProtoMessage,
-	res *engif.Result,
+	res *interfaces.Result,
 ) error {
 	violations, err := evaluateHomoglyphs(ctx, mse.processor, res, mse.reviewHandler)
 	if err != nil {

internal/engine/eval/jq/fuzz_test.go

@@ -8,8 +8,8 @@ import (
 	"testing"
 
 	"github.com/mindersec/minder/internal/engine/eval/jq"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 )
 
 func FuzzJqEval(f *testing.F) {
@@ -38,6 +38,6 @@ func FuzzJqEval(f *testing.F) {
 		}
 
 		//nolint:gosec // Do not validate the return values so ignore them
-		jqe.Eval(context.Background(), pol, nil, &engif.Result{Object: obj})
+		jqe.Eval(context.Background(), pol, nil, &interfaces.Result{Object: obj})
 	})
 }

internal/engine/eval/jq/jq.go

@@ -15,10 +15,10 @@ import (
 
 	evalerrors "github.com/mindersec/minder/internal/engine/errors"
 	"github.com/mindersec/minder/internal/engine/eval/templates"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	eoptions "github.com/mindersec/minder/internal/engine/options"
 	"github.com/mindersec/minder/internal/util"
 	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 )
 
 // Evaluator is an Evaluator that uses the jq library to evaluate rules
@@ -70,7 +70,7 @@ func NewJQEvaluator(
 }
 
 // Eval calls the jq library to evaluate the rule
-func (jqe *Evaluator) Eval(ctx context.Context, pol map[string]any, _ protoreflect.ProtoMessage, res *engif.Result) error {
+func (jqe *Evaluator) Eval(ctx context.Context, pol map[string]any, _ protoreflect.ProtoMessage, res *interfaces.Result) error {
 	if res.Object == nil {
 		return fmt.Errorf("missing object")
 	}

internal/engine/eval/jq/jq_test.go

@@ -14,8 +14,8 @@ import (
 	evalerrors "github.com/mindersec/minder/internal/engine/errors"
 	"github.com/mindersec/minder/internal/engine/eval/jq"
 	"github.com/mindersec/minder/internal/engine/eval/templates"
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	pb "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 )
 
 func TestNewJQEvaluatorValid(t *testing.T) {
@@ -366,7 +366,7 @@ func TestValidJQEvals(t *testing.T) {
 			assert.NoError(t, err, "Got unexpected error")
 			assert.NotNil(t, jqe, "Got unexpected nil")
 
-			err = jqe.Eval(context.Background(), tt.args.pol, nil, &engif.Result{Object: tt.args.obj})
+			err = jqe.Eval(context.Background(), tt.args.pol, nil, &interfaces.Result{Object: tt.args.obj})
 			assert.NoError(t, err, "Got unexpected error")
 		})
 	}
@@ -576,7 +576,7 @@ func TestValidJQEvalsFailed(t *testing.T) {
 			assert.NoError(t, err, "Got unexpected error")
 			assert.NotNil(t, jqe, "Got unexpected nil")
 
-			err = jqe.Eval(context.Background(), tt.args.pol, nil, &engif.Result{Object: tt.args.obj})
+			err = jqe.Eval(context.Background(), tt.args.pol, nil, &interfaces.Result{Object: tt.args.obj})
 			assert.ErrorIs(t, err, evalerrors.ErrEvaluationFailed, "Got unexpected error")
 		})
 	}
@@ -648,7 +648,7 @@ func TestInvalidJQEvals(t *testing.T) {
 			assert.NoError(t, err, "Got unexpected error")
 			assert.NotNil(t, jqe, "Got unexpected nil")
 
-			err = jqe.Eval(context.Background(), tt.args.pol, nil, &engif.Result{Object: tt.args.obj})
+			err = jqe.Eval(context.Background(), tt.args.pol, nil, &interfaces.Result{Object: tt.args.obj})
 			assert.Error(t, err, "Got unexpected error")
 			assert.NotErrorIs(t, err, evalerrors.ErrEvaluationFailed, "Got unexpected error")
 		})

internal/engine/eval/rego/eval.go

@@ -13,9 +13,9 @@ import (
 	"github.com/open-policy-agent/opa/topdown/print"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	eoptions "github.com/mindersec/minder/internal/engine/options"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 )
 
 const (
@@ -105,7 +105,9 @@ func (e *Evaluator) newRegoFromOptions(opts ...func(*rego.Rego)) *rego.Rego {
 }
 
 // Eval implements the Evaluator interface.
-func (e *Evaluator) Eval(ctx context.Context, pol map[string]any, entity protoreflect.ProtoMessage, res *engif.Result) error {
+func (e *Evaluator) Eval(
+	ctx context.Context, pol map[string]any, entity protoreflect.ProtoMessage, res *interfaces.Result,
+) error {
 	// The rego engine is actually able to handle nil
 	// objects quite gracefully, so we don't need to check
 	// this explicitly.

internal/engine/eval/rego/fuzz_test.go

@@ -7,8 +7,8 @@ import (
 	"context"
 	"testing"
 
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 )
 
 // FuzzRegoEval tests for unexpected behavior in e.Eval().
@@ -31,7 +31,7 @@ func FuzzRegoEval(f *testing.F) {
 		// The fuzzer tests for unexpected behavior, so it is not
 		// important what e.Eval() returns.
 		//nolint:gosec // Ignore the return values from e.Eval()
-		e.Eval(context.Background(), emptyPol, nil, &engif.Result{
+		e.Eval(context.Background(), emptyPol, nil, &interfaces.Result{
 			Object: map[string]any{
 				"data": data,
 			},

internal/engine/eval/rego/lib.go

@@ -20,11 +20,11 @@ import (
 	"github.com/stacklok/frizbee/pkg/replacer"
 	"github.com/stacklok/frizbee/pkg/utils/config"
 
-	engif "github.com/mindersec/minder/internal/engine/interfaces"
+	"github.com/mindersec/minder/pkg/engine/v1/interfaces"
 )
 
 // MinderRegoLib contains the minder-specific functions for rego
-var MinderRegoLib = []func(res *engif.Result) func(*rego.Rego){
+var MinderRegoLib = []func(res *interfaces.Result) func(*rego.Rego){
 	FileExists,
 	FileLs,
 	FileLsGlob,
@@ -34,7 +34,7 @@ var MinderRegoLib = []func(res *engif.Result) func(*rego.Rego){
 	ListGithubActions,
 }
 
-func instantiateRegoLib(res *engif.Result) []func(*rego.Rego) {
+func instantiateRegoLib(res *interfaces.Result) []func(*rego.Rego) {
 	var lib []func(*rego.Rego)
 	for _, f := range MinderRegoLib {
 		lib = append(lib, f(res))
@@ -46,7 +46,7 @@ func instantiateRegoLib(res *engif.Result) []func(*rego.Rego) {
 // in the filesystem being evaluated (which comes from the ingester).
 // It takes one argument, the path to the file to check.
 // It's exposed as `file.exists`.
-func FileExists(res *engif.Result) func(*rego.Rego) {
+func FileExists(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "file.exists",
@@ -81,7 +81,7 @@ func FileExists(res *engif.Result) func(*rego.Rego) {
 // FileRead is a rego function that reads a file from the filesystem
 // being evaluated (which comes from the ingester). It takes one argument,
 // the path to the file to read. It's exposed as `file.read`.
-func FileRead(res *engif.Result) func(*rego.Rego) {
+func FileRead(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "file.read",
@@ -126,7 +126,7 @@ func FileRead(res *engif.Result) func(*rego.Rego) {
 // If the file is a directory, it returns the files in the directory.
 // If the file is a symlink, it follows the symlink and returns the files
 // in the target.
-func FileLs(res *engif.Result) func(*rego.Rego) {
+func FileLs(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "file.ls",
@@ -196,7 +196,7 @@ func FileLs(res *engif.Result) func(*rego.Rego) {
 // in the filesystem being evaluated (which comes from the ingester).
 // It takes one argument, the path to the pattern to match. It's exposed
 // as `file.ls_glob`.
-func FileLsGlob(res *engif.Result) func(*rego.Rego) {
+func FileLsGlob(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "file.ls_glob",
@@ -235,7 +235,7 @@ func FileLsGlob(res *engif.Result) func(*rego.Rego) {
 // in the filesystem being evaluated (which comes from the ingester).
 // It takes one argument, the path to the directory to walk. It's exposed
 // as `file.walk`.
-func FileWalk(res *engif.Result) func(*rego.Rego) {
+func FileWalk(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "file.walk",
@@ -329,7 +329,7 @@ func fileLsHandleDir(path string, bfs billy.Filesystem) (*ast.Term, error) {
 // as `github_workflow.ls_actions`.
 // The function returns a set of strings, each string being the name of an action.
 // The frizbee library guarantees that the actions are unique.
-func ListGithubActions(res *engif.Result) func(*rego.Rego) {
+func ListGithubActions(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "github_workflow.ls_actions",
@@ -368,7 +368,7 @@ func ListGithubActions(res *engif.Result) func(*rego.Rego) {
 // in the filesystem being evaluated (which comes from the ingester).
 // It takes one argument, the path to the file to check. It's exposed
 // as `file.http_type`.
-func FileHTTPType(res *engif.Result) func(*rego.Rego) {
+func FileHTTPType(res *interfaces.Result) func(*rego.Rego) {
 	return rego.Function1(
 		&rego.Function{
 			Name: "file.http_type",