Organization in-progress checkpoint, still working on items from #3839

internal/controlplane/handlers_evalstatus.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/mindersec/minder/internal/db"
 	"github.com/mindersec/minder/internal/engine/engcontext"
+	"github.com/mindersec/minder/internal/engine/entities"
 	entmodels "github.com/mindersec/minder/internal/entities/models"
 	"github.com/mindersec/minder/internal/entities/properties"
 	propSvc "github.com/mindersec/minder/internal/entities/properties/service"
@@ -722,26 +723,7 @@ func buildEvalResultAlertFromLRERow(
 }
 
 func dbEntityToEntity(dbEnt db.Entities) minderv1.Entity {
-	switch dbEnt {
-	case db.EntitiesPullRequest:
-		return minderv1.Entity_ENTITY_PULL_REQUESTS
-	case db.EntitiesArtifact:
-		return minderv1.Entity_ENTITY_ARTIFACTS
-	case db.EntitiesRepository:
-		return minderv1.Entity_ENTITY_REPOSITORIES
-	case db.EntitiesBuildEnvironment:
-		return minderv1.Entity_ENTITY_BUILD_ENVIRONMENTS
-	case db.EntitiesRelease:
-		return minderv1.Entity_ENTITY_RELEASE
-	case db.EntitiesPipelineRun:
-		return minderv1.Entity_ENTITY_PIPELINE_RUN
-	case db.EntitiesTaskRun:
-		return minderv1.Entity_ENTITY_TASK_RUN
-	case db.EntitiesBuild:
-		return minderv1.Entity_ENTITY_BUILD
-	default:
-		return minderv1.Entity_ENTITY_UNSPECIFIED
-	}
+	return entities.EntityTypeFromDB(dbEnt)
 }
 
 func dbSeverityToSeverity(dbSev db.Severity) (*minderv1.Severity, error) {

internal/engine/entities/entities.go

@@ -52,6 +52,8 @@ func EntityTypeFromDB(entity db.Entities) minderv1.Entity {
 		return minderv1.Entity_ENTITY_TASK_RUN
 	case db.EntitiesBuild:
 		return minderv1.Entity_ENTITY_BUILD
+	case db.EntitiesOrganization:
+		return minderv1.Entity_ENTITY_ORGANIZATION
 	default:
 		return minderv1.Entity_ENTITY_UNSPECIFIED
 	}

internal/logger/telemetry_store_watermill.go

@@ -82,7 +82,8 @@ func newTelemetryStoreFromEntity(inf *entities.EntityInfoWrapper) (*TelemetrySto
 		ts.PullRequest = ent
 	case minderv1.Entity_ENTITY_BUILD_ENVIRONMENTS,
 		minderv1.Entity_ENTITY_RELEASE, minderv1.Entity_ENTITY_PIPELINE_RUN,
-		minderv1.Entity_ENTITY_TASK_RUN, minderv1.Entity_ENTITY_BUILD:
+		minderv1.Entity_ENTITY_TASK_RUN, minderv1.Entity_ENTITY_BUILD,
+		minderv1.Entity_ENTITY_ORGANIZATION:
 		// Noop, see https://github.com/mindersec/minder/issues/3838
 	case minderv1.Entity_ENTITY_UNSPECIFIED:
 		// Do nothing

internal/providers/github/entities.go

@@ -249,6 +249,8 @@ func (c *GitHub) PropertiesToProtoMessage(
 		return ghprop.PullRequestV1FromProperties(props)
 	case minderv1.Entity_ENTITY_RELEASE:
 		return ghprop.EntityInstanceV1FromReleaseProperties(props)
+	case minderv1.Entity_ENTITY_ORGANIZATION:
+		return ghprop.EntityInstanceV1FromOrganizationProperties(props)
 	}
 
 	return nil, fmt.Errorf("conversion of entity type %s is not handled by the github provider", entType)

internal/providers/github/properties/fetcher.go

@@ -54,6 +54,8 @@ func (_ ghEntityFetcher) EntityPropertyFetcher(entType minderv1.Entity) GhProper
 		return NewArtifactFetcher()
 	case minderv1.Entity_ENTITY_RELEASE:
 		return NewReleaseFetcher()
+	case minderv1.Entity_ENTITY_ORGANIZATION:
+		return NewOrganizationFetcher()
 	}
 
 	return nil

internal/providers/github/properties/organization.go

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package properties
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	go_github "github.com/google/go-github/v63/github"
+
+	"github.com/mindersec/minder/internal/entities/properties"
+	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	v1 "github.com/mindersec/minder/pkg/providers/v1"
+)
+
+// Organization Properties
+const (
+	// OrganizationPropertyOwner represents the github owner
+	OrganizationPropertyOwner = "github/owner"
+	// OrganizationPropertyWebsite represents the github website
+	OrganizationPropertyWebsite = "github/website"
+)
+
+// OrganizationFetcher is a property fetcher for organizations
+type OrganizationFetcher struct {
+	propertyFetcherBase
+}
+
+// NewOrganizationFetcher creates a new OrganizationFetcher
+func NewOrganizationFetcher() *OrganizationFetcher {
+	return &OrganizationFetcher{
+		propertyFetcherBase: propertyFetcherBase{
+			propertyOrigins: []propertyOrigin{
+				{
+					keys: []string{
+						// general entity
+						properties.PropertyName,
+						properties.PropertyUpstreamID,
+						// general organization
+						OrganizationPropertyOwner,
+					},
+					wrapper: getOrganizationWrapper,
+				},
+			},
+			operationalProperties: []string{},
+		},
+	}
+}
+
+// GetName returns the name of the release
+func (_ *OrganizationFetcher) GetName(props *properties.Properties) (string, error) {
+	owner := props.GetProperty(OrganizationPropertyOwner).GetString()
+
+	return owner, nil
+}
+
+func getOrganizationWrapper(
+	ctx context.Context, ghCli *go_github.Client, _ bool, getByProps *properties.Properties,
+) (map[string]any, error) {
+	owner, err := getByProps.GetProperty(OrganizationPropertyOwner).AsString()
+	if err != nil {
+		return nil, fmt.Errorf("owner not found or invalid: %w", err)
+	}
+
+	org, result, err := ghCli.Organizations.Get(ctx, owner)
+	if err != nil {
+		if result != nil && result.StatusCode == http.StatusNotFound {
+			return nil, v1.ErrEntityNotFound
+		}
+		return nil, fmt.Errorf("failed to fetch organization: %w", err)
+	}
+
+	props := map[string]any{
+		properties.PropertyUpstreamID: properties.NumericalValueToUpstreamID(org.GetID()),
+		properties.PropertyName:       owner,
+		OrganizationPropertyOwner:     owner,
+	}
+
+	if org.GetBlog() != "" {
+		props[OrganizationPropertyWebsite] = org.GetBlog()
+	}
+
+	return props, nil
+}
+
+// EntityInstanceV1FromOrganizationProperties creates a new EntityInstance from the given properties
+func EntityInstanceV1FromOrganizationProperties(props *properties.Properties) (*minderv1.EntityInstance, error) {
+	owner := props.GetProperty(OrganizationPropertyOwner).GetString()
+
+	name := owner
+
+	return &minderv1.EntityInstance{
+		Type:       minderv1.Entity_ENTITY_ORGANIZATION,
+		Name:       name,
+		Properties: props.ToProtoStruct(),
+	}, nil
+}

internal/providers/github/webhook/app.go

@@ -57,6 +57,7 @@ type installationRepositoriesEvent struct {
 	RepositorySelection *string       `json:"repository_selection,omitempty"`
 	Sender              *user         `json:"sender,omitempty"`
 	Installation        *installation `json:"installation,omitempty"`
+	Organization        *organization `json:"organization,omitempty"`
 }
 
 func (i *installationRepositoriesEvent) GetAction() string {
@@ -340,6 +341,7 @@ func processInstallationRepositoriesAppEvent(
 		// repository, which might be inefficient at scale.
 		res, err := repositoryRemoved(
 			repo,
+			event.Organization,
 		)
 		if errors.Is(err, errRepoNotFound) {
 			continue
@@ -356,8 +358,9 @@ func processInstallationRepositoriesAppEvent(
 
 func repositoryRemoved(
 	repo *repo,
+	org *organization,
 ) (*processingResult, error) {
-	return sendEvaluateRepoMessage(repo, constants.TopicQueueGetEntityAndDelete)
+	return sendEvaluateRepoMessage(repo, org, constants.TopicQueueGetEntityAndDelete)
 }
 
 func repositoryAdded(

internal/providers/github/webhook/handlers_githubwebhooks_test.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"github.com/ThreeDotsLabs/watermill/message"
-	"github.com/cenkalti/backoff/v4"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/go-github/v63/github"
 	"github.com/google/uuid"
@@ -157,8 +156,9 @@ func (s *UnitTestSuite) TestHandleWebHookPing() {
 	// the ping event has an empty body ({}), the value below is a SHA256 hmac of the empty body with the shared key "test"
 	req.Header.Add("X-Hub-Signature-256", "sha256=5f5863b9805ad4e66e954a260f9cab3f2e95718798dec0bb48a655195893d10e")
 	req.Header.Add("Content-Type", "application/json")
-	resp, err := httpDoWithRetry(ts.Client(), req)
+	resp, err := httpDoWithRetry(http.DefaultClient, req)
 	require.NoError(t, err, "failed to make request")
+	t.Cleanup(func() { _ = resp.Body.Close() })
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "unexpected status code")
 	assert.Len(t, queued, 0, "unexpected number of queued events")
 }
@@ -215,8 +215,9 @@ func (s *UnitTestSuite) TestHandleWebHookUnexistentRepository() {
 	req.Header.Add("X-GitHub-Event", "meta")
 	req.Header.Add("X-GitHub-Delivery", "12345")
 	req.Header.Add("Content-Type", "application/json")
-	resp, err := httpDoWithRetry(ts.Client(), req)
+	resp, err := httpDoWithRetry(http.DefaultClient, req)
 	require.NoError(t, err, "failed to make request")
+	t.Cleanup(func() { _ = resp.Body.Close() })
 	// We expect OK since we don't want to leak information about registered repositories
 	require.Equal(t, http.StatusOK, resp.StatusCode, "unexpected status code")
 	assert.Len(t, queued, 0)
@@ -376,8 +377,9 @@ func (s *UnitTestSuite) TestHandleWebHookUnexistentRepoPackage() {
 	req.Header.Add("X-GitHub-Event", "package")
 	req.Header.Add("X-GitHub-Delivery", "12345")
 	req.Header.Add("Content-Type", "application/json")
-	resp, err := httpDoWithRetry(ts.Client(), req)
+	resp, err := httpDoWithRetry(http.DefaultClient, req)
 	require.NoError(t, err, "failed to make request")
+	t.Cleanup(func() { _ = resp.Body.Close() })
 	// We expect OK since we don't want to leak information about registered repositories
 	require.Equal(t, http.StatusOK, resp.StatusCode, "unexpected status code")
 	assert.Len(t, queued, 0)
@@ -419,9 +421,10 @@ func (s *UnitTestSuite) TestNoopWebhookHandler() {
 	req.Header.Add("X-GitHub-Event", "marketplace_purchase")
 	req.Header.Add("X-GitHub-Delivery", "12345")
 	req.Header.Add("Content-Type", "application/json")
-	resp, err := httpDoWithRetry(ts.Client(), req)
+	resp, err := httpDoWithRetry(http.DefaultClient, req)
 
 	require.NoError(t, err, "failed to make request")
+	t.Cleanup(func() { _ = resp.Body.Close() })
 	assert.Equal(t, http.StatusOK, resp.StatusCode, "unexpected status code")
 }
 
@@ -2700,16 +2703,16 @@ func (s *UnitTestSuite) TestHandleGitHubWebHook() {
 
 			expectedMAC := sign(packageJson, "test")
 
-			client := &http.Client{}
 			req, err := http.NewRequest("POST", ts.URL, bytes.NewBuffer(packageJson))
 			require.NoError(t, err, "failed to create request")
 
 			req.Header.Add("X-GitHub-Event", tt.event)
 			req.Header.Add("X-GitHub-Delivery", "12345")
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("X-Hub-Signature-256", fmt.Sprintf("sha256=%s", expectedMAC))
-			resp, err := httpDoWithRetry(client, req)
+			resp, err := httpDoWithRetry(http.DefaultClient, req)
 			require.NoError(t, err, "failed to make request")
+			t.Cleanup(func() { _ = resp.Body.Close() })
 			// We expect OK since we don't want to leak information about registered repositories
 			require.Equal(t, tt.statusCode, resp.StatusCode, "unexpected status code")
 
@@ -3312,16 +3315,16 @@ func (s *UnitTestSuite) TestHandleGitHubAppWebHook() {
 			}
 
 			expectedMAC := sign(packageJson, "test")
-			client := &http.Client{}
 			req, err := http.NewRequest("POST", ts.URL, bytes.NewBuffer(packageJson))
 			require.NoError(t, err, "failed to create request")
 
 			req.Header.Add("X-GitHub-Event", tt.event)
 			req.Header.Add("X-GitHub-Delivery", "12345")
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("X-Hub-Signature-256", fmt.Sprintf("sha256=%s", expectedMAC))
-			resp, err := httpDoWithRetry(client, req)
+			resp, err := httpDoWithRetry(http.DefaultClient, req)
 			require.NoError(t, err, "failed to make request")
+			t.Cleanup(func() { _ = resp.Body.Close() })
 			// We expect OK since we don't want to leak information about registered repositories
 			require.Equal(t, tt.statusCode, resp.StatusCode, "unexpected status code")
 
@@ -3404,14 +3407,16 @@ func sign(payload []byte, key string) string {
 }
 
 func httpDoWithRetry(client *http.Client, req *http.Request) (*http.Response, error) {
-	var resp *http.Response
-	err := backoff.Retry(func() error {
-		var err error
-		resp, err = client.Do(req)
-		return err
-	}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), 3))
-
-	return resp, err
+	/*	var resp *http.Response
+		err := backoff.Retry(func() error {
+			var err error
+			resp, err = client.Do(req)
+			return err
+		}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Second), 3))
+
+		return resp, err
+	*/
+	return client.Do(req)
 }
 
 type garbage struct {