Signal classification support tools

cmd/registry-bigquery/common/common.go

@@ -0,0 +1,49 @@
+package common
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"cloud.google.com/go/bigquery"
+	"google.golang.org/api/googleapi"
+)
+
+// Used by subcommands, now provides a single common timestap for a command invocation.
+var Now = time.Now()
+
+// Get a BigQuery dataset by name and create it if it doesn't exist.
+func GetOrCreateDataset(ctx context.Context, client *bigquery.Client, name string) (*bigquery.Dataset, error) {
+	dataset := client.Dataset(name)
+	if err := dataset.Create(ctx, nil); err != nil {
+		switch v := err.(type) {
+		case *googleapi.Error:
+			if v.Code != http.StatusConflict { // already exists
+				return nil, err
+			}
+		default:
+			return nil, err
+		}
+	}
+	return dataset, nil
+}
+
+// Get a BigQuery table by name and create it if it doesn't exist.
+func GetOrCreateTable(ctx context.Context, dataset *bigquery.Dataset, name string, prototype interface{}) (*bigquery.Table, error) {
+	table := dataset.Table(name)
+	schema, err := bigquery.InferSchema(prototype)
+	if err != nil {
+		return nil, err
+	}
+	if err := table.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
+		switch v := err.(type) {
+		case *googleapi.Error:
+			if v.Code != http.StatusConflict { // already exists
+				return nil, err
+			}
+		default:
+			return nil, err
+		}
+	}
+	return table, nil
+}

cmd/registry-bigquery/index/index.go

@@ -15,13 +15,7 @@
 package index
 
 import (
-	"context"
-	"net/http"
-	"time"
-
-	"cloud.google.com/go/bigquery"
 	"github.com/spf13/cobra"
-	"google.golang.org/api/googleapi"
 )
 
 func Command() *cobra.Command {
@@ -31,46 +25,8 @@ func Command() *cobra.Command {
 	}
 
 	cmd.AddCommand(infoCommand())
+	cmd.AddCommand(linksCommand())
 	cmd.AddCommand(operationsCommand())
 	cmd.AddCommand(serversCommand())
 	return cmd
 }
-
-// Used by subcommands, now provides a single common timestap for a command invocation.
-var now = time.Now()
-
-// Get a BigQuery dataset by name and create it if it doesn't exist.
-func getOrCreateDataset(ctx context.Context, client *bigquery.Client, name string) (*bigquery.Dataset, error) {
-	dataset := client.Dataset(name)
-	if err := dataset.Create(ctx, nil); err != nil {
-		switch v := err.(type) {
-		case *googleapi.Error:
-			if v.Code != http.StatusConflict { // already exists
-				return nil, err
-			}
-		default:
-			return nil, err
-		}
-	}
-	return dataset, nil
-}
-
-// Get a BigQuery table by name and create it if it doesn't exist.
-func getOrCreateTable(ctx context.Context, dataset *bigquery.Dataset, name string, prototype interface{}) (*bigquery.Table, error) {
-	table := dataset.Table(name)
-	schema, err := bigquery.InferSchema(prototype)
-	if err != nil {
-		return nil, err
-	}
-	if err := table.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
-		switch v := err.(type) {
-		case *googleapi.Error:
-			if v.Code != http.StatusConflict { // already exists
-				return nil, err
-			}
-		default:
-			return nil, err
-		}
-	}
-	return table, nil
-}

cmd/registry-bigquery/index/info.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/bigquery"
+	"github.com/apigee/registry-experimental/cmd/registry-bigquery/common"
 	"github.com/apigee/registry-experimental/pkg/yamlquery"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/mime"
@@ -62,11 +63,11 @@ func infoCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			ds, err := getOrCreateDataset(ctx, client, dataset)
+			ds, err := common.GetOrCreateDataset(ctx, client, dataset)
 			if err != nil {
 				return err
 			}
-			table, err := getOrCreateTable(ctx, ds, "info", info{})
+			table, err := common.GetOrCreateTable(ctx, ds, "info", info{})
 			if err != nil {
 				return err
 			}
@@ -161,7 +162,7 @@ func (v *infoVisitor) getOpenAPIInfo(specName names.Spec, b []byte) error {
 				Api:         specName.ApiID,
 				Version:     specName.VersionID,
 				Spec:        specName.SpecID,
-				Timestamp:   now,
+				Timestamp:   common.Now,
 			})
 	}
 	return nil

cmd/registry-bigquery/index/links.go

@@ -0,0 +1,155 @@
+// Copyright 2023 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package index
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/apigee/registry-experimental/cmd/registry-bigquery/common"
+	"github.com/apigee/registry/cmd/registry/patch"
+	"github.com/apigee/registry/pkg/application/apihub"
+	"github.com/apigee/registry/pkg/connection"
+	"github.com/apigee/registry/pkg/mime"
+	"github.com/apigee/registry/pkg/names"
+	"github.com/apigee/registry/pkg/visitor"
+	"github.com/apigee/registry/rpc"
+	"github.com/spf13/cobra"
+)
+
+func linksCommand() *cobra.Command {
+	var filter string
+	var project string
+	var dataset string
+	var batchSize int
+	cmd := &cobra.Command{
+		Use:   "links PATTERN",
+		Short: "Build a BigQuery index of links between resources",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			c, err := connection.ActiveConfig()
+			if err != nil {
+				return err
+			}
+			pattern := c.FQName(args[0])
+			adminClient, err := connection.NewAdminClientWithSettings(ctx, c)
+			if err != nil {
+				return err
+			}
+			registryClient, err := connection.NewRegistryClientWithSettings(ctx, c)
+			if err != nil {
+				return err
+			}
+			if project == "" {
+				project = c.Project
+			}
+			client, err := bigquery.NewClient(ctx, project)
+			if err != nil {
+				return err
+			}
+			ds, err := common.GetOrCreateDataset(ctx, client, dataset)
+			if err != nil {
+				return err
+			}
+			table, err := common.GetOrCreateTable(ctx, ds, "links", link{})
+			if err != nil {
+				return err
+			}
+			v := &linksVisitor{
+				registryClient: registryClient,
+			}
+			err = visitor.Visit(ctx, v, visitor.VisitorOptions{
+				RegistryClient:  registryClient,
+				AdminClient:     adminClient,
+				Pattern:         pattern,
+				Filter:          filter,
+				ImplicitProject: &rpc.Project{Name: "projects/implicit"},
+			})
+			if err != nil {
+				return err
+			}
+			u := table.Inserter()
+			log.Printf("uploading %d links", len(v.links))
+			for start := 0; start < len(v.links); start += batchSize {
+				log.Printf("%d", start)
+				end := min(start+batchSize, len(v.links))
+				if err := u.Put(ctx, v.links[start:end]); err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&filter, "filter", "", "Filter selected resources")
+	cmd.Flags().StringVar(&project, "project", "", "Project to use for BigQuery upload (defaults to registry project)")
+	cmd.Flags().StringVar(&dataset, "dataset", "registry", "BigQuery dataset name")
+	cmd.Flags().IntVar(&batchSize, "batch-size", 10000, "Batch size to use when uploading records to BigQuery")
+	return cmd
+}
+
+type link struct {
+	Source    string
+	Target    string
+	Kind      string
+	Timestamp time.Time
+}
+
+type linksVisitor struct {
+	visitor.Unsupported
+	registryClient connection.RegistryClient
+	links          []*link
+}
+
+func (v *linksVisitor) ArtifactHandler() visitor.ArtifactHandler {
+	return func(ctx context.Context, message *rpc.Artifact) error {
+		artifactName, err := names.ParseArtifact(message.Name)
+		if err != nil {
+			return err
+		}
+		kind := mime.KindForMimeType(message.MimeType)
+		if kind != "ReferenceList" {
+			return nil // skip it
+		}
+		m := &apihub.ReferenceList{}
+		err = visitor.FetchArtifactContents(ctx, v.registryClient, message)
+		if err != nil {
+			return err
+		}
+		if err := patch.UnmarshalContents(message.GetContents(), message.GetMimeType(), m); err != nil {
+			return err
+		}
+		for _, l := range m.References {
+			if l.Resource != "" {
+				n, err := names.ParseApi(l.Resource)
+				if err != nil {
+					continue
+				}
+				fmt.Printf("%s -->%s (%s)\n", artifactName.ApiID(), n.ApiID, l.Category)
+				v.links = append(v.links,
+					&link{
+						Source:    artifactName.ApiID(),
+						Target:    n.ApiID,
+						Kind:      l.Category,
+						Timestamp: common.Now,
+					})
+			}
+		}
+		return nil
+	}
+}

cmd/registry-bigquery/index/operations.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/bigquery"
+	"github.com/apigee/registry-experimental/cmd/registry-bigquery/common"
 	"github.com/apigee/registry-experimental/pkg/yamlquery"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/mime"
@@ -63,11 +64,11 @@ func operationsCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			ds, err := getOrCreateDataset(ctx, client, dataset)
+			ds, err := common.GetOrCreateDataset(ctx, client, dataset)
 			if err != nil {
 				return err
 			}
-			table, err := getOrCreateTable(ctx, ds, "operations", operation{})
+			table, err := common.GetOrCreateTable(ctx, ds, "operations", operation{})
 			if err != nil {
 				return err
 			}
@@ -175,7 +176,7 @@ func (v *operationsVisitor) getOpenAPIOperations(specName names.Spec, b []byte)
 						Api:       specName.ApiID,
 						Version:   specName.VersionID,
 						Spec:      specName.SpecID,
-						Timestamp: now,
+						Timestamp: common.Now,
 					})
 
 			}

cmd/registry-bigquery/index/servers.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"cloud.google.com/go/bigquery"
+	"github.com/apigee/registry-experimental/cmd/registry-bigquery/common"
 	"github.com/apigee/registry-experimental/pkg/yamlquery"
 	"github.com/apigee/registry/pkg/connection"
 	"github.com/apigee/registry/pkg/mime"
@@ -62,11 +63,11 @@ func serversCommand() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			ds, err := getOrCreateDataset(ctx, client, dataset)
+			ds, err := common.GetOrCreateDataset(ctx, client, dataset)
 			if err != nil {
 				return err
 			}
-			table, err := getOrCreateTable(ctx, ds, "servers", server{})
+			table, err := common.GetOrCreateTable(ctx, ds, "servers", server{})
 			if err != nil {
 				return err
 			}
@@ -162,7 +163,7 @@ func (v *serversVisitor) getOpenAPIServers(specName names.Spec, b []byte) error
 						Api:       specName.ApiID,
 						Version:   specName.VersionID,
 						Spec:      specName.SpecID,
-						Timestamp: now,
+						Timestamp: common.Now,
 					})
 			}
 		}
@@ -180,7 +181,7 @@ func (v *serversVisitor) getOpenAPIServers(specName names.Spec, b []byte) error
 				Api:       specName.ApiID,
 				Version:   specName.VersionID,
 				Spec:      specName.SpecID,
-				Timestamp: now,
+				Timestamp: common.Now,
 			})
 	}
 	return nil

cmd/registry-bigquery/main.go

@@ -20,6 +20,7 @@ import (
 	"os"
 
 	"github.com/apigee/registry-experimental/cmd/registry-bigquery/index"
+	"github.com/apigee/registry-experimental/cmd/registry-bigquery/match"
 	"github.com/apigee/registry/pkg/log"
 	"github.com/google/uuid"
 	"github.com/spf13/cobra"
@@ -33,6 +34,7 @@ func main() {
 
 	cmd := &cobra.Command{}
 	cmd.AddCommand(index.Command())
+	cmd.AddCommand(match.Command())
 	if err := cmd.ExecuteContext(ctx); err != nil {
 		os.Exit(1)
 	}