October Release Candidate - Object Storage

delivery-tooling/go.mod

@@ -2,10 +2,18 @@ module github.com/finos/common-cloud-controls
 
 go 1.22.5
 
-require gopkg.in/yaml.v3 v3.0.1
+require (
+	github.com/google/go-github/v53 v53.2.0
+	golang.org/x/oauth2 v0.18.0
+	gopkg.in/yaml.v3 v3.0.1
+)
 
 require (
+	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -18,9 +26,12 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )
 

delivery-tooling/go.sum

@@ -1,8 +1,24 @@
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
+github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
+github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
+github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI=
+github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -42,16 +58,52 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
+golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=

delivery-tooling/update-metadata.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/google/go-github/v53/github"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"golang.org/x/oauth2"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	// baseCmd represents the base command when called without any subcommands
+	updateMetadataCmd = &cobra.Command{
+		Use:   "update-metadata",
+		Short: "",
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			fmt.Print(divider)
+			fmt.Print(logo)
+			fmt.Println(divider)
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			fmt.Println(divider)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			checkArgs()
+
+			servicesDir := viper.GetString("services-dir")
+			buildTarget := viper.GetString("build-target")
+			err := updateMetadata(servicesDir, buildTarget)
+			if err != nil {
+				fmt.Println(err)
+			} else {
+				fmt.Printf("Metadata for %s has been updated successfully!\n", buildTarget)
+			}
+		},
+	}
+)
+
+func init() {
+	baseCmd.AddCommand(updateMetadataCmd)
+}
+
+func updateMetadata(servicesDir string, buildTarget string) (err error) {
+	// Replace with your GitHub personal access token
+	accessToken := os.Getenv("GITHUB_TOKEN")
+	repoOwner := "finos"
+	repoName := "common-cloud-controls"
+
+	// Specify the directory for which you want to get commit messages
+	directoryPath := "/services/storage/object"
+
+	// Create a new OAuth2 token for GitHub API access
+	ctx := context.Background()
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: accessToken},
+	)
+	tc := oauth2.NewClient(ctx, ts)
+
+	// Create a new GitHub client
+	client := github.NewClient(tc)
+
+	// Prepare the options to filter commits by the specified path (directory)
+	opts := &github.CommitsListOptions{
+		Path: directoryPath,
+	}
+
+	// Fetch the list of commits from the repository
+	commits, _, err := client.Repositories.ListCommits(ctx, repoOwner, repoName, opts)
+	if err != nil {
+		log.Fatalf("Error fetching commits: %v", err)
+	}
+
+	// Store unique contributors
+	var contributors []Contributors
+
+	// Collect the changelog information
+	changelog := []string{}
+
+	// Process commits to extract contributors and changelog details
+	for _, commit := range commits {
+		if commit.Commit != nil {
+			// Get the commit author's name and GitHub username
+			commitAuthorName := commit.Commit.Author.GetName()
+			commitAuthorLogin := "Unknown"
+			if commit.Author != nil && commit.Author.Login != nil {
+				commitAuthorLogin = *commit.Author.Login
+			}
+
+			// Add the contributor to the map (set-like behavior)
+			newContributor := Contributors{
+				Name:     commitAuthorName,
+				GithubId: commitAuthorLogin,
+				Company:  "REPLACE_ME",
+			}
+			contributors = append(contributors, newContributor)
+
+			// Collect changelog details
+			commitMessage := *commit.Commit.Message
+
+			// Split the commit message into lines to filter out lines like "Co-authored-by:"
+			lines := strings.Split(commitMessage, "\n")
+			filteredMessage := ""
+			for _, line := range lines {
+				// Filter out "Co-authored-by:" and other unwanted patterns
+				if !strings.HasPrefix(line, "Co-authored-by:") && !strings.HasPrefix(line, "Signed-off-by:") {
+					filteredMessage += line
+				}
+			}
+
+			// Format the changelog entry
+			changelog = append(changelog, filteredMessage)
+		}
+	}
+
+	// Read YAML
+	metadata := readMetadataYaml(servicesDir, buildTarget)
+
+	// Update metadata struct to include change log and contributors
+	metadata.ReleaseDetails[0].ChangeLog = changelog
+	metadata.ReleaseDetails[0].Contributors = removeDuplicates(contributors)
+
+	// Marshal the updated struct back to YAML
+	metadataData, err := yaml.Marshal(&metadata)
+	if err != nil {
+		log.Fatalf("Error marshaling YAML: %v", err)
+	}
+
+	err = os.WriteFile(fmt.Sprintf("%s/%s/metadata.yaml", servicesDir, buildTarget), metadataData, os.FileMode(0666))
+	if err != nil {
+		log.Fatalf("Error writing to the YAML file: %v", err)
+	}
+
+	fmt.Println("Contributors and Change Log has been updated.")
+	return
+}
+
+func readMetadataYaml(servicesDir string, buildTarget string) Metadata {
+	// Read the YAML file
+	yamlFile, err := os.ReadFile(fmt.Sprintf("%s/%s/metadata.yaml", servicesDir, buildTarget))
+	if err != nil {
+		log.Fatalf("Error reading YAML file: %v", err)
+	}
+
+	// Create a ReleaseDetails struct to hold the data
+	var metadata Metadata
+
+	// Unmarshal the YAML into the struct
+	err = yaml.Unmarshal(yamlFile, &metadata)
+	if err != nil {
+		log.Fatalf("Error unmarshaling YAML: %v", err)
+	}
+
+	return metadata
+}

delivery-tooling/utils.go

@@ -78,3 +78,17 @@ func addPageBreaksBeforeH2(content []byte) []byte {
 		return append(pageBreak, match...)
 	})
 }
+
+func removeDuplicates[T comparable](slice []T) []T {
+	uniqueMap := make(map[T]bool)
+	var result []T
+
+	for _, item := range slice {
+		if _, exists := uniqueMap[item]; !exists {
+			uniqueMap[item] = true
+			result = append(result, item)
+		}
+	}
+
+	return result
+}

services/storage/object/features.yaml

@@ -60,7 +60,7 @@ features:
     title: Object Level Access Control
     description: |
       Supports controlling access to specific objects within the object store.
-    - id: CCC.ObjStor.F11 # Querying
+  - id: CCC.ObjStor.F11 # Querying
     title: Querying
     description: |
       Supports performing simple select queries to retrieve only a subset of

services/storage/object/metadata.yaml

@@ -1,28 +1,63 @@
 title: Object Storage
 id: CCC.ObjStor
 description: |
-  Object storage is a data storage architecture that manages data as objects,
-  rather than as files or blocks. Each object contains the data itself,
-  metadata, and a unique identifier, making it ideal for storing large amounts
-  of unstructured data such as multimedia files, backups, and archives. It is
-  highly scalable and often used in cloud environments due to its flexibility
-  and accessibility.
+    Object storage is a data storage architecture that manages data as objects,
+    rather than as files or blocks. Each object contains the data itself,
+    metadata, and a unique identifier, making it ideal for storing large amounts
+    of unstructured data such as multimedia files, backups, and archives. It is
+    highly scalable and often used in cloud environments due to its flexibility
+    and accessibility.
 release_details:
-  - version: "2024.09"
-    assurance_level: None
-    threat_model_url: None
-    threat_model_author: None
-    red_team: None
-    red_team_exercise_url: None
-    release_manager:
-      name: Damien Burks
-      github_id: damienjburks
-      company: Citi
-      summary: Initial release
-    change_log:
-      - "Test"
-      - "Test"
-    contributors:
-      - name: Damien Burks
+    - version: "2024.10"
+      assurance_level: None
+      threat_model_url: None
+      threat_model_author: None
+      red_team: None
+      red_team_exercise_url: None
+      release_manager:
+        name: Damien Burks
         github_id: damienjburks
-        company: Citi
+        company: Citi
+        summary: Initial release
+      change_log:
+        - Add in fixes to object storage threats, controls, features (#436)
+        - Typofix on control IDs (#432)
+        - Object storage final polish (#419)
+        - Adding Contributors key to metadata schema (#409)
+        - Object Storage Controls Revision (#394)
+        - GenAI taxonomy (#393)
+        - Updates to the structure of the metadata yaml (#383)
+        - Schema updates (#377)
+        - Handling common entries for features, threats, & controls (#327)
+        - Convert Object Storage development files from MD to YAML (#325)
+        - Add in new object storage controls on encryption for impact and replication to untrusted destinations (#305)
+        - Convert existing taxonomies to the new yaml format (#319)
+        - Fix broken links (#317)
+        - Added CCC.OS.C6 Control (#298)Looks good
+        - Extend Object Storage Controls (#263)
+        - Creation of Global Markdown Formatting and Linting GitHub Actions (#223)
+        - Draft Proposal for Threat Catalog and Control Catalog Taxonomy  (#153)
+        - Signed URLs added
+        - addressed review comments by @rgriffiths-scottlogic
+        - address review comments by stevie from scott logic and further improvements
+        - updates
+        - initial version of the object store taxonomy
+      contributors:
+        - name: Michael Lysaght
+          github_id: mlysaght2017
+          company: Citi
+        - name: Eddie Knight
+          github_id: eddie-knight
+          company: Sonatype
+        - name: Damien Burks
+          github_id: damienjburks
+          company: Citi
+        - name: Sonali Mendis
+          github_id: smendis-scottlogic
+          company: Scott Logic
+        - name: Dave Ogle
+          github_id: dogle-scottlogic
+          company: Scott Logic
+        - name: Naseer Mohammad
+          github_id: nas-hub
+          company: Google