chore: adds unit tests for apigeecli (#387)

* chore: restructure folder structure

* chore: use internal package

* bug: fixes typo in method

* chore: adds unit tests for apis

* chore: move cmd as internal package

* chore: adds unit tests for apigeecli #386

* bug: fixes missing region param for orgs #388

* bug: fixes missing setting params #386

* bug: fixes cleanup #386

* bug: uses id for delete #386

* bug: use id instead of name #386

* chore: adds unit tests for fh and res #386

* feat: adds quota scope

* chore: adds unit tests

* chore: adds unit tests for apps keys and bundles

* fix: disables validation for specs with refs #389

* feat: adds support to import and exp env details #390

* chore: adds unit tests for debug mask #386

* chore: adds unit tests env and reports #386

* feat: adds ability to update entries  #391

* chore: adds unit tests for registry  #386

* chore: gofumpt files  #386

* chore: print req and resp headers  #386

* feat: add the update kvm cmd  #386

* bug: fixes path for kvm update  #386

* bug: fixes code to improve test cases  #386

.gitignore

@@ -30,3 +30,5 @@ remote-service.*
 !test/*.json
 !test/*.yaml
 !test/*.zip
+!cmd/apigeecli*
+!internal/apigeecli*

.goreleaser.yml

@@ -22,7 +22,7 @@ before:
 
 builds:
   - id: apigeecli
-    main: ./main.go
+    main: ./cmd/apigeecli/apigeecli.go
     binary: apigeecli
     goos:
       - linux

Dockerfile

@@ -20,7 +20,6 @@ ARG COMMIT
 ADD ./internal /go/src/apigeecli/internal
 ADD ./cmd /go/src/apigeecli/cmd
 
-COPY main.go /go/src/apigeecli/main.go
 COPY go.mod go.sum /go/src/apigeecli/
 
 WORKDIR /go/src/apigeecli
@@ -29,7 +28,7 @@ ENV GO111MODULE=on
 RUN go mod tidy
 RUN go mod download
 RUN date +%FT%H:%I:%M+%Z > /tmp/date
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=true -a -gcflags='all="-l"' -ldflags='-s -w -extldflags "-static" -X main.version='${TAG}' -X main.commit='${COMMIT}' -X main.date='$(cat /tmp/date) -o /go/bin/apigeecli /go/src/apigeecli/main.go
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=true -a -gcflags='all="-l"' -ldflags='-s -w -extldflags "-static" -X main.version='${TAG}' -X main.commit='${COMMIT}' -X main.date='$(cat /tmp/date) -o /go/bin/apigeecli /go/src/apigeecli/cmd/apigeecli/apigeecli.go
 
 FROM ghcr.io/jqlang/jq:latest as jq
 

main.go → cmd/apigeecli/apigeecli.go

@@ -18,7 +18,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/apigee/apigeecli/cmd"
+	"internal/cmd"
 )
 
 // https://goreleaser.com/cookbooks/using-main.version/?h=ldflags

docs/docs.go

@@ -19,7 +19,7 @@ import (
 	"os"
 	"path/filepath"
 
-	cmd "github.com/apigee/apigeecli/cmd"
+	cmd "internal/cmd"
 
 	"github.com/spf13/cobra/doc"
 )

go.mod

@@ -2,22 +2,26 @@ module github.com/apigee/apigeecli
 
 go 1.21
 
-require internal/apiclient v1.0.0
+require internal/apiclient v1.0.0 // indirect
 
 replace internal/apiclient => ./internal/apiclient
 
-require internal/client v1.0.0
+require internal/client v1.0.0 // indirect
 
 replace internal/client => ./internal/client
 
-require internal/bundlegen v1.0.0
+require internal/bundlegen v1.0.0 // indirect
 
 replace internal/bundlegen => ./internal/bundlegen
 
-require internal/clilog v1.0.0
+require internal/clilog v1.0.0 // indirect
 
 replace internal/clilog => ./internal/clilog
 
+require internal/cmd v1.0.0
+
+replace internal/cmd => ./internal/cmd
+
 require github.com/spf13/cobra v1.6.1
 
 require (

internal/apiclient/httpclient.go

@@ -56,7 +56,7 @@ var apigeeAnalyticsAPIRateLimit = rate.NewLimiter(rate.Every(time.Second), 1)
 // disable rate limit
 var noAPIRateLimit = rate.NewLimiter(rate.Inf, 1)
 
-// PostHttpZip method is used to send resources, proxy bundles, shared flows etc.
+// PostHttpZip method is used to archives to Apigee control plane.
 func PostHttpZip(auth bool, method string, url string, headers map[string]string, zipfile string) (err error) {
 	var req *http.Request
 
@@ -93,6 +93,9 @@ func PostHttpZip(auth bool, method string, url string, headers map[string]string
 		}
 	}
 
+	printHeaders(req.Header)
+
+	// Send the request
 	resp, err := ApigeeAPIClient.Do(req)
 	if err != nil {
 		clilog.Error.Println("error connecting: ", err)
@@ -202,7 +205,7 @@ func DownloadFile(url string, auth bool) (resp *http.Response, err error) {
 			return nil, err
 		}
 	}
-
+	printHeaders(req.Header)
 	resp, err = ApigeeAPIClient.Do(req)
 
 	if err != nil {
@@ -217,6 +220,7 @@ func DownloadFile(url string, auth bool) (resp *http.Response, err error) {
 		clilog.Error.Println("error in response: Response was null")
 		return nil, fmt.Errorf("error in response: Response was null")
 	}
+	printHeaders(resp.Header)
 	return resp, err
 }
 
@@ -317,13 +321,12 @@ func HttpClient(params ...string) (respBody []byte, err error) {
 		return nil, err
 	}
 
-	clilog.Debug.Println("Content-Type : ", contentType)
 	req.Header.Set("Content-Type", contentType)
 
 	if DryRun() {
 		return nil, nil
 	}
-
+	printHeaders(req.Header)
 	resp, err := ApigeeAPIClient.Do(req)
 	if err != nil {
 		clilog.Error.Println("error connecting: ", err)
@@ -439,7 +442,6 @@ func SetAuthHeader(req *http.Request) (*http.Request, error) {
 			return nil, err
 		}
 	}
-	clilog.Debug.Println("Setting token : ", GetApigeeToken())
 	req.Header.Add("Authorization", "Bearer "+GetApigeeToken())
 	return req, nil
 }
@@ -454,6 +456,8 @@ func handleResponse(resp *http.Response) (respBody []byte, err error) {
 		return nil, nil
 	}
 
+	printHeaders(resp.Header)
+
 	respBody, err = io.ReadAll(resp.Body)
 	if err != nil {
 		clilog.Error.Println("error in response: ", err)
@@ -497,3 +501,14 @@ func getErrorMessage(statusCode int) string {
 		return "unknown error"
 	}
 }
+
+func printHeaders(httpHeaders http.Header) {
+	headers := make(map[string]string)
+	for headerName, headerValues := range httpHeaders {
+		for _, value := range headerValues {
+			headers[headerName] = value
+		}
+	}
+	jsonHeaders, _ := json.Marshal(headers)
+	clilog.Debug.Println(string(jsonHeaders))
+}

internal/bundlegen/generateapiv2.go

@@ -15,6 +15,7 @@
 package bundlegen
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -28,6 +29,7 @@ import (
 	"internal/bundlegen/policies"
 	"internal/bundlegen/proxies"
 	targets "internal/bundlegen/targets"
+	"internal/clilog"
 
 	"github.com/pb33f/libopenapi"
 	validator "github.com/pb33f/libopenapi-validator"
@@ -44,11 +46,15 @@ func LoadDocument(specBasePath string, specBaseURL string, specName string, vali
 	config := datamodel.DocumentConfiguration{}
 	var specBytes []byte
 	var errs []error
+	references := []byte("$ref")
+
 	if specBasePath != "" {
 		config = datamodel.DocumentConfiguration{
-			AllowFileReferences:   true,
-			AllowRemoteReferences: true,
-			BasePath:              specBasePath,
+			AllowFileReferences:     true,
+			AllowRemoteReferences:   true,
+			BasePath:                specBasePath,
+			BundleInlineRefs:        true,
+			ExtractRefsSequentially: true,
 		}
 		specBytes, err = os.ReadFile(filepath.Join(specBasePath, specName))
 		if err != nil {
@@ -57,9 +63,11 @@ func LoadDocument(specBasePath string, specBaseURL string, specName string, vali
 	} else {
 		baseURL, _ := url.Parse(specBaseURL)
 		config = datamodel.DocumentConfiguration{
-			AllowFileReferences:   true,
-			AllowRemoteReferences: true,
-			BaseURL:               baseURL,
+			AllowFileReferences:     true,
+			AllowRemoteReferences:   true,
+			BaseURL:                 baseURL,
+			BundleInlineRefs:        true,
+			ExtractRefsSequentially: true,
 		}
 		specURL, _ := url.JoinPath(specBaseURL, specName)
 		resp, err := apiclient.DownloadFile(specURL, false)
@@ -76,6 +84,10 @@ func LoadDocument(specBasePath string, specBaseURL string, specName string, vali
 	if err != nil {
 		return nil, err
 	}
+	if index := bytes.Index(specBytes, references); index != -1 && validate {
+		clilog.Warning.Println("found references in the spec. disabling validation.")
+		validate = false
+	}
 	if validate {
 		docValidator, err := validator.NewValidator(document)
 		if err != nil {
@@ -118,7 +130,6 @@ func GenerateAPIProxyDefFromOASv2(name string,
 	oasTargetURLRef string,
 	targetURL string,
 ) (err error) {
-
 	if docModel == nil {
 		return fmt.Errorf("the Open API document not loaded")
 	}

internal/bundlegen/generateapiv2_test.go

@@ -0,0 +1,85 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bundlegen
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"regexp"
+	"testing"
+
+	"internal/client/clienttest"
+)
+
+var cliPath = os.Getenv("APIGEECLI_PATH")
+
+var specNames = []string{
+	"petstore-v3.1.yaml",
+	"md-ext1.yaml",
+	"windfarm3-quota.yaml",
+	"open_weather_api.yaml",
+}
+
+func TestSpecs(t *testing.T) {
+	for _, specName := range specNames {
+		fmt.Println("Testing " + specName + " ...")
+		testLoadDocument(specName, t)
+		testGenerateAPIProxyDefFromOASv2(specName, t)
+	}
+}
+
+func testLoadDocument(specName string, t *testing.T) {
+	if err := clienttest.TestSetup(clienttest.ENV_NOT_REQD,
+		clienttest.SITEID_NOT_REQD, clienttest.CLIPATH_REQD); err != nil {
+		t.Fatalf("%v", err)
+	}
+	if _, err := LoadDocument(path.Join(cliPath, "test"), "", specName, true); err != nil {
+		t.Fatalf("%v", err)
+	}
+}
+
+func testGenerateAPIProxyDefFromOASv2(specName string, t *testing.T) {
+	skipPolicy := false
+	name := "test"
+	addCORS := true
+	integrationEndpoint := false
+	basePath := ""
+	oasGoogleAcessTokenScopeLiteral := ""
+	targetURL := "http://api.example.com"
+	oasGoogleIDTokenAudLiteral := ""
+	oasGoogleIDTokenAudRef := ""
+	targetURLRef := ""
+	version := GetModelVersion()
+	if version != "" {
+		re := regexp.MustCompile(`3\.1\.[0-9]`)
+		if re.MatchString(version) {
+			skipPolicy = true
+		}
+	}
+	if err := GenerateAPIProxyDefFromOASv2(name,
+		basePath,
+		specName,
+		skipPolicy,
+		addCORS,
+		integrationEndpoint,
+		oasGoogleAcessTokenScopeLiteral,
+		oasGoogleIDTokenAudLiteral,
+		oasGoogleIDTokenAudRef,
+		targetURLRef,
+		targetURL); err != nil {
+		t.Fatalf("%v", err)
+	}
+}

internal/bundlegen/generateswag.go

@@ -34,7 +34,8 @@ import (
 	"internal/bundlegen/proxies"
 	"internal/bundlegen/targets"
 
-	"github.com/apigee/apigeecli/cmd/utils"
+	"internal/cmd/utils"
+
 	"github.com/getkin/kin-openapi/openapi2"
 	"github.com/ghodss/yaml"
 )

internal/client/apicategories/apicategories.go

@@ -62,9 +62,9 @@ func Create(siteid string, name string) (respBody []byte, err error) {
 }
 
 // Get
-func Get(siteid string, name string) (respBody []byte, err error) {
+func Get(siteid string, id string) (respBody []byte, err error) {
 	u, _ := url.Parse(apiclient.GetApigeeBaseURL())
-	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apicategories", name)
+	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apicategories", id)
 	respBody, err = apiclient.HttpClient(u.String())
 	return respBody, err
 }
@@ -78,9 +78,9 @@ func List(siteid string) (respBody []byte, err error) {
 }
 
 // Delete
-func Delete(siteid string, name string) (respBody []byte, err error) {
+func Delete(siteid string, id string) (respBody []byte, err error) {
 	u, _ := url.Parse(apiclient.GetApigeeBaseURL())
-	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apicategories", name)
+	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "sites", siteid, "apicategories", id)
 	respBody, err = apiclient.HttpClient(u.String(), "", "DELETE")
 	return respBody, err
 }