From ce695e914d7d9f3446a170478dec107ddf8dbf88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Fri, 25 Oct 2024 14:43:17 +0200 Subject: [PATCH 1/2] feat: git sparse checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uses includePaths and excludePaths to configure non-cone sparse checkout for a git repo. Fixes #400 Signed-off-by: Zoltán Reegn --- pkg/vendir/config/config.go | 16 +++++++------ pkg/vendir/config/directory.go | 16 +++++++------ pkg/vendir/directory/directory.go | 3 ++- pkg/vendir/fetch/git/git.go | 38 +++++++++++++++++++++---------- pkg/vendir/fetch/git/sync.go | 19 ++++++++-------- 5 files changed, 56 insertions(+), 36 deletions(-) diff --git a/pkg/vendir/config/config.go b/pkg/vendir/config/config.go index f56b2536..add46a21 100644 --- a/pkg/vendir/config/config.go +++ b/pkg/vendir/config/config.go @@ -178,13 +178,15 @@ func (c Config) UseDirectory(path, dirPath string) error { matched = true newCon := DirectoryContents{ - Path: con.Path, - Directory: &DirectoryContentsDirectory{Path: dirPath}, - IncludePaths: con.IncludePaths, - ExcludePaths: con.ExcludePaths, - IgnorePaths: con.IgnorePaths, - LegalPaths: con.LegalPaths, - Lazy: con.Lazy, + Path: con.Path, + Directory: &DirectoryContentsDirectory{Path: dirPath}, + ContentPaths: ContentPaths{ + IncludePaths: con.IncludePaths, + ExcludePaths: con.ExcludePaths, + IgnorePaths: con.IgnorePaths, + }, + LegalPaths: con.LegalPaths, + Lazy: con.Lazy, } dir.Contents[j] = newCon c.Directories[i] = dir diff --git a/pkg/vendir/config/directory.go b/pkg/vendir/config/directory.go index 72c78696..2faf313a 100644 --- a/pkg/vendir/config/directory.go +++ b/pkg/vendir/config/directory.go @@ -46,17 +46,18 @@ type DirectoryContents struct { Manual *DirectoryContentsManual `json:"manual,omitempty"` Directory *DirectoryContentsDirectory `json:"directory,omitempty"` Inline *DirectoryContentsInline `json:"inline,omitempty"` - - IncludePaths []string `json:"includePaths,omitempty"` - ExcludePaths []string `json:"excludePaths,omitempty"` - IgnorePaths []string `json:"ignorePaths,omitempty"` + Permissions *os.FileMode `json:"permissions,omitempty"` + NewRootPath string `json:"newRootPath,omitempty"` + ContentPaths `json:",inline"` // By default LICENSE/LICENCE/NOTICE/COPYRIGHT files are kept LegalPaths *[]string `json:"legalPaths,omitempty"` +} - NewRootPath string `json:"newRootPath,omitempty"` - - Permissions *os.FileMode `json:"permissions,omitempty"` +type ContentPaths struct { + IncludePaths []string `json:"includePaths,omitempty"` + ExcludePaths []string `json:"excludePaths,omitempty"` + IgnorePaths []string `json:"ignorePaths,omitempty"` } type DirectoryContentsGit struct { @@ -73,6 +74,7 @@ type DirectoryContentsGit struct { SkipInitSubmodules bool `json:"skipInitSubmodules,omitempty"` Depth int `json:"depth,omitempty"` ForceHTTPBasicAuth bool `json:"forceHTTPBasicAuth,omitempty"` + SparseCheckout bool `json:"sparseCheckout,omitempty"` } type DirectoryContentsGitVerification struct { diff --git a/pkg/vendir/directory/directory.go b/pkg/vendir/directory/directory.go index 5ee173c5..2d94d103 100644 --- a/pkg/vendir/directory/directory.go +++ b/pkg/vendir/directory/directory.go @@ -108,7 +108,8 @@ func (d *Directory) Sync(syncOpts SyncOpts) (ctlconf.LockDirectory, error) { switch { case contents.Git != nil: - gitSync := ctlgit.NewSync(*contents.Git, NewInfoLog(d.ui), syncOpts.RefFetcher, syncOpts.Cache) + contentPaths := contents.ContentPaths + gitSync := ctlgit.NewSync(*contents.Git, contentPaths, NewInfoLog(d.ui), syncOpts.RefFetcher, syncOpts.Cache) d.ui.PrintLinef("Fetching: %s + %s (git from %s)", d.opts.Path, contents.Path, gitSync.Desc()) diff --git a/pkg/vendir/fetch/git/git.go b/pkg/vendir/fetch/git/git.go index 07883a7f..ccad4e68 100644 --- a/pkg/vendir/fetch/git/git.go +++ b/pkg/vendir/fetch/git/git.go @@ -21,23 +21,24 @@ import ( ) type Git struct { - opts ctlconf.DirectoryContentsGit - infoLog io.Writer - refFetcher ctlfetch.RefFetcher - cmdRunner CommandRunner + opts ctlconf.DirectoryContentsGit + contentPaths ctlconf.ContentPaths + infoLog io.Writer + refFetcher ctlfetch.RefFetcher + cmdRunner CommandRunner } -func NewGit(opts ctlconf.DirectoryContentsGit, - infoLog io.Writer, refFetcher ctlfetch.RefFetcher) *Git { - - return &Git{opts, infoLog, refFetcher, &runner{infoLog}} +func NewGit(opts ctlconf.DirectoryContentsGit, contentPaths ctlconf.ContentPaths, + infoLog io.Writer, refFetcher ctlfetch.RefFetcher, +) *Git { + return &Git{opts, contentPaths, infoLog, refFetcher, &runner{infoLog}} } // NewGitWithRunner creates a Git retriever with a provided runner -func NewGitWithRunner(opts ctlconf.DirectoryContentsGit, - infoLog io.Writer, refFetcher ctlfetch.RefFetcher, cmdRunner CommandRunner) *Git { - - return &Git{opts, infoLog, refFetcher, cmdRunner} +func NewGitWithRunner(opts ctlconf.DirectoryContentsGit, contentPaths ctlconf.ContentPaths, + infoLog io.Writer, refFetcher ctlfetch.RefFetcher, cmdRunner CommandRunner, +) *Git { + return &Git{opts, contentPaths, infoLog, refFetcher, cmdRunner} } //nolint:revive @@ -142,6 +143,19 @@ func (t *Git) fetch(dstPath string, tempArea ctlfetch.TempArea, bundle string) e {"remote", "add", "origin", gitURL}, } + if t.opts.SparseCheckout { + if len(t.contentPaths.IncludePaths) > 0 || len(t.contentPaths.ExcludePaths) > 0 { + sparseCheckoutArgs := []string{"sparse-checkout", "set", "--no-cone"} + for _, include := range t.contentPaths.IncludePaths { + sparseCheckoutArgs = append(sparseCheckoutArgs, include) + } + for _, exclude := range t.contentPaths.ExcludePaths { + sparseCheckoutArgs = append(sparseCheckoutArgs, fmt.Sprintf("!%s", exclude)) + } + argss = append(argss, sparseCheckoutArgs) + } + } + if authOpts.Username != nil && authOpts.Password != nil { if !strings.HasPrefix(gitURL, "https://") { return fmt.Errorf("Username/password authentication is only supported for https remotes") diff --git a/pkg/vendir/fetch/git/sync.go b/pkg/vendir/fetch/git/sync.go index 7a9f213b..9207046a 100644 --- a/pkg/vendir/fetch/git/sync.go +++ b/pkg/vendir/fetch/git/sync.go @@ -19,16 +19,17 @@ import ( const gitCacheType = "git-bundle" type Sync struct { - opts ctlconf.DirectoryContentsGit - log io.Writer - refFetcher ctlfetch.RefFetcher - cache ctlcache.Cache + opts ctlconf.DirectoryContentsGit + contentPaths ctlconf.ContentPaths + log io.Writer + refFetcher ctlfetch.RefFetcher + cache ctlcache.Cache } -func NewSync(opts ctlconf.DirectoryContentsGit, - log io.Writer, refFetcher ctlfetch.RefFetcher, cache ctlcache.Cache) Sync { - - return Sync{opts, log, refFetcher, cache} +func NewSync(opts ctlconf.DirectoryContentsGit, contentPaths ctlconf.ContentPaths, + log io.Writer, refFetcher ctlfetch.RefFetcher, cache ctlcache.Cache, +) Sync { + return Sync{opts, contentPaths, log, refFetcher, cache} } func (d Sync) Desc() string { @@ -54,7 +55,7 @@ func (d Sync) Sync(dstPath string, tempArea ctlfetch.TempArea) (ctlconf.LockDire cacheID := fmt.Sprintf("%x", sha256.Sum256([]byte(d.opts.URL))) - git := NewGit(d.opts, d.log, d.refFetcher) + git := NewGit(d.opts, d.contentPaths, d.log, d.refFetcher) var bundle string if cacheEntry, hasCache := d.cache.Has(gitCacheType, cacheID); hasCache { From a11f56de9ba5622e4a664c288feff239f8a2de93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Fri, 25 Oct 2024 14:48:07 +0200 Subject: [PATCH 2/2] fix lint errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zoltán Reegn --- pkg/vendir/fetch/git/git_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/vendir/fetch/git/git_test.go b/pkg/vendir/fetch/git/git_test.go index 3ccdd286..b7c0517b 100644 --- a/pkg/vendir/fetch/git/git_test.go +++ b/pkg/vendir/fetch/git/git_test.go @@ -31,7 +31,7 @@ func TestGit_Retrieve(t *testing.T) { Ref: "origin/main", SecretRef: &config.DirectoryContentsLocalRef{Name: "some-secret"}, ForceHTTPBasicAuth: true, - }, os.Stdout, secretFetcher, runner) + }, config.ContentPaths{}, os.Stdout, secretFetcher, runner) _, err := gitRetriever.Retrieve("", &tmpFolder{t}, "") require.NoError(t, err) isPresent := false @@ -61,7 +61,7 @@ func TestGit_Retrieve(t *testing.T) { Ref: "origin/main", SecretRef: &config.DirectoryContentsLocalRef{Name: "some-secret"}, ForceHTTPBasicAuth: true, - }, os.Stdout, secretFetcher, runner) + }, config.ContentPaths{}, os.Stdout, secretFetcher, runner) _, err := gitRetriever.Retrieve("", &tmpFolder{t}, "") require.ErrorContains(t, err, "Username/password authentication is only supported for https remotes") })