From 5001979d652e9f990c455ea694abed1e5c82f3ee Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 06:49:40 -0800 Subject: [PATCH 1/7] /go/libraries/doltcore/{doltdb,env}: add paginated tag visitor --- go/libraries/doltcore/doltdb/doltdb.go | 23 ++++ go/libraries/doltcore/env/actions/tag.go | 110 +++++++++++++++ go/libraries/doltcore/env/actions/tag_test.go | 129 ++++++++++++++++++ 3 files changed, 262 insertions(+) create mode 100644 go/libraries/doltcore/env/actions/tag_test.go diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 133d74ea7d..4a137b3dbe 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -594,6 +594,29 @@ func (ddb *DoltDB) ResolveTag(ctx context.Context, tagRef ref.TagRef) (*Tag, err return NewTag(ctx, tagRef.GetPath(), ds, ddb.vrw, ddb.ns) } +// ResolveTagMeta takes a TagRef and returns the corresponding TagMeta object. +func (ddb *DoltDB) ResolveTagMeta(ctx context.Context, tagRef ref.TagRef) (*datas.TagMeta, error) { + ds, err := ddb.db.GetDataset(ctx, tagRef.String()) + if err != nil { + return nil, ErrTagNotFound + } + + if !ds.HasHead() { + return nil, ErrTagNotFound + } + + if !ds.IsTag() { + return nil, fmt.Errorf("tagRef head is not a tag") + } + + meta, _, err := ds.HeadTag() + if err != nil { + return nil, err + } + + return meta, nil +} + // ResolveWorkingSet takes a WorkingSetRef and returns the corresponding WorkingSet object. func (ddb *DoltDB) ResolveWorkingSet(ctx context.Context, workingSetRef ref.WorkingSetRef) (*WorkingSet, error) { ds, err := ddb.db.GetDataset(ctx, workingSetRef.String()) diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index 3dc7b40ab2..9c96223015 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -138,3 +138,113 @@ func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *dolt return nil } + +type TagRefWithMeta struct { + TagRef ref.TagRef + Meta *datas.TagMeta +} + +type RefPageToken struct { + TagName string +} + +const DefaultPageSize = 100 + +// IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb(). +// Returns a next page token if there are more results available. +func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, pageToken *RefPageToken, cb func(tag *doltdb.Tag) (stop bool, err error)) (*RefPageToken, error) { + tagRefs, err := ddb.GetTags(ctx) + if err != nil { + return nil, err + } + + // for each tag, get the meta + tagMetas := make([]*TagRefWithMeta, 0, len(tagRefs)) + for _, r := range tagRefs { + tr, ok := r.(ref.TagRef) + if !ok { + return nil, fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") + } + meta, err := ddb.ResolveTagMeta(ctx, tr) + if err != nil { + return nil, err + } + tagMetas = append(tagMetas, &TagRefWithMeta{TagRef: tr, Meta: meta}) + } + + // sort by meta timestamp + sort.Slice(tagMetas, func(i, j int) bool { + return tagMetas[i].Meta.Timestamp > tagMetas[j].Meta.Timestamp + }) + + // find starting index based on page token + startIdx := 0 + if pageToken != nil && pageToken.TagName != "" { + for i, tm := range tagMetas { + if tm.TagRef.GetPath() == pageToken.TagName { + startIdx = i + 1 // start after the token + break + } + } + } + + // get page of results + endIdx := startIdx + DefaultPageSize + if endIdx > len(tagMetas) { + endIdx = len(tagMetas) + } + + pageTagMetas := tagMetas[startIdx:endIdx] + + // resolve tags for this page + for _, tm := range pageTagMetas { + tag, err := ddb.ResolveTag(ctx, tm.TagRef) + if err != nil { + return nil, err + } + + stop, err := cb(tag) + if err != nil { + return nil, err + } + if stop { + break + } + } + + // return next page token if there are more results + var nextPageToken *RefPageToken + if endIdx < len(tagMetas) { + lastTag := pageTagMetas[len(pageTagMetas)-1] + nextPageToken = &RefPageToken{ + TagName: lastTag.TagRef.GetPath(), + } + } + + return nextPageToken, nil +} + +// VisitResolvedTag iterates over tags in ddb until the given tag name is found, then calls cb() with the resolved tag. +func VisitResolvedTag(ctx context.Context, ddb *doltdb.DoltDB, tagName string, cb func(tag *doltdb.Tag) error) error { + tagRefs, err := ddb.GetTags(ctx) + if err != nil { + return err + } + + for _, r := range tagRefs { + tr, ok := r.(ref.TagRef) + if !ok { + return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") + } + + if tr.GetPath() == tagName { + tag, err := ddb.ResolveTag(ctx, tr) + if err != nil { + return err + } + return cb(tag) + } + } + + return doltdb.ErrTagNotFound +} diff --git a/go/libraries/doltcore/env/actions/tag_test.go b/go/libraries/doltcore/env/actions/tag_test.go new file mode 100644 index 0000000000..751fb1d4bd --- /dev/null +++ b/go/libraries/doltcore/env/actions/tag_test.go @@ -0,0 +1,129 @@ +package actions + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" + "github.com/dolthub/dolt/go/libraries/doltcore/env" + "github.com/dolthub/dolt/go/libraries/utils/filesys" + "github.com/dolthub/dolt/go/store/types" +) + +const ( + testHomeDir = "/user/bheni" + workingDir = "/user/bheni/datasets/addresses" + credsDir = "creds" + + configFile = "config.json" + GlobalConfigFile = "config_global.json" +) + +func testHomeDirFunc() (string, error) { + return testHomeDir, nil +} + +func createTestEnv() (*env.DoltEnv, *filesys.InMemFS) { + initialDirs := []string{testHomeDir, workingDir} + initialFiles := map[string][]byte{} + + fs := filesys.NewInMemFS(initialDirs, initialFiles, workingDir) + dEnv := env.Load(context.Background(), testHomeDirFunc, fs, doltdb.InMemDoltDB, "test") + + return dEnv, fs +} + +func TestVisitResolvedTag(t *testing.T) { + dEnv, _ := createTestEnv() + ctx := context.Background() + + // Initialize repo + err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "test@test.com", "main") + require.NoError(t, err) + + // Create a tag + tagName := "test-tag" + tagMsg := "test tag message" + err = CreateTag(ctx, dEnv, tagName, "main", TagProps{TaggerName: "test user", TaggerEmail: "test@test.com", Description: tagMsg}) + require.NoError(t, err) + + // Visit the tag and verify its properties + var foundTag *doltdb.Tag + err = VisitResolvedTag(ctx, dEnv.DoltDB, tagName, func(tag *doltdb.Tag) error { + foundTag = tag + return nil + }) + require.NoError(t, err) + require.NotNil(t, foundTag) + require.Equal(t, tagName, foundTag.Name) + require.Equal(t, tagMsg, foundTag.Meta.Description) + + // Test visiting non-existent tag + err = VisitResolvedTag(ctx, dEnv.DoltDB, "non-existent-tag", func(tag *doltdb.Tag) error { + return nil + }) + require.Equal(t, doltdb.ErrTagNotFound, err) +} + +func TestIterResolvedTagsPaginated(t *testing.T) { + dEnv, _ := createTestEnv() + ctx := context.Background() + + // Initialize repo + err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "test@test.com", "main") + require.NoError(t, err) + + expectedTagNames := make([]string, DefaultPageSize*2) + // Create multiple tags with different timestamps + tagNames := make([]string, DefaultPageSize*2) + for i := range tagNames { + tagName := fmt.Sprintf("tag-%d", i) + err = CreateTag(ctx, dEnv, tagName, "main", TagProps{ + TaggerName: "test user", + TaggerEmail: "test@test.com", + Description: fmt.Sprintf("test tag %s", tagName), + }) + time.Sleep(2 * time.Millisecond) + require.NoError(t, err) + tagNames[i] = tagName + expectedTagNames[len(expectedTagNames)-i-1] = tagName + } + + // Test first page + var foundTags []string + pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, nil, func(tag *doltdb.Tag) (bool, error) { + foundTags = append(foundTags, tag.Name) + return false, nil + }) + require.NoError(t, err) + require.NotNil(t, pageToken) // Should have next page + require.Equal(t, DefaultPageSize, len(foundTags)) // Default page size tags returned + + // Test second page + var secondPageTags []string + nextPageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, pageToken, func(tag *doltdb.Tag) (bool, error) { + secondPageTags = append(secondPageTags, tag.Name) + return false, nil + }) + require.NoError(t, err) + require.Nil(t, nextPageToken) // Should be no more pages + require.Equal(t, DefaultPageSize, len(secondPageTags)) // Remaining tags + + // Verify all tags were found + allFoundTags := append(foundTags, secondPageTags...) + require.Equal(t, len(tagNames), len(allFoundTags)) + require.Equal(t, expectedTagNames, allFoundTags) + + // Test early termination + var earlyTermTags []string + _, err = IterResolvedTagsPaginated(ctx, dEnv.DoltDB, nil, func(tag *doltdb.Tag) (bool, error) { + earlyTermTags = append(earlyTermTags, tag.Name) + return true, nil // Stop after first tag + }) + require.NoError(t, err) + require.Equal(t, 1, len(earlyTermTags)) +} From 3a055ebca4ab8d0cd46f1e39067426cb60e322ff Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 11:29:33 -0800 Subject: [PATCH 2/7] /go/libraries/doltcore/env/actions: fix tag page --- go/libraries/doltcore/env/actions/tag.go | 35 ++++++++----------- go/libraries/doltcore/env/actions/tag_test.go | 8 ++--- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index 9c96223015..bc801bc13e 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -144,18 +144,14 @@ type TagRefWithMeta struct { Meta *datas.TagMeta } -type RefPageToken struct { - TagName string -} - const DefaultPageSize = 100 // IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb(). -// Returns a next page token if there are more results available. -func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, pageToken *RefPageToken, cb func(tag *doltdb.Tag) (stop bool, err error)) (*RefPageToken, error) { +// Returns the next tag name if there are more results available. +func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag string, cb func(tag *doltdb.Tag) (stop bool, err error)) (string, error) { tagRefs, err := ddb.GetTags(ctx) if err != nil { - return nil, err + return "", err } // for each tag, get the meta @@ -163,11 +159,11 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, pageToke for _, r := range tagRefs { tr, ok := r.(ref.TagRef) if !ok { - return nil, fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") + return "", fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") } meta, err := ddb.ResolveTagMeta(ctx, tr) if err != nil { - return nil, err + return "", err } tagMetas = append(tagMetas, &TagRefWithMeta{TagRef: tr, Meta: meta}) } @@ -177,12 +173,12 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, pageToke return tagMetas[i].Meta.Timestamp > tagMetas[j].Meta.Timestamp }) - // find starting index based on page token + // find starting index based on start tag startIdx := 0 - if pageToken != nil && pageToken.TagName != "" { + if startTag != "" { for i, tm := range tagMetas { - if tm.TagRef.GetPath() == pageToken.TagName { - startIdx = i + 1 // start after the token + if tm.TagRef.GetPath() == startTag { + startIdx = i + 1 // start after the given tag break } } @@ -200,28 +196,25 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, pageToke for _, tm := range pageTagMetas { tag, err := ddb.ResolveTag(ctx, tm.TagRef) if err != nil { - return nil, err + return "", err } stop, err := cb(tag) if err != nil { - return nil, err + return "", err } if stop { break } } - // return next page token if there are more results - var nextPageToken *RefPageToken + // return next tag name if there are more results if endIdx < len(tagMetas) { lastTag := pageTagMetas[len(pageTagMetas)-1] - nextPageToken = &RefPageToken{ - TagName: lastTag.TagRef.GetPath(), - } + return lastTag.TagRef.GetPath(), nil } - return nextPageToken, nil + return "", nil } // VisitResolvedTag iterates over tags in ddb until the given tag name is found, then calls cb() with the resolved tag. diff --git a/go/libraries/doltcore/env/actions/tag_test.go b/go/libraries/doltcore/env/actions/tag_test.go index 751fb1d4bd..614e54601f 100644 --- a/go/libraries/doltcore/env/actions/tag_test.go +++ b/go/libraries/doltcore/env/actions/tag_test.go @@ -95,12 +95,12 @@ func TestIterResolvedTagsPaginated(t *testing.T) { // Test first page var foundTags []string - pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, nil, func(tag *doltdb.Tag) (bool, error) { + pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) { foundTags = append(foundTags, tag.Name) return false, nil }) require.NoError(t, err) - require.NotNil(t, pageToken) // Should have next page + require.NotEmpty(t, pageToken) // Should have next page require.Equal(t, DefaultPageSize, len(foundTags)) // Default page size tags returned // Test second page @@ -110,7 +110,7 @@ func TestIterResolvedTagsPaginated(t *testing.T) { return false, nil }) require.NoError(t, err) - require.Nil(t, nextPageToken) // Should be no more pages + require.Empty(t, nextPageToken) // Should be no more pages require.Equal(t, DefaultPageSize, len(secondPageTags)) // Remaining tags // Verify all tags were found @@ -120,7 +120,7 @@ func TestIterResolvedTagsPaginated(t *testing.T) { // Test early termination var earlyTermTags []string - _, err = IterResolvedTagsPaginated(ctx, dEnv.DoltDB, nil, func(tag *doltdb.Tag) (bool, error) { + _, err = IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) { earlyTermTags = append(earlyTermTags, tag.Name) return true, nil // Stop after first tag }) From c067fd22ebeab03b44dd66593ee53256b2ac1a8c Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 12:24:11 -0800 Subject: [PATCH 3/7] /go/libraries/doltcore/{doltdb,env}: another optimization --- go/libraries/doltcore/doltdb/doltdb.go | 19 +++++++++++--- go/libraries/doltcore/doltdb/tag.go | 25 +++++++++++++++++++ go/libraries/doltcore/env/actions/tag.go | 13 +++------- go/libraries/doltcore/env/actions/tag_test.go | 1 - 4 files changed, 45 insertions(+), 13 deletions(-) diff --git a/go/libraries/doltcore/doltdb/doltdb.go b/go/libraries/doltcore/doltdb/doltdb.go index 4a137b3dbe..b05564eb37 100644 --- a/go/libraries/doltcore/doltdb/doltdb.go +++ b/go/libraries/doltcore/doltdb/doltdb.go @@ -88,6 +88,13 @@ type DoltDB struct { databaseName string } +type TagRefWithMeta struct { + TagRef ref.TagRef + Meta *datas.TagMeta + CommitAddr hash.Hash + MaybeHeadAddr hash.Hash +} + // DoltDBFromCS creates a DoltDB from a noms chunks.ChunkStore func DoltDBFromCS(cs chunks.ChunkStore, databaseName string) *DoltDB { vrw := types.NewValueStore(cs) @@ -594,8 +601,13 @@ func (ddb *DoltDB) ResolveTag(ctx context.Context, tagRef ref.TagRef) (*Tag, err return NewTag(ctx, tagRef.GetPath(), ds, ddb.vrw, ddb.ns) } +// ResolveTag takes a TagRef and returns the corresponding Tag object. +func (ddb *DoltDB) ResolveTagFromTagRefWithMeta(ctx context.Context, tagRefWithMeta *TagRefWithMeta) (*Tag, error) { + return NewTagFromTagRefWithMeta(ctx, tagRefWithMeta.TagRef.GetPath(), tagRefWithMeta, ddb.vrw, ddb.ns) +} + // ResolveTagMeta takes a TagRef and returns the corresponding TagMeta object. -func (ddb *DoltDB) ResolveTagMeta(ctx context.Context, tagRef ref.TagRef) (*datas.TagMeta, error) { +func (ddb *DoltDB) ResolveTagMeta(ctx context.Context, tagRef ref.TagRef) (*TagRefWithMeta, error) { ds, err := ddb.db.GetDataset(ctx, tagRef.String()) if err != nil { return nil, ErrTagNotFound @@ -609,12 +621,13 @@ func (ddb *DoltDB) ResolveTagMeta(ctx context.Context, tagRef ref.TagRef) (*data return nil, fmt.Errorf("tagRef head is not a tag") } - meta, _, err := ds.HeadTag() + meta, commitAddr, err := ds.HeadTag() if err != nil { return nil, err } - return meta, nil + addr, _ := ds.MaybeHeadAddr() + return &TagRefWithMeta{TagRef: tagRef, Meta: meta, CommitAddr: commitAddr, MaybeHeadAddr: addr}, nil } // ResolveWorkingSet takes a WorkingSetRef and returns the corresponding WorkingSet object. diff --git a/go/libraries/doltcore/doltdb/tag.go b/go/libraries/doltcore/doltdb/tag.go index 2186e8bd91..a85e2fc028 100644 --- a/go/libraries/doltcore/doltdb/tag.go +++ b/go/libraries/doltcore/doltdb/tag.go @@ -32,6 +32,31 @@ type Tag struct { Commit *Commit } +// NewTagFromTagRefWithMeta creates a new Tag object from a TagRefWithMeta. +func NewTagFromTagRefWithMeta(ctx context.Context, name string, tagRefWithMeta *TagRefWithMeta, vrw types.ValueReadWriter, ns tree.NodeStore) (*Tag, error) { + dc, err := datas.LoadCommitAddr(ctx, vrw, tagRefWithMeta.CommitAddr) + if err != nil { + return nil, err + } + + if dc.IsGhost() { + return nil, ErrGhostCommitEncountered + } + + commit, err := NewCommit(ctx, vrw, ns, dc) + if err != nil { + return nil, err + } + + return &Tag{ + Name: name, + vrw: vrw, + addr: tagRefWithMeta.MaybeHeadAddr, + Meta: tagRefWithMeta.Meta, + Commit: commit, + }, nil +} + // NewTag creates a new Tag object. func NewTag(ctx context.Context, name string, ds datas.Dataset, vrw types.ValueReadWriter, ns tree.NodeStore) (*Tag, error) { meta, commitAddr, err := ds.HeadTag() diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index bc801bc13e..9d68803003 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -139,11 +139,6 @@ func IterResolvedTags(ctx context.Context, ddb *doltdb.DoltDB, cb func(tag *dolt return nil } -type TagRefWithMeta struct { - TagRef ref.TagRef - Meta *datas.TagMeta -} - const DefaultPageSize = 100 // IterResolvedTagsPaginated iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb(). @@ -155,17 +150,17 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag } // for each tag, get the meta - tagMetas := make([]*TagRefWithMeta, 0, len(tagRefs)) + tagMetas := make([]*doltdb.TagRefWithMeta, 0, len(tagRefs)) for _, r := range tagRefs { tr, ok := r.(ref.TagRef) if !ok { return "", fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") } - meta, err := ddb.ResolveTagMeta(ctx, tr) + tm, err := ddb.ResolveTagMeta(ctx, tr) if err != nil { return "", err } - tagMetas = append(tagMetas, &TagRefWithMeta{TagRef: tr, Meta: meta}) + tagMetas = append(tagMetas, tm) } // sort by meta timestamp @@ -194,7 +189,7 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag // resolve tags for this page for _, tm := range pageTagMetas { - tag, err := ddb.ResolveTag(ctx, tm.TagRef) + tag, err := ddb.ResolveTagFromTagRefWithMeta(ctx, tm) if err != nil { return "", err } diff --git a/go/libraries/doltcore/env/actions/tag_test.go b/go/libraries/doltcore/env/actions/tag_test.go index 614e54601f..508d7fcb04 100644 --- a/go/libraries/doltcore/env/actions/tag_test.go +++ b/go/libraries/doltcore/env/actions/tag_test.go @@ -68,7 +68,6 @@ func TestVisitResolvedTag(t *testing.T) { }) require.Equal(t, doltdb.ErrTagNotFound, err) } - func TestIterResolvedTagsPaginated(t *testing.T) { dEnv, _ := createTestEnv() ctx := context.Background() From d95877b6d33ec301841b1aef5b48a15459745132 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 13:39:26 -0800 Subject: [PATCH 4/7] /go/libraries/doltcore/env/actions/tag_test.go: fix header --- go/libraries/doltcore/env/actions/tag_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/libraries/doltcore/env/actions/tag_test.go b/go/libraries/doltcore/env/actions/tag_test.go index 508d7fcb04..af9f85af0b 100644 --- a/go/libraries/doltcore/env/actions/tag_test.go +++ b/go/libraries/doltcore/env/actions/tag_test.go @@ -1,3 +1,17 @@ +// Copyright 2025 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package actions import ( From bff68790f889b92240ac54f3afb058a50f08f031 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 13:50:10 -0800 Subject: [PATCH 5/7] /go/libraries/doltcore/env/actions: add name ranked list tags paginated option --- go/libraries/doltcore/env/actions/tag.go | 53 +++++++++++++++++ go/libraries/doltcore/env/actions/tag_test.go | 59 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index 9d68803003..7a1a3b9d5a 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -212,6 +212,59 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag return "", nil } +// IterResolvedTagsByNamePaginated iterates over tags in dEnv.DoltDB from newest to oldest, resolving the tag to a commit and calling cb(). +// Returns the next tag name if there are more results available. +func IterResolvedTagsByNamePaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag string, cb func(tag *doltdb.Tag) (stop bool, err error)) (string, error) { + // tags returned here are sorted lexicographically + tagRefs, err := ddb.GetTags(ctx) + if err != nil { + return "", err + } + + // find starting index based on start tag + startIdx := 0 + if startTag != "" { + for i, tr := range tagRefs { + if tr.GetPath() == startTag { + startIdx = i + 1 // start after the given tag + break + } + } + } + + // get page of results + endIdx := startIdx + DefaultPageSize + if endIdx > len(tagRefs) { + endIdx = len(tagRefs) + } + + pageTagRefs := tagRefs[startIdx:endIdx] + + // resolve tags for this page + for _, tr := range pageTagRefs { + tag, err := ddb.ResolveTag(ctx, tr.(ref.TagRef)) + if err != nil { + return "", err + } + + stop, err := cb(tag) + if err != nil { + return "", err + } + if stop { + break + } + } + + // return next tag name if there are more results + if endIdx < len(tagRefs) { + lastTag := pageTagRefs[len(pageTagRefs)-1] + return lastTag.GetPath(), nil + } + + return "", nil +} + // VisitResolvedTag iterates over tags in ddb until the given tag name is found, then calls cb() with the resolved tag. func VisitResolvedTag(ctx context.Context, ddb *doltdb.DoltDB, tagName string, cb func(tag *doltdb.Tag) error) error { tagRefs, err := ddb.GetTags(ctx) diff --git a/go/libraries/doltcore/env/actions/tag_test.go b/go/libraries/doltcore/env/actions/tag_test.go index af9f85af0b..40df0048d5 100644 --- a/go/libraries/doltcore/env/actions/tag_test.go +++ b/go/libraries/doltcore/env/actions/tag_test.go @@ -140,3 +140,62 @@ func TestIterResolvedTagsPaginated(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, len(earlyTermTags)) } + +func TestIterResolvedTagsByNamePaginated(t *testing.T) { + dEnv, _ := createTestEnv() + ctx := context.Background() + + // Initialize repo + err := dEnv.InitRepo(ctx, types.Format_Default, "test user", "test@test.com", "main") + require.NoError(t, err) + + expectedTagNames := make([]string, DefaultPageSize*2) + // Create multiple tags with different timestamps + tagNames := make([]string, DefaultPageSize*2) + for i := range tagNames { + tagName := fmt.Sprintf("tag-%d", i) + err = CreateTag(ctx, dEnv, tagName, "main", TagProps{ + TaggerName: "test user", + TaggerEmail: "test@test.com", + Description: fmt.Sprintf("test tag %s", tagName), + }) + time.Sleep(2 * time.Millisecond) + require.NoError(t, err) + tagNames[i] = tagName + expectedTagNames[len(expectedTagNames)-i-1] = tagName + } + + // Test first page + var foundTags []string + pageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) { + foundTags = append(foundTags, tag.Name) + return false, nil + }) + require.NoError(t, err) + require.NotEmpty(t, pageToken) // Should have next page + require.Equal(t, DefaultPageSize, len(foundTags)) // Default page size tags returned + + // Test second page + var secondPageTags []string + nextPageToken, err := IterResolvedTagsPaginated(ctx, dEnv.DoltDB, pageToken, func(tag *doltdb.Tag) (bool, error) { + secondPageTags = append(secondPageTags, tag.Name) + return false, nil + }) + require.NoError(t, err) + require.Empty(t, nextPageToken) // Should be no more pages + require.Equal(t, DefaultPageSize, len(secondPageTags)) // Remaining tags + + // Verify all tags were found + allFoundTags := append(foundTags, secondPageTags...) + require.Equal(t, len(tagNames), len(allFoundTags)) + require.Equal(t, expectedTagNames, allFoundTags) + + // Test early termination + var earlyTermTags []string + _, err = IterResolvedTagsByNamePaginated(ctx, dEnv.DoltDB, "", func(tag *doltdb.Tag) (bool, error) { + earlyTermTags = append(earlyTermTags, tag.Name) + return true, nil // Stop after first tag + }) + require.NoError(t, err) + require.Equal(t, 1, len(earlyTermTags)) +} From cb1b879fe1520a24748762416baa161ad4ebc84b Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 14:29:16 -0800 Subject: [PATCH 6/7] /go/libraries/doltcore/env/actions/tag.go: minimal parallelism too --- go/libraries/doltcore/env/actions/tag.go | 42 +++++++++++++++++------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index 7a1a3b9d5a..85e7c5b030 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -23,6 +23,7 @@ import ( "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/store/datas" + "golang.org/x/sync/errgroup" ) type TagProps struct { @@ -149,18 +150,37 @@ func IterResolvedTagsPaginated(ctx context.Context, ddb *doltdb.DoltDB, startTag return "", err } + eg, egCtx := errgroup.WithContext(ctx) + eg.SetLimit(5) + // for each tag, get the meta - tagMetas := make([]*doltdb.TagRefWithMeta, 0, len(tagRefs)) - for _, r := range tagRefs { - tr, ok := r.(ref.TagRef) - if !ok { - return "", fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") - } - tm, err := ddb.ResolveTagMeta(ctx, tr) - if err != nil { - return "", err - } - tagMetas = append(tagMetas, tm) + tagMetas := make([]*doltdb.TagRefWithMeta, len(tagRefs)) + for idx, r := range tagRefs { + idx, r := idx, r + + eg.Go(func() error { + if egCtx.Err() != nil { + return egCtx.Err() + } + + tr, ok := r.(ref.TagRef) + if !ok { + return fmt.Errorf("DoltDB.GetTags() returned non-tag DoltRef") + } + + tm, err := ddb.ResolveTagMeta(ctx, tr) + if err != nil { + return err + } + + tagMetas[idx] = tm + return nil + }) + + } + + if err := eg.Wait(); err != nil { + return "", err } // sort by meta timestamp From 4b6f429413ad354ad94609eb2df99bb83a68ba7c Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Fri, 17 Jan 2025 22:37:19 +0000 Subject: [PATCH 7/7] [ga-format-pr] Run go/utils/repofmt/format_repo.sh and go/Godeps/update.sh --- go/libraries/doltcore/env/actions/tag.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/libraries/doltcore/env/actions/tag.go b/go/libraries/doltcore/env/actions/tag.go index 85e7c5b030..b46c99deb2 100644 --- a/go/libraries/doltcore/env/actions/tag.go +++ b/go/libraries/doltcore/env/actions/tag.go @@ -19,11 +19,12 @@ import ( "fmt" "sort" + "golang.org/x/sync/errgroup" + "github.com/dolthub/dolt/go/libraries/doltcore/doltdb" "github.com/dolthub/dolt/go/libraries/doltcore/env" "github.com/dolthub/dolt/go/libraries/doltcore/ref" "github.com/dolthub/dolt/go/store/datas" - "golang.org/x/sync/errgroup" ) type TagProps struct {