alice.pds.demo.boltless.dev
This repository has no description
cmd/zoekt-tngl-indexserver: validate branch names & object ids before running git
Lewis: May this revision serve well! <did:plc:3fwecdnvtcscjnrx2p4n7alz>
author
did:plc:3fwecdnvtcscjnrx2p4n7a…
committer
Tangled
date
(Jul 30, 2026, 3:00 PM UTC)
commit
1a044ae3
1a044ae3117e694fe2abdf5273186391d4925c3f
parent
9f81b7fa
9f81b7fa6889326f8ea5724a2244266006b261b5
change-id
zoqmwmum
zoqmwmumswrkkqyrsmrvoopkkvxuyxow
did:plc:3fwecdnvtcscjnrx2p4n7a…
committer
+164
-39
+15
-25
cmd/zoekt-tngl-indexserver/index.go
+15
-25
cmd/zoekt-tngl-indexserver/index.go
···
4
4
"context"
5
5
"crypto/sha1"
6
6
"encoding/json"
7
+
"errors"
7
8
"fmt"
8
9
"io"
9
10
"net/url"
···
12
13
13
14
"github.com/bluesky-social/indigo/atproto/identity"
14
15
"github.com/bluesky-social/indigo/atproto/syntax"
15
-
"github.com/sourcegraph/zoekt"
16
+
"github.com/samber/lo"
16
17
"tangled.org/core/repoident"
17
18
"tangled.org/core/repoverify"
18
19
)
···
71
72
}, nil
72
73
}
73
74
74
-
func fetchRepo(ctx context.Context, gitDir, cloneUrl string, branches []zoekt.RepositoryBranch) error {
75
+
func fetchRepo(ctx context.Context, gitDir, cloneUrl string, branches []indexBranch) error {
75
76
// Create a repo to fetch into
76
77
if err := executeCmd(ctx,
77
78
"git",
···
87
88
return err
88
89
}
89
90
90
-
fetchArgs := []string{
91
+
fetchArgs := append([]string{
91
92
"-C", gitDir,
92
93
"-c", "protocol.version=2",
93
94
"fetch", "--depth=1", "--no-tags",
94
-
}
95
-
// Git's blob:limit filter excludes blobs whose size is >= the given limit,
96
-
// while zoekt indexes files up to and including FileLimit bytes.
97
-
fetchArgs = append(fetchArgs, fmt.Sprintf("--filter=blob:limit=%d", int64(MaxFileSize)+1))
98
-
99
-
fetchArgs = append(fetchArgs, cloneUrl)
100
-
101
-
var commits []string
102
-
for _, b := range branches {
103
-
commits = append(commits, b.Version)
104
-
}
105
-
fetchArgs = append(fetchArgs, commits...)
95
+
// Git's blob:limit filter excludes blobs whose size is >= the given limit,
96
+
// while zoekt indexes files up to and including FileLimit bytes.
97
+
fmt.Sprintf("--filter=blob:limit=%d", int64(MaxFileSize)+1),
98
+
cloneUrl,
99
+
}, lo.Map(branches, func(b indexBranch, _ int) string { return string(b.Version) })...)
106
100
107
101
if err := executeCmd(ctx, "git", fetchArgs...); err != nil {
108
102
return err
109
103
}
110
104
111
-
for _, b := range branches {
112
-
ref := b.Name
113
-
if ref != "HEAD" {
114
-
ref = "refs/heads/" + ref
115
-
}
116
-
if err := executeCmd(ctx, "git", "-C", gitDir, "update-ref", ref, b.Version); err != nil {
117
-
return fmt.Errorf("failed update-ref %s to %s: %w", ref, b.Version, err)
105
+
return errors.Join(lo.FilterMap(branches, func(b indexBranch, _ int) (error, bool) {
106
+
err := executeCmd(ctx, "git", "-C", gitDir, "update-ref", b.Name.Ref(), string(b.Version))
107
+
if err == nil {
108
+
return nil, false
118
109
}
119
-
}
120
-
121
-
return nil
110
+
return fmt.Errorf("failed update-ref %s to %s: %w", b.Name.Ref(), b.Version, err), true
111
+
})...)
122
112
}
123
113
124
114
func indexRepo(ctx context.Context, cfg *Config, gitDir string, repo Repo) error {
+2
-3
cmd/zoekt-tngl-indexserver/main.go
+2
-3
cmd/zoekt-tngl-indexserver/main.go
···
21
21
"github.com/bluesky-social/indigo/atproto/syntax"
22
22
"github.com/carlmjohnson/versioninfo"
23
23
"github.com/samber/lo"
24
-
"github.com/sourcegraph/zoekt"
25
24
"github.com/sourcegraph/zoekt/gitindex"
26
25
"github.com/sourcegraph/zoekt/index"
27
26
"github.com/urfave/cli/v3"
···
175
174
Owner repoident.OwnerDid
176
175
Slug syntax.RecordKey
177
176
Knot repoident.KnotURL
178
-
Branches []zoekt.RepositoryBranch
177
+
Branches []indexBranch
179
178
}
180
179
181
180
func (r *Repo) CloneURL() string {
···
227
226
return fmt.Errorf("repo is missing did, owner, or knot: %q", repoRaw)
228
227
}
229
228
230
-
branches := lo.Map(repo.Branches, func(b zoekt.RepositoryBranch, _ int) string { return b.Name })
229
+
branches := lo.Map(repo.Branches, func(b indexBranch, _ int) string { return string(b.Name) })
231
230
232
231
buildOpts := index.Options{}
233
232
buildOpts.SetDefaults()
+63
-11
cmd/zoekt-tngl-indexserver/server.go
+63
-11
cmd/zoekt-tngl-indexserver/server.go
···
3
3
import (
4
4
"context"
5
5
"encoding/json"
6
+
"errors"
6
7
"fmt"
7
8
"log"
8
9
"net/http"
10
+
"regexp"
9
11
"strconv"
12
+
"strings"
10
13
"time"
11
14
12
15
"github.com/bluesky-social/indigo/atproto/identity"
16
+
"github.com/go-git/go-git/v5/plumbing"
13
17
"github.com/prometheus/client_golang/prometheus"
14
18
"github.com/prometheus/client_golang/prometheus/promhttp"
15
-
"github.com/sourcegraph/zoekt"
19
+
"github.com/samber/lo"
16
20
"tangled.org/core/repoident"
17
21
)
18
22
···
58
62
promhttp.Handler().ServeHTTP(w, r)
59
63
}
60
64
65
+
type branchName string
66
+
67
+
func (b branchName) Ref() string {
68
+
return lo.Ternary(b == "HEAD", "HEAD", "refs/heads/"+string(b))
69
+
}
70
+
71
+
func (b *branchName) UnmarshalText(text []byte) error {
72
+
name := branchName(text)
73
+
if strings.HasPrefix(string(name), "refs/") {
74
+
return fmt.Errorf("branch %q must be a short name, without the refs/ prefix", name)
75
+
}
76
+
if err := plumbing.ReferenceName(name.Ref()).Validate(); err != nil {
77
+
return fmt.Errorf("branch %q isn't a valid ref: %w", name, err)
78
+
}
79
+
*b = name
80
+
return nil
81
+
}
82
+
83
+
type objectID string
84
+
85
+
var objectIDPattern = regexp.MustCompile(`^([0-9a-fA-F]{40}|[0-9a-fA-F]{64})$`)
86
+
87
+
func (o *objectID) UnmarshalText(text []byte) error {
88
+
if !objectIDPattern.Match(text) {
89
+
return fmt.Errorf("%q isn't a sha1 or sha256 object id", text)
90
+
}
91
+
*o = objectID(text)
92
+
return nil
93
+
}
94
+
95
+
type indexBranch struct {
96
+
Name branchName `json:"name"`
97
+
Version objectID `json:"version"`
98
+
}
99
+
61
100
type indexRequest struct {
62
-
Repo repoident.RepoDid `json:"repo"`
63
-
Branches []zoekt.RepositoryBranch `json:"branches"`
101
+
Repo repoident.RepoDid `json:"repo"`
102
+
Branches []indexBranch `json:"branches"`
103
+
}
104
+
105
+
func decodeIndexRequest(r *http.Request) (indexRequest, error) {
106
+
dec := json.NewDecoder(r.Body)
107
+
dec.DisallowUnknownFields()
108
+
var req indexRequest
109
+
if err := dec.Decode(&req); err != nil {
110
+
return indexRequest{}, err
111
+
}
112
+
if req.Repo == "" {
113
+
return indexRequest{}, errors.New("index request has no repo did")
114
+
}
115
+
if len(req.Branches) == 0 {
116
+
return indexRequest{}, fmt.Errorf("index request for %s has no branches", req.Repo)
117
+
}
118
+
return req, nil
64
119
}
65
120
66
121
func (s *IndexServer) handleDebugQueue(w http.ResponseWriter, r *http.Request) {
···
74
129
75
130
func (s *IndexServer) handleEnqueueIndex(w http.ResponseWriter, r *http.Request) {
76
131
route := "enqueueIndex"
77
-
dec := json.NewDecoder(r.Body)
78
-
dec.DisallowUnknownFields()
79
-
var req indexRequest
80
-
if err := dec.Decode(&req); err != nil {
132
+
req, err := decodeIndexRequest(r)
133
+
if err != nil {
81
134
log.Printf("Error decoding index request: %v", err)
82
135
http.Error(w, "JSON parser error", http.StatusBadRequest)
83
136
s.incrementRequestsTotal(r.Method, route, http.StatusBadRequest)
···
97
150
98
151
func (s *IndexServer) handleForceIndex(w http.ResponseWriter, r *http.Request) {
99
152
route := "index"
100
-
dec := json.NewDecoder(r.Body)
101
-
dec.DisallowUnknownFields()
102
-
var req indexRequest
103
-
if err := dec.Decode(&req); err != nil {
153
+
req, err := decodeIndexRequest(r)
154
+
if err != nil {
104
155
log.Printf("Error decoding index request: %v", err)
105
156
http.Error(w, "JSON parser error", http.StatusBadRequest)
157
+
s.incrementRequestsTotal(r.Method, route, http.StatusBadRequest)
106
158
return
107
159
}
108
160
+84
cmd/zoekt-tngl-indexserver/server_test.go
+84
cmd/zoekt-tngl-indexserver/server_test.go
···
1
+
package main
2
+
3
+
import (
4
+
"fmt"
5
+
"net/http/httptest"
6
+
"strings"
7
+
"testing"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/syntax"
10
+
"tangled.org/core/repoident"
11
+
)
12
+
13
+
var (
14
+
sha1Oid = strings.Repeat("a", 40)
15
+
sha256Oid = strings.Repeat("b", 64)
16
+
)
17
+
18
+
func decodeBody(t *testing.T, body string) (indexRequest, error) {
19
+
t.Helper()
20
+
return decodeIndexRequest(httptest.NewRequest("POST", "/admin/enqueueIndex", strings.NewReader(body)))
21
+
}
22
+
23
+
func TestDecodeIndexRequest_Accepts(t *testing.T) {
24
+
cases := map[string]struct {
25
+
name, oid, wantRef string
26
+
}{
27
+
"a branch and a sha1": {name: "main", oid: sha1Oid, wantRef: "refs/heads/main"},
28
+
"HEAD and a sha256": {name: "HEAD", oid: sha256Oid, wantRef: "HEAD"},
29
+
}
30
+
for label, tc := range cases {
31
+
t.Run(label, func(t *testing.T) {
32
+
req, err := decodeBody(t, fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":%q,"Version":%q}]}`, tc.name, tc.oid))
33
+
if err != nil {
34
+
t.Fatalf("decodeIndexRequest: %v", err)
35
+
}
36
+
if req.Repo.String() != "did:plc:limpet" {
37
+
t.Errorf("Repo = %q, want did:plc:limpet", req.Repo)
38
+
}
39
+
want := indexBranch{Name: branchName(tc.name), Version: objectID(tc.oid)}
40
+
if len(req.Branches) != 1 || req.Branches[0] != want {
41
+
t.Errorf("Branches = %v, want %v", req.Branches, want)
42
+
}
43
+
if got := req.Branches[0].Name.Ref(); got != tc.wantRef {
44
+
t.Errorf("Ref = %q, want %q", got, tc.wantRef)
45
+
}
46
+
})
47
+
}
48
+
}
49
+
50
+
func TestDecodeIndexRequest_RejectsBadRequests(t *testing.T) {
51
+
cases := map[string]string{
52
+
"branch name is a git option": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"-d","Version":%q}]}`, sha1Oid),
53
+
"version is a git option": `{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":"--upload-pack=touch /tmp/pwned"}]}`,
54
+
"version is a ref": `{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":"refs/heads/main"}]}`,
55
+
"version is short hex": `{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":"deadbeef"}]}`,
56
+
"branch name walks up": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"../../objects","Version":%q}]}`, sha1Oid),
57
+
"branch name is a full ref": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"refs/heads/main","Version":%q}]}`, sha1Oid),
58
+
"branch name is empty": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"","Version":%q}]}`, sha1Oid),
59
+
"repo isn't a did": fmt.Sprintf(`{"repo":"limpet","branches":[{"Name":"main","Version":%q}]}`, sha1Oid),
60
+
"repo is absent": fmt.Sprintf(`{"branches":[{"Name":"main","Version":%q}]}`, sha1Oid),
61
+
"branches are absent": `{"repo":"did:plc:limpet"}`,
62
+
"branches are empty": `{"repo":"did:plc:limpet","branches":[]}`,
63
+
"unknown field": fmt.Sprintf(`{"repo":"did:plc:limpet","branches":[{"Name":"main","Version":%q}],"shards":3}`, sha1Oid),
64
+
}
65
+
for name, body := range cases {
66
+
t.Run(name, func(t *testing.T) {
67
+
if _, err := decodeBody(t, body); err == nil {
68
+
t.Errorf("decodeIndexRequest accepted %s", body)
69
+
}
70
+
})
71
+
}
72
+
}
73
+
74
+
func TestRepoCloneURL(t *testing.T) {
75
+
knot, err := repoident.ParseKnotURL("https://knot.oyster.cafe", repoident.RequireHTTPS)
76
+
if err != nil {
77
+
t.Fatalf("ParseKnotURL: %v", err)
78
+
}
79
+
repo := Repo{Did: "did:plc:limpet", Owner: "did:plc:akshay", Slug: syntax.RecordKey("3kkkkkkkkkkkk"), Knot: knot}
80
+
const want = "https://knot.oyster.cafe/did:plc:limpet"
81
+
if got := repo.CloneURL(); got != want {
82
+
t.Errorf("CloneURL = %q, want %q", got, want)
83
+
}
84
+
}