alice.pds.demo.boltless.dev
This repository has no description
gitutil,appview,knot{server,mirror}: serve archives standardized
Lewis: May this revision serve well! <did:plc:3fwecdnvtcscjnrx2p4n7alz>
author
did:plc:3fwecdnvtcscjnrx2p4n7a…
date
(Aug 4, 2026, 12:22 PM +0300)
commit
132d2bcf
132d2bcf6052fa5145fd1d25e6070b52d96cc1c0
parent
81b27e8d
81b27e8d13603d796391be5cd01a8135a7d19ea9
change-id
zkrzxqto
zkrzxqtozvrnnkyprmtsspxyxwkqslyy
did:plc:3fwecdnvtcscjnrx2p4n7a…
date
(Aug 4, 2026, 12:22 PM +0300)
commit
132d2bcf
132d2bcf6052fa5145fd1d25e6070b52d96cc1c0
parent
81b27e8d
81b27e8d13603d796391be5cd01a8135a7d19ea9
change-id
zkrzxqto
zkrzxqtozvrnnkyprmtsspxyxwkqslyy
+722
-285
+2
-1
appview/config/config.go
+2
-1
appview/config/config.go
···
69
69
}
70
70
71
71
type KnotMirrorConfig struct {
72
-
Url string `env:"URL, default=https://mirror.tangled.network"`
72
+
Url string `env:"URL, default=https://mirror.tangled.network"`
73
+
ArchiveHeaderTimeout time.Duration `env:"ARCHIVE_HEADER_TIMEOUT, default=60s"`
73
74
}
74
75
75
76
type JetstreamConfig struct {
+2
-2
appview/pages/templates/repo/fragments/cloneDropdown.html
+2
-2
appview/pages/templates/repo/fragments/cloneDropdown.html
···
68
68
69
69
<div class="flex gap-2 mt-4">
70
70
<a
71
-
href="/{{ .RepoInfo.FullName }}/archive/{{ .Ref | urlquery }}?format=tar.gz"
71
+
href="/{{ .RepoInfo.FullName }}/archive/{{ pathEscape .Ref }}?format=tar.gz"
72
72
class="btn flex-1"
73
73
>
74
74
{{ i "download" "w-4 h-4" }}
75
75
Download tar.gz
76
76
</a>
77
77
<a
78
-
href="/{{ .RepoInfo.FullName }}/archive/{{ .Ref | urlquery }}?format=zip"
78
+
href="/{{ .RepoInfo.FullName }}/archive/{{ pathEscape .Ref }}?format=zip"
79
79
class="btn flex-1"
80
80
>
81
81
{{ i "download" "w-4 h-4" }}
+81
-74
appview/repo/archive.go
+81
-74
appview/repo/archive.go
···
6
6
"net/http"
7
7
"net/url"
8
8
"strings"
9
+
"time"
9
10
10
11
"github.com/go-chi/chi/v5"
12
+
"github.com/samber/lo"
11
13
"tangled.org/core/api/tangled"
14
+
"tangled.org/core/appview/models"
15
+
"tangled.org/core/gitutil"
12
16
)
13
17
18
+
const archiveRoute = "/archive/*"
19
+
20
+
func newArchiveClient(headerTimeout time.Duration) *http.Client {
21
+
transport := http.DefaultTransport.(*http.Transport).Clone()
22
+
transport.ResponseHeaderTimeout = headerTimeout
23
+
return &http.Client{Transport: transport}
24
+
}
25
+
14
26
func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) {
15
27
l := rp.logger.With("handler", "DownloadArchive")
16
-
ref := chi.URLParam(r, "ref")
17
-
ref, _ = url.PathUnescape(ref)
18
-
format := r.URL.Query().Get("format")
19
-
ref, format = archiveRefAndFormat(ref, format, r.UserAgent())
28
+
fail := func(status int) {
29
+
w.WriteHeader(status)
30
+
lo.Ternary(status == http.StatusServiceUnavailable, rp.pages.Error503, rp.pages.Error404)(w)
31
+
}
32
+
33
+
params, err := parseArchiveRequest(r)
34
+
if err != nil {
35
+
l.Warn("rejecting archive request", "err", err)
36
+
fail(http.StatusNotFound)
37
+
return
38
+
}
39
+
20
40
f, err := rp.repoResolver.Resolve(r)
21
41
if err != nil {
22
42
l.Error("failed to get repo and knot", "err", err)
43
+
fail(http.StatusNotFound)
23
44
return
24
45
}
25
46
47
+
name := gitutil.RepoName(f.Slug())
48
+
params.Prefix = params.Prefix.OrDefault(name, params.Rev)
49
+
26
50
// build the xrpc url
27
-
query := url.Values{}
28
-
query.Set("repo", f.RepoDid)
29
-
query.Set("ref", ref)
30
-
query.Set("format", format)
31
-
query.Set("prefix", r.URL.Query().Get("prefix"))
32
-
xrpcURL := fmt.Sprintf(
33
-
"%s/xrpc/%s?%s",
34
-
rp.config.KnotMirror.Url,
35
-
tangled.GitTempGetArchiveNSID,
36
-
query.Encode(),
37
-
)
51
+
xrpcURL := fmt.Sprintf("%s/xrpc/%s?%s",
52
+
rp.config.KnotMirror.Url, tangled.GitTempGetArchiveNSID, params.Query(f.RepoDid).Encode())
38
53
39
54
// make the get request
40
-
resp, err := http.Get(xrpcURL)
55
+
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, xrpcURL, nil)
56
+
if err != nil {
57
+
l.Error("failed to build XRPC repo.archive request", "err", err)
58
+
fail(http.StatusServiceUnavailable)
59
+
return
60
+
}
61
+
resp, err := rp.archiveClient.Do(req)
41
62
if err != nil {
42
63
l.Error("failed to call XRPC repo.archive", "err", err)
43
-
rp.pages.Error503(w)
64
+
fail(http.StatusServiceUnavailable)
44
65
return
45
66
}
46
67
defer resp.Body.Close()
47
68
48
-
w.Header().Set("Content-Type", archiveContentType(format))
49
-
50
-
filename := ""
51
-
if cd := resp.Header.Get("Content-Disposition"); strings.HasPrefix(cd, "attachment;") {
52
-
filename = cd // knot has already set the attachment CD
69
+
if resp.StatusCode != http.StatusOK {
70
+
l.Error("XRPC repo.archive failed", "status", resp.StatusCode, "ref", params.Rev)
71
+
overloaded := resp.StatusCode >= http.StatusInternalServerError || resp.StatusCode == http.StatusTooManyRequests
72
+
fail(lo.Ternary(overloaded, http.StatusServiceUnavailable, http.StatusNotFound))
73
+
return
53
74
}
54
-
if filename == "" {
55
-
filename = fmt.Sprintf("attachment; filename=\"%s-%s.%s\"", f.Name, ref, format)
56
-
}
57
-
w.Header().Set("Content-Disposition", filename)
58
-
w.Header().Set("X-Content-Type-Options", "nosniff")
75
+
76
+
params.SetHeaders(w.Header(), name)
59
77
60
-
if link := resp.Header.Get("Link"); link != "" {
61
-
if resolvedRef, err := extractImmutableLink(link); err == nil {
62
-
newLink := fmt.Sprintf("<%s/%s/archive/%s.%s>; rel=\"immutable\"",
63
-
rp.config.Core.BaseUrl(), f.RepoIdentifier(), resolvedRef, format)
64
-
w.Header().Set("Link", newLink)
65
-
}
78
+
if resolvedRev, err := gitutil.ParseImmutableLink(resp.Header.Get("Link")); err == nil {
79
+
w.Header().Set("Link", gitutil.ImmutableLink(rp.immutableArchiveURL(f, params.WithRev(resolvedRev))))
66
80
}
67
81
68
82
// stream the archive data directly
···
71
85
}
72
86
}
73
87
74
-
func archiveRefAndFormat(ref string, requestedFormat string, userAgent string) (string, string) {
75
-
switch {
76
-
case strings.HasSuffix(ref, ".tar.gz"):
77
-
ref = strings.TrimSuffix(ref, ".tar.gz")
78
-
if requestedFormat == "" {
79
-
requestedFormat = "tar.gz"
80
-
}
81
-
case strings.HasSuffix(ref, ".zip"):
82
-
ref = strings.TrimSuffix(ref, ".zip")
83
-
if requestedFormat == "" {
84
-
requestedFormat = "zip"
85
-
}
88
+
func parseArchiveRequest(r *http.Request) (gitutil.ArchiveParams, error) {
89
+
ref := chi.URLParam(r, "*")
90
+
if unescaped, err := url.PathUnescape(ref); err == nil && r.URL.RawPath != "" {
91
+
ref = unescaped
86
92
}
87
93
88
-
switch requestedFormat {
89
-
case "zip", "tar.gz":
90
-
return ref, requestedFormat
91
-
default:
92
-
if prefersZipArchive(userAgent) {
93
-
return ref, "zip"
94
-
}
95
-
return ref, "tar.gz"
94
+
suffix, found := lo.Find(gitutil.ArchiveFormats, func(f gitutil.ArchiveFormat) bool {
95
+
return strings.HasSuffix(ref, "."+f.String())
96
+
})
97
+
if found {
98
+
ref = strings.TrimSuffix(ref, "."+suffix.String())
96
99
}
97
-
}
98
100
99
-
func prefersZipArchive(userAgent string) bool {
100
-
ua := strings.ToLower(userAgent)
101
-
return strings.Contains(ua, "windows") || strings.Contains(ua, "win64") || strings.Contains(ua, "win32")
102
-
}
101
+
rev, err := gitutil.ParseRev(ref)
102
+
if err != nil {
103
+
return gitutil.ArchiveParams{}, err
104
+
}
103
105
104
-
func archiveContentType(format string) string {
105
-
if format == "zip" {
106
-
return "application/zip"
106
+
query := r.URL.Query()
107
+
query.Del("ref")
108
+
query.Set("format", archiveFormat(query.Get("format"), suffix, r.UserAgent()).String())
109
+
params, err := gitutil.ParseArchiveParams(query)
110
+
if err != nil {
111
+
return gitutil.ArchiveParams{}, err
107
112
}
108
-
return "application/gzip"
113
+
return params.WithRev(rev), nil
109
114
}
110
115
111
-
func extractImmutableLink(linkHeader string) (string, error) {
112
-
trimmed := strings.TrimPrefix(linkHeader, "<")
113
-
trimmed = strings.TrimSuffix(trimmed, ">; rel=\"immutable\"")
114
-
115
-
parsedLink, err := url.Parse(trimmed)
116
-
if err != nil {
117
-
return "", err
116
+
func archiveFormat(requested string, suffix gitutil.ArchiveFormat, userAgent string) gitutil.ArchiveFormat {
117
+
if format, err := gitutil.ParseArchiveFormat(requested); err == nil {
118
+
return format
118
119
}
119
-
120
-
resolvedRef := parsedLink.Query().Get("ref")
121
-
if resolvedRef == "" {
122
-
return "", fmt.Errorf("no ref found in link header")
120
+
if suffix != "" {
121
+
return suffix
123
122
}
123
+
ua := strings.ToLower(userAgent)
124
+
windows := lo.SomeBy([]string{"windows", "win64", "win32"}, func(s string) bool { return strings.Contains(ua, s) })
125
+
return lo.Ternary(windows, gitutil.ArchiveZip, gitutil.ArchiveTarGz)
126
+
}
124
127
125
-
return resolvedRef, nil
128
+
func (rp *Repo) immutableArchiveURL(f *models.Repo, params gitutil.ArchiveParams) string {
129
+
return fmt.Sprintf("%s/%s/archive/%s.%s?%s",
130
+
rp.config.Core.BaseUrl(), f.RepoIdentifier(),
131
+
url.PathEscape(params.Rev.String()), params.Format,
132
+
url.Values{"prefix": {params.Prefix.String()}}.Encode())
126
133
}
+94
appview/repo/archive_test.go
+94
appview/repo/archive_test.go
···
1
+
package repo
2
+
3
+
import (
4
+
"net/http"
5
+
"net/http/httptest"
6
+
"net/url"
7
+
"strings"
8
+
"testing"
9
+
10
+
"github.com/go-chi/chi/v5"
11
+
"tangled.org/core/appview/config"
12
+
"tangled.org/core/appview/models"
13
+
"tangled.org/core/gitutil"
14
+
)
15
+
16
+
const (
17
+
testRepoDid = "did:plc:limpet"
18
+
testRepoOwner = "did:plc:boltless"
19
+
testRepoRkey = "3kzabcdefghij"
20
+
testRepoPath = "/boltless.dev/squid"
21
+
)
22
+
23
+
func TestParseArchiveRequest(t *testing.T) {
24
+
var (
25
+
got gitutil.ArchiveParams
26
+
gotErr error
27
+
)
28
+
router := chi.NewRouter()
29
+
router.Get("/{user}/{repo}"+archiveRoute, func(w http.ResponseWriter, r *http.Request) {
30
+
got, gotErr = parseArchiveRequest(r)
31
+
})
32
+
33
+
resolvedParams := gitutil.ArchiveParams{
34
+
Rev: "6f1d3a2b4c5d6e7f8091a2b3c4d5e6f708192a3b",
35
+
Format: gitutil.ArchiveZip,
36
+
Prefix: gitutil.ArchivePrefix("").OrDefault("squid", "refs/heads/feat/uni"),
37
+
}
38
+
rp := &Repo{config: &config.Config{Core: config.CoreConfig{Dev: true, AppviewHost: "tangled.org"}}}
39
+
immutable, err := url.Parse(rp.immutableArchiveURL(
40
+
&models.Repo{Did: testRepoOwner, Rkey: testRepoRkey, Name: "squid", RepoDid: testRepoDid},
41
+
resolvedParams,
42
+
))
43
+
if err != nil {
44
+
t.Fatalf("the immutable URL must parse: %v", err)
45
+
}
46
+
47
+
windows := "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
48
+
targz, zip := gitutil.ArchiveTarGz, gitutil.ArchiveZip
49
+
cases := []struct {
50
+
name string
51
+
path string
52
+
userAgent string
53
+
want gitutil.ArchiveParams
54
+
wantErr bool
55
+
}{
56
+
{"short ref", testRepoPath + "/archive/v1.0.0?format=tar.gz", "", gitutil.ArchiveParams{Rev: "v1.0.0", Format: targz}, false},
57
+
{"full ref unescaped", testRepoPath + "/archive/refs/tags/v1.0.0?prefix=did:plc:boltless", "", gitutil.ArchiveParams{Rev: "refs/tags/v1.0.0", Format: targz, Prefix: "did:plc:boltless"}, false},
58
+
{"full ref escaped", testRepoPath + "/archive/refs%2Ftags%2Fv1.0.0?prefix=did:plc:boltless", "", gitutil.ArchiveParams{Rev: "refs/tags/v1.0.0", Format: targz, Prefix: "did:plc:boltless"}, false},
59
+
{"format from suffix", testRepoPath + "/archive/refs/tags/v1.0.0.zip", "", gitutil.ArchiveParams{Rev: "refs/tags/v1.0.0", Format: zip}, false},
60
+
{"unknown format query with a zip suffix", testRepoPath + "/archive/refs/tags/v1.0.0.zip?format=tar.xz", "", gitutil.ArchiveParams{Rev: "refs/tags/v1.0.0", Format: zip}, false},
61
+
{"zip for a windows user agent", testRepoPath + "/archive/main?format=tar.xz", windows, gitutil.ArchiveParams{Rev: "main", Format: zip}, false},
62
+
{"percent in the ref itself", testRepoPath + "/archive/refs/tags/a%252Fb", "", gitutil.ArchiveParams{Rev: "refs/tags/a%2Fb", Format: targz}, false},
63
+
{"prefix wrapped in slashes", testRepoPath + "/archive/main?prefix=/kelp/", "", gitutil.ArchiveParams{Rev: "main", Format: targz, Prefix: "kelp"}, false},
64
+
{"traversal escaped", testRepoPath + "/archive/..%2F..%2Fetc", "", gitutil.ArchiveParams{Rev: "../../etc", Format: targz}, false},
65
+
{"parse deletes a ref query", testRepoPath + "/archive/main?ref=other", "", gitutil.ArchiveParams{Rev: "main", Format: targz}, false},
66
+
{"our own immutable URL", immutable.RequestURI(), "", resolvedParams, false},
67
+
68
+
{"empty ref", testRepoPath + "/archive/", "", gitutil.ArchiveParams{}, true},
69
+
{"escaped space", testRepoPath + "/archive/refs/tags/a%20b", "", gitutil.ArchiveParams{}, true},
70
+
{"ref that git would read as an option", testRepoPath + "/archive/--output=%2Ftmp%2Fevil", "", gitutil.ArchiveParams{}, true},
71
+
}
72
+
73
+
for _, tc := range cases {
74
+
t.Run(tc.name, func(t *testing.T) {
75
+
got, gotErr = gitutil.ArchiveParams{}, nil
76
+
path := tc.path
77
+
if rest, isRepoDid := strings.CutPrefix(path, "/"+testRepoDid); isRepoDid {
78
+
path = "/" + testRepoOwner + "/" + testRepoRkey + rest
79
+
}
80
+
81
+
req := httptest.NewRequest(http.MethodGet, path, nil)
82
+
req.Header.Set("User-Agent", tc.userAgent)
83
+
rec := httptest.NewRecorder()
84
+
router.ServeHTTP(rec, req)
85
+
86
+
if rec.Code != http.StatusOK {
87
+
t.Fatalf("%s: status = %d, want the archive route to match", path, rec.Code)
88
+
}
89
+
if got != tc.want || (gotErr != nil) != tc.wantErr {
90
+
t.Errorf("params = %+v with err %v, want %+v and rejected = %v", got, gotErr, tc.want, tc.wantErr)
91
+
}
92
+
})
93
+
}
94
+
}
+2
appview/repo/repo.go
+2
appview/repo/repo.go
···
61
61
codesearch *codesearch.CodeSearch
62
62
63
63
knotMirrorXRPC *indigoxrpc.Client
64
+
archiveClient *http.Client
64
65
}
65
66
66
67
func New(
···
93
94
codesearch: codesearch,
94
95
95
96
knotMirrorXRPC: newKnotMirrorXRPCClient(config.KnotMirror.Url),
97
+
archiveClient: newArchiveClient(config.KnotMirror.ArchiveHeaderTimeout),
96
98
}
97
99
}
98
100
+1
-3
appview/repo/router.go
+1
-3
appview/repo/router.go
···
45
45
r.Get("/blob/{ref}/*", rp.Blob)
46
46
r.Get("/raw/{ref}/*", rp.RepoBlobRaw)
47
47
48
-
// intentionally doesn't use /* as this isn't
49
-
// a file path
50
-
r.Get("/archive/{ref}", rp.DownloadArchive)
48
+
r.Get(archiveRoute, rp.DownloadArchive)
51
49
52
50
r.With(middleware.Paginate).Get("/stars", rp.Stars)
53
51
r.With(middleware.Paginate).Get("/forks", rp.Forks)
+1
-1
appview/reporesolver/resolver.go
+1
-1
appview/reporesolver/resolver.go
···
40
40
}
41
41
42
42
func CanonicalRedirectTarget(req *http.Request, canonical string) string {
43
-
parts := strings.SplitN(strings.TrimPrefix(req.URL.Path, "/"), "/", 3)
43
+
parts := strings.SplitN(strings.TrimPrefix(req.URL.EscapedPath(), "/"), "/", 3)
44
44
target := "/" + canonical
45
45
if len(parts) == 3 {
46
46
target += "/" + parts[2]
+23
appview/reporesolver/resolver_test.go
+23
appview/reporesolver/resolver_test.go
···
51
51
}
52
52
}
53
53
54
+
func TestCanonicalRedirectTargetKeepsTheTailEscaped(t *testing.T) {
55
+
cases := []struct {
56
+
name string
57
+
path string
58
+
want string
59
+
}{
60
+
{"plain tail", "/boltless.dev/limpet/tree/main", "/akshay.dev/anemone/tree/main"},
61
+
{"space in a blob path", "/boltless.dev/limpet/blob/main/a%20b.txt", "/akshay.dev/anemone/blob/main/a%20b.txt"},
62
+
{"escaped slash in a ref", "/boltless.dev/limpet/archive/refs%2Fheads%2Fmain", "/akshay.dev/anemone/archive/refs%2Fheads%2Fmain"},
63
+
{"hash in a filename", "/boltless.dev/limpet/raw/main/c%23.cs", "/akshay.dev/anemone/raw/main/c%23.cs"},
64
+
{"repo root", "/boltless.dev/limpet", "/akshay.dev/anemone"},
65
+
}
66
+
67
+
for _, c := range cases {
68
+
t.Run(c.name, func(t *testing.T) {
69
+
req := httptest.NewRequest("GET", c.path, nil)
70
+
if got := CanonicalRedirectTarget(req, "akshay.dev/anemone"); got != c.want {
71
+
t.Errorf("CanonicalRedirectTarget = %q, want %q", got, c.want)
72
+
}
73
+
})
74
+
}
75
+
}
76
+
54
77
func reqWithChiParams(user, repo string) *http.Request {
55
78
r := httptest.NewRequest("GET", "/", nil)
56
79
rctx := chi.NewRouteContext()
+198
gitutil/archive.go
+198
gitutil/archive.go
···
1
+
package gitutil
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"io"
8
+
"mime"
9
+
"net/http"
10
+
"net/url"
11
+
"os/exec"
12
+
"path"
13
+
"slices"
14
+
"strings"
15
+
"syscall"
16
+
"time"
17
+
"unicode"
18
+
19
+
"github.com/go-git/go-git/v5/plumbing"
20
+
"github.com/samber/lo"
21
+
)
22
+
23
+
type ArchiveFormat string
24
+
25
+
const (
26
+
ArchiveTarGz ArchiveFormat = "tar.gz"
27
+
ArchiveZip ArchiveFormat = "zip"
28
+
)
29
+
30
+
var ArchiveFormats = []ArchiveFormat{ArchiveTarGz, ArchiveZip}
31
+
32
+
func ParseArchiveFormat(raw string) (ArchiveFormat, error) {
33
+
if format := ArchiveFormat(raw); slices.Contains(ArchiveFormats, format) {
34
+
return format, nil
35
+
}
36
+
return "", fmt.Errorf("only tar.gz and zip formats are supported, got %q", raw)
37
+
}
38
+
39
+
func (f ArchiveFormat) String() string { return string(f) }
40
+
41
+
func (f ArchiveFormat) contentType() string {
42
+
return lo.Ternary(f == ArchiveZip, "application/zip", "application/gzip")
43
+
}
44
+
45
+
var pathSeparators = strings.NewReplacer("/", "-", `\`, "-")
46
+
47
+
type RepoName string
48
+
49
+
type Rev string
50
+
51
+
const RevHead Rev = "HEAD"
52
+
53
+
func ParseRev(raw string) (Rev, error) {
54
+
switch {
55
+
case raw == "":
56
+
return "", fmt.Errorf("ref is empty")
57
+
case strings.ContainsFunc(raw, func(c rune) bool { return unicode.IsSpace(c) || unicode.IsControl(c) }):
58
+
return "", fmt.Errorf("ref contains whitespace or a control character: %q", raw)
59
+
case strings.HasPrefix(raw, "-"):
60
+
return "", fmt.Errorf("ref starts with a dash: %q", raw)
61
+
}
62
+
return Rev(raw), nil
63
+
}
64
+
65
+
func RevFromHash(h plumbing.Hash) Rev { return Rev(h.String()) }
66
+
67
+
func (r Rev) String() string { return string(r) }
68
+
69
+
func (r Rev) Slug() string { return pathSeparators.Replace(plumbing.ReferenceName(r).Short()) }
70
+
71
+
func (r Rev) Or(fallback Rev) Rev { return lo.Ternary(r == "", fallback, r) }
72
+
73
+
func (r Rev) OrHash(h plumbing.Hash) Rev { return r.Or(RevFromHash(h)) }
74
+
75
+
type ArchivePrefix string
76
+
77
+
const MaxArchivePrefixLen = 255
78
+
79
+
func ParseArchivePrefix(raw string) (ArchivePrefix, error) {
80
+
switch {
81
+
case len(raw) > MaxArchivePrefixLen:
82
+
return "", fmt.Errorf("prefix is %d bytes, over the %d byte limit", len(raw), MaxArchivePrefixLen)
83
+
case strings.ContainsFunc(raw, unicode.IsControl):
84
+
return "", fmt.Errorf("prefix contains a control character: %q", raw)
85
+
case strings.Contains(raw, `\`):
86
+
return "", fmt.Errorf("prefix contains a backslash: %q", raw)
87
+
}
88
+
trimmed := strings.Trim(raw, "/")
89
+
if trimmed == "" {
90
+
return "", nil
91
+
}
92
+
cleaned := path.Clean(trimmed)
93
+
if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
94
+
return "", fmt.Errorf("prefix escapes the archive root: %q", raw)
95
+
}
96
+
return ArchivePrefix(cleaned), nil
97
+
}
98
+
99
+
func (p ArchivePrefix) String() string { return string(p) }
100
+
101
+
func (p ArchivePrefix) OrDefault(repo RepoName, rev Rev) ArchivePrefix {
102
+
if p == "" {
103
+
return ArchivePrefix(archiveStem(repo, rev))
104
+
}
105
+
return p
106
+
}
107
+
108
+
func archiveStem(repo RepoName, rev Rev) string {
109
+
stem := pathSeparators.Replace(string(repo)) + "-" + rev.Slug()
110
+
if len(stem) <= MaxArchivePrefixLen {
111
+
return stem
112
+
}
113
+
return strings.ToValidUTF8(stem[:MaxArchivePrefixLen], "")
114
+
}
115
+
116
+
type ArchiveParams struct {
117
+
Rev Rev
118
+
Format ArchiveFormat
119
+
Prefix ArchivePrefix
120
+
}
121
+
122
+
func ParseArchiveParams(q url.Values) (ArchiveParams, error) {
123
+
p := ArchiveParams{Format: ArchiveTarGz}
124
+
var err error
125
+
if raw := q.Get("ref"); raw != "" {
126
+
if p.Rev, err = ParseRev(raw); err != nil {
127
+
return ArchiveParams{}, err
128
+
}
129
+
}
130
+
if raw := q.Get("format"); raw != "" {
131
+
if p.Format, err = ParseArchiveFormat(raw); err != nil {
132
+
return ArchiveParams{}, err
133
+
}
134
+
}
135
+
if p.Prefix, err = ParseArchivePrefix(q.Get("prefix")); err != nil {
136
+
return ArchiveParams{}, err
137
+
}
138
+
return p, nil
139
+
}
140
+
141
+
func (p ArchiveParams) WithRev(rev Rev) ArchiveParams {
142
+
p.Rev = rev
143
+
return p
144
+
}
145
+
146
+
func (p ArchiveParams) Query(repo string) url.Values {
147
+
return url.Values{
148
+
"repo": {repo},
149
+
"ref": {p.Rev.String()},
150
+
"format": {p.Format.String()},
151
+
"prefix": {p.Prefix.String()},
152
+
}
153
+
}
154
+
155
+
func (p ArchiveParams) SetHeaders(h http.Header, repo RepoName) {
156
+
h.Set("Content-Type", p.Format.contentType())
157
+
h.Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{
158
+
"filename": archiveStem(repo, p.Rev) + "." + p.Format.String(),
159
+
}))
160
+
h.Set("X-Content-Type-Options", "nosniff")
161
+
}
162
+
163
+
func ImmutableLink(target string) string { return `<` + target + `>; rel="immutable"` }
164
+
165
+
func ParseImmutableLink(header string) (Rev, error) {
166
+
target := strings.TrimSuffix(strings.TrimPrefix(header, "<"), `>; rel="immutable"`)
167
+
parsed, err := url.Parse(target)
168
+
if err != nil {
169
+
return "", err
170
+
}
171
+
return ParseRev(parsed.Query().Get("ref"))
172
+
}
173
+
174
+
const archiveWaitDelay = 10 * time.Second
175
+
176
+
func WriteArchive(ctx context.Context, w io.Writer, repoPath string, rev Rev, format ArchiveFormat, prefix ArchivePrefix) error {
177
+
args := []string{"archive", "--format=" + format.String()}
178
+
if prefix != "" {
179
+
args = append(args, "--prefix="+prefix.String()+"/")
180
+
}
181
+
182
+
cmd := exec.CommandContext(ctx, "git", append(args, "--", rev.String())...)
183
+
cmd.Dir = repoPath
184
+
cmd.Stdout = w
185
+
stderr := new(bytes.Buffer)
186
+
cmd.Stderr = stderr
187
+
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
188
+
cmd.WaitDelay = archiveWaitDelay
189
+
cmd.Cancel = func() error {
190
+
err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
191
+
return lo.Ternary(err == syscall.ESRCH, nil, err)
192
+
}
193
+
194
+
if err := cmd.Run(); err != nil {
195
+
return fmt.Errorf("%w, stderr: %s", err, stderr.String())
196
+
}
197
+
return nil
198
+
}
+187
gitutil/archive_test.go
+187
gitutil/archive_test.go
···
1
+
package gitutil
2
+
3
+
import (
4
+
"archive/zip"
5
+
"bytes"
6
+
"context"
7
+
"mime"
8
+
"net/http"
9
+
"net/url"
10
+
"os"
11
+
"os/exec"
12
+
"path/filepath"
13
+
"strings"
14
+
"testing"
15
+
"unicode/utf8"
16
+
17
+
"github.com/go-git/go-git/v5/plumbing"
18
+
"github.com/samber/lo"
19
+
"github.com/stretchr/testify/assert"
20
+
"github.com/stretchr/testify/require"
21
+
)
22
+
23
+
const testArchiveEndpoint = "https://knot.nel.pet/xrpc/sh.tangled.repo.archive"
24
+
25
+
func TestParseArchiveParams(t *testing.T) {
26
+
cases := []struct {
27
+
name string
28
+
query url.Values
29
+
repo RepoName
30
+
want ArchiveParams
31
+
wantStem ArchivePrefix
32
+
wantFilename string
33
+
wantType string
34
+
wantErr string
35
+
}{
36
+
{"all empty", url.Values{}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""},
37
+
{"tar.gz", url.Values{"format": {"tar.gz"}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""},
38
+
{"zip", url.Values{"format": {"zip"}}, "", ArchiveParams{Format: ArchiveZip}, "", "", "", ""},
39
+
{
40
+
"every param",
41
+
url.Values{"ref": {"refs/tags/v1.0.0"}, "format": {"zip"}, "prefix": {"/kelp/"}},
42
+
"", ArchiveParams{Rev: "refs/tags/v1.0.0", Format: ArchiveZip, Prefix: "kelp"},
43
+
"kelp", "squid-v1.0.0.zip", "application/zip", "",
44
+
},
45
+
46
+
{"branch", url.Values{"ref": {"main"}}, "", ArchiveParams{Rev: "main", Format: ArchiveTarGz}, "squid-main", "squid-main.tar.gz", "application/gzip", ""},
47
+
{"full ref", url.Values{"ref": {"refs/heads/feat/uni"}}, "", ArchiveParams{Rev: "refs/heads/feat/uni", Format: ArchiveTarGz}, "squid-feat-uni", "squid-feat-uni.tar.gz", "application/gzip", ""},
48
+
{"head", url.Values{"ref": {"HEAD"}}, "", ArchiveParams{Rev: "HEAD", Format: ArchiveTarGz}, "squid-HEAD", "", "", ""},
49
+
{"trailing slash", url.Values{"ref": {"refs/heads/main/"}}, "", ArchiveParams{Rev: "refs/heads/main/", Format: ArchiveTarGz}, "squid-main-", "", "", ""},
50
+
{"traversal in a ref", url.Values{"ref": {"../../etc"}}, "", ArchiveParams{Rev: "../../etc", Format: ArchiveTarGz}, "squid-..-..-etc", "", "", ""},
51
+
{"windows separator in a ref", url.Values{"ref": {`feat\uni`}}, "", ArchiveParams{Rev: `feat\uni`, Format: ArchiveTarGz}, "squid-feat-uni", "", "", ""},
52
+
{"slash in a repo name", url.Values{"ref": {"main"}}, "kelp/limpet", ArchiveParams{Rev: "main", Format: ArchiveTarGz}, "kelp-limpet-main", "kelp-limpet-main.tar.gz", "application/gzip", ""},
53
+
{"quote in a repo name", url.Values{"ref": {"main"}}, `squid-a"b`, ArchiveParams{Rev: "main", Format: ArchiveTarGz}, `squid-a"b-main`, `squid-a"b-main.tar.gz`, "application/gzip", ""},
54
+
{"non-ascii repo name", url.Values{"ref": {"main"}, "format": {"zip"}}, "squid-über", ArchiveParams{Rev: "main", Format: ArchiveZip}, "squid-über-main", "squid-über-main.zip", "application/zip", ""},
55
+
56
+
{"bare slash prefix", url.Values{"prefix": {"/"}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""},
57
+
{"did prefix", url.Values{"prefix": {"did:plc:boltless"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "did:plc:boltless"}, "", "", "", ""},
58
+
{"nested prefix", url.Values{"prefix": {"squid/main"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid/main"}, "", "", "", ""},
59
+
{"prefix wrapped in slashes", url.Values{"prefix": {"/squid/main/"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid/main"}, "", "", "", ""},
60
+
{"redundant prefix segments", url.Values{"prefix": {"squid/../limpet"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "limpet"}, "", "", "", ""},
61
+
{"space in a prefix", url.Values{"prefix": {"squid main"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid main"}, "", "", "", ""},
62
+
63
+
{"unsupported format", url.Values{"format": {"tar"}}, "", ArchiveParams{}, "", "", "", "only tar.gz and zip formats are supported"},
64
+
{"space in a ref", url.Values{"ref": {"refs/tags/a b"}}, "", ArchiveParams{}, "", "", "", "ref contains whitespace"},
65
+
{"control character in a ref", url.Values{"ref": {"refs/tags/a\nb"}}, "", ArchiveParams{}, "", "", "", "ref contains whitespace"},
66
+
{"ref that git would read as an option", url.Values{"ref": {"--output=/tmp/evil"}}, "", ArchiveParams{}, "", "", "", "ref starts with a dash"},
67
+
{"prefix escaping the root", url.Values{"prefix": {"../../evil"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"},
68
+
{"prefix escaping after cleaning", url.Values{"prefix": {"squid/../../evil"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"},
69
+
{"bare dot prefix", url.Values{"prefix": {"."}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"},
70
+
{"control character in a prefix", url.Values{"prefix": {"squid\nmain"}}, "", ArchiveParams{}, "", "", "", "prefix contains a control character"},
71
+
{"windows separator in a prefix", url.Values{"prefix": {`..\..\evil`}}, "", ArchiveParams{}, "", "", "", "prefix contains a backslash"},
72
+
{"prefix over the length limit", url.Values{"prefix": {strings.Repeat("a", MaxArchivePrefixLen+1)}}, "", ArchiveParams{}, "", "", "", "over the 255 byte limit"},
73
+
}
74
+
75
+
for _, tc := range cases {
76
+
t.Run(tc.name, func(t *testing.T) {
77
+
got, err := ParseArchiveParams(tc.query)
78
+
rejected := err != nil
79
+
if got != tc.want || rejected != (tc.wantErr != "") || (rejected && !strings.Contains(err.Error(), tc.wantErr)) {
80
+
t.Fatalf("params = %+v with err %v, want %+v and an error mentioning %q", got, err, tc.want, tc.wantErr)
81
+
}
82
+
if rejected {
83
+
return
84
+
}
85
+
86
+
repo := RepoName("squid")
87
+
if tc.repo != "" {
88
+
repo = tc.repo
89
+
}
90
+
if stem := got.Prefix.OrDefault(repo, got.Rev); tc.wantStem != "" && stem != tc.wantStem {
91
+
t.Errorf("default prefix = %q, want %q", stem, tc.wantStem)
92
+
}
93
+
if tc.wantFilename != "" {
94
+
header := http.Header{}
95
+
got.SetHeaders(header, repo)
96
+
mediatype, fields, err := mime.ParseMediaType(header.Get("Content-Disposition"))
97
+
if err != nil || mediatype != "attachment" || fields["filename"] != tc.wantFilename {
98
+
t.Errorf("Content-Disposition = %q (err %v), want an attachment with filename %q", header.Get("Content-Disposition"), err, tc.wantFilename)
99
+
}
100
+
if ct, sniff := header.Get("Content-Type"), header.Get("X-Content-Type-Options"); ct != tc.wantType || sniff != "nosniff" {
101
+
t.Errorf("Content-Type = %q with X-Content-Type-Options %q, want %q and nosniff", ct, sniff, tc.wantType)
102
+
}
103
+
}
104
+
105
+
query := got.Query("did:plc:limpet")
106
+
if back, err := ParseArchiveParams(query); err != nil || back != got {
107
+
t.Errorf("query round trip = %+v (err %v), want %+v", back, err, got)
108
+
}
109
+
back, err := ParseImmutableLink(ImmutableLink(testArchiveEndpoint + "?" + query.Encode()))
110
+
if got.Rev != "" && (err != nil || back != got.Rev) {
111
+
t.Errorf("Link round trip = %q (err %v), want %q", back, err, got.Rev)
112
+
}
113
+
})
114
+
}
115
+
}
116
+
117
+
func TestArchiveFallbacks(t *testing.T) {
118
+
hash := plumbing.NewHash("6f1d3a2b4c5d6e7f8091a2b3c4d5e6f708192a3b")
119
+
if kept, filled := Rev("refs/heads/main").OrHash(hash), Rev("").OrHash(hash); kept != "refs/heads/main" || filled != RevFromHash(hash) {
120
+
t.Errorf("OrHash kept %q and filled %q, want refs/heads/main and the hash %q", kept, filled, hash)
121
+
}
122
+
if got := Rev("").Or(RevHead); got != RevHead {
123
+
t.Errorf("empty rev = %q, want HEAD", got)
124
+
}
125
+
if _, err := ParseImmutableLink(""); err == nil {
126
+
t.Error("ParseImmutableLink must reject an empty header")
127
+
}
128
+
129
+
params := ArchiveParams{Rev: "main", Format: ArchiveZip, Prefix: "kelp"}
130
+
if got := params.WithRev("6f1d3a2"); got != (ArchiveParams{Rev: "6f1d3a2", Format: ArchiveZip, Prefix: "kelp"}) || params.Rev != "main" {
131
+
t.Errorf("WithRev = %+v leaving the receiver at %q, want only the rev replaced", got, params.Rev)
132
+
}
133
+
134
+
long := ArchivePrefix("").OrDefault("squid", Rev("refs/heads/"+strings.Repeat("ü", 400)))
135
+
if _, err := ParseArchivePrefix(long.String()); len(long) > MaxArchivePrefixLen || !utf8.ValidString(long.String()) || err != nil {
136
+
t.Errorf("default prefix is %d bytes %q (err %v), want at most %d bytes ending on a rune boundary", len(long), long, err, MaxArchivePrefixLen)
137
+
}
138
+
}
139
+
140
+
func TestWriteArchive(t *testing.T) {
141
+
repoPath := t.TempDir()
142
+
require.NoError(t, os.WriteFile(filepath.Join(repoPath, "README.md"), []byte("# squid\n"), 0644))
143
+
for _, args := range [][]string{
144
+
{"init", "-q", "-b", "main"},
145
+
{"add", "README.md"},
146
+
{"-c", "user.name=nel", "-c", "user.email=nel@nel.pet", "commit", "-qm", "Initial commit"},
147
+
} {
148
+
cmd := exec.Command("git", args...)
149
+
cmd.Dir = repoPath
150
+
require.NoError(t, cmd.Run(), "git %v", args)
151
+
}
152
+
153
+
canceled, cancel := context.WithCancel(context.Background())
154
+
cancel()
155
+
156
+
cases := []struct {
157
+
name string
158
+
ctx context.Context
159
+
prefix ArchivePrefix
160
+
want []string
161
+
}{
162
+
{
163
+
"prefix on every entry",
164
+
context.Background(),
165
+
ArchivePrefix("").OrDefault("squid", "refs/heads/feat/uni"),
166
+
[]string{"squid-feat-uni/", "squid-feat-uni/README.md"},
167
+
},
168
+
{"empty prefix", context.Background(), "", []string{"README.md"}},
169
+
{"canceled context", canceled, "squid-main", nil},
170
+
}
171
+
172
+
for _, tc := range cases {
173
+
t.Run(tc.name, func(t *testing.T) {
174
+
var body bytes.Buffer
175
+
err := WriteArchive(tc.ctx, &body, repoPath, RevHead, ArchiveZip, tc.prefix)
176
+
if tc.want == nil {
177
+
assert.Error(t, err)
178
+
return
179
+
}
180
+
require.NoError(t, err)
181
+
182
+
entries, err := zip.NewReader(bytes.NewReader(body.Bytes()), int64(body.Len()))
183
+
require.NoError(t, err)
184
+
assert.Equal(t, tc.want, lo.Map(entries.File, func(f *zip.File, _ int) string { return f.Name }))
185
+
})
186
+
}
187
+
}
+34
-100
knotmirror/xrpc/git_get_archive.go
+34
-100
knotmirror/xrpc/git_get_archive.go
···
1
1
package xrpc
2
2
3
3
import (
4
-
"bytes"
5
-
"context"
6
4
"fmt"
7
-
"io"
8
5
"net/http"
9
-
"net/url"
10
-
"os/exec"
11
-
"strings"
12
6
13
7
"github.com/bluesky-social/indigo/atproto/atclient"
14
8
"github.com/bluesky-social/indigo/atproto/syntax"
15
-
"github.com/go-git/go-git/v5/plumbing"
16
9
"tangled.org/core/api/tangled"
10
+
"tangled.org/core/gitutil"
17
11
"tangled.org/core/knotmirror/db"
18
12
"tangled.org/core/knotmirror/xrpc/gitea"
19
13
)
20
14
21
15
func (x *Xrpc) GetArchive(w http.ResponseWriter, r *http.Request) {
22
-
var (
23
-
repoQuery = r.URL.Query().Get("repo")
24
-
ref = r.URL.Query().Get("ref")
25
-
format = r.URL.Query().Get("format")
26
-
prefix = r.URL.Query().Get("prefix")
27
-
)
16
+
invalid := func(err error) {
17
+
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "InvalidRequest", Message: err.Error()})
18
+
}
28
19
20
+
repoQuery := r.URL.Query().Get("repo")
29
21
repo, err := syntax.ParseDID(repoQuery)
30
22
if err != nil {
31
-
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)})
23
+
invalid(fmt.Errorf("repo parameter invalid: %s", repoQuery))
32
24
return
33
25
}
34
26
35
-
if format == "" {
36
-
format = "tar.gz"
37
-
}
38
-
if format != "tar.gz" && format != "zip" {
39
-
writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "only tar.gz and zip formats are supported"})
27
+
params, err := gitutil.ParseArchiveParams(r.URL.Query())
28
+
if err != nil {
29
+
invalid(err)
40
30
return
41
31
}
42
32
43
-
l := x.logger.With("repo", repo, "ref", ref, "format", format, "prefix", prefix)
33
+
l := x.logger.With("repo", repo, "ref", params.Rev, "format", params.Format, "prefix", params.Prefix)
44
34
l.Debug("request")
45
35
46
36
ctx := r.Context()
47
-
48
-
repoPath, err := x.makeRepoPath(ctx, repo)
49
-
if err != nil {
37
+
proxy := func(err error, message string) {
50
38
l.Warn("local mirror failed, trying proxy", "err", err)
51
-
if x.proxyToKnot(w, r, repo) {
52
-
return
39
+
if !x.proxyToKnot(w, r, repo) {
40
+
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: message})
53
41
}
54
-
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to resolve repo"})
55
-
return
56
42
}
57
43
58
-
rev := ref
59
-
if rev == "" {
60
-
rev = "HEAD"
61
-
}
62
-
commit, err := gitea.GetCommit(ctx, repoPath, rev)
44
+
repoPath, err := x.makeRepoPath(ctx, repo)
63
45
if err != nil {
64
-
l.Warn("local mirror failed, trying proxy", "err", err)
65
-
if x.proxyToKnot(w, r, repo) {
66
-
return
67
-
}
68
-
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to resolve ref"})
46
+
proxy(err, "failed to resolve repo")
69
47
return
70
48
}
71
49
72
-
repoName, err := func() (string, error) {
73
-
r, err := db.GetRepoByRepoDid(ctx, x.db, repo)
74
-
if err != nil {
75
-
return "", err
76
-
}
77
-
if r == nil {
78
-
return "", fmt.Errorf("repo not found: %s", repo)
79
-
}
80
-
return r.Name, nil
81
-
}()
50
+
commit, err := gitea.GetCommit(ctx, repoPath, params.Rev.Or(gitutil.RevHead).String())
82
51
if err != nil {
83
-
l.Warn("local mirror failed, trying proxy", "err", err)
84
-
if x.proxyToKnot(w, r, repo) {
85
-
return
86
-
}
87
-
writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to retrieve repo name"})
52
+
proxy(err, "failed to resolve ref")
88
53
return
89
54
}
90
55
91
-
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
92
-
if safeRefFilename == "" {
93
-
safeRefFilename = commit.Hash.String()
56
+
mirrored, err := db.GetRepoByRepoDid(ctx, x.db, repo)
57
+
if err == nil && mirrored == nil {
58
+
err = fmt.Errorf("repo not found: %s", repo)
94
59
}
95
-
immutableLink := func() string {
96
-
params := url.Values{}
97
-
params.Set("repo", repo.String())
98
-
params.Set("ref", commit.Hash.String())
99
-
params.Set("format", format)
100
-
params.Set("prefix", prefix)
101
-
return fmt.Sprintf("%s/xrpc/%s?%s", x.cfg.BaseUrl(), tangled.GitTempGetArchiveNSID, params.Encode())
102
-
}()
103
-
104
-
var archivePrefix string
105
-
if prefix != "" {
106
-
archivePrefix = prefix
107
-
} else {
108
-
archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename)
60
+
if err != nil {
61
+
proxy(err, "failed to retrieve repo name")
62
+
return
109
63
}
110
64
111
-
filename := fmt.Sprintf("%s-%s.%s", repoName, safeRefFilename, format)
112
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
113
-
w.Header().Set("Content-Type", archiveContentType(format))
114
-
w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink))
65
+
name := gitutil.RepoName(mirrored.Name)
66
+
resolvedRev := gitutil.RevFromHash(commit.Hash)
67
+
params.Rev = params.Rev.OrHash(commit.Hash)
68
+
params.Prefix = params.Prefix.OrDefault(name, params.Rev)
69
+
70
+
params.SetHeaders(w.Header(), name)
71
+
w.Header().Set("Link", gitutil.ImmutableLink(fmt.Sprintf("%s/xrpc/%s?%s",
72
+
x.cfg.BaseUrl(), tangled.GitTempGetArchiveNSID, params.WithRev(resolvedRev).Query(repo.String()).Encode(),
73
+
)))
115
74
116
-
if err := writeLocalArchive(ctx, w, repoPath, commit.Hash.String(), format, archivePrefix); err != nil {
117
-
l.Error("writing archive", "err", err.Error(), "format", format)
75
+
if err := gitutil.WriteArchive(ctx, w, repoPath, resolvedRev, params.Format, params.Prefix); err != nil {
76
+
l.Error("writing archive", "err", err.Error(), "format", params.Format)
118
77
w.WriteHeader(http.StatusInternalServerError)
119
78
}
120
79
}
121
-
122
-
func archiveContentType(format string) string {
123
-
if format == "zip" {
124
-
return "application/zip"
125
-
}
126
-
return "application/gzip"
127
-
}
128
-
129
-
func writeLocalArchive(ctx context.Context, w io.Writer, repoPath, rev, format, prefix string) error {
130
-
args := []string{"-C", repoPath, "archive", "--format=" + format}
131
-
if prefix != "" {
132
-
args = append(args, "--prefix="+strings.TrimRight(prefix, "/")+"/")
133
-
}
134
-
args = append(args, rev)
135
-
136
-
cmd := exec.CommandContext(ctx, "git", args...)
137
-
cmd.Stdout = w
138
-
stderr := new(bytes.Buffer)
139
-
cmd.Stderr = stderr
140
-
141
-
if err := cmd.Run(); err != nil {
142
-
return fmt.Errorf("%w, stderr: %s", err, stderr.String())
143
-
}
144
-
return nil
145
-
}
+26
knotmirror/xrpc/git_get_archive_test.go
+26
knotmirror/xrpc/git_get_archive_test.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"net/http"
5
+
"net/http/httptest"
6
+
"strings"
7
+
"testing"
8
+
)
9
+
10
+
func TestGetArchiveRejectsBadParams(t *testing.T) {
11
+
for query, wantMessage := range map[string]string{
12
+
"": "repo parameter invalid",
13
+
"repo=oyster.cafe%2Fsquid": "repo parameter invalid",
14
+
"repo=did:plc:boltless&format=tar.xz": "only tar.gz and zip formats are supported",
15
+
"repo=did:plc:boltless&ref=--output=/x": "ref starts with a dash",
16
+
"repo=did:plc:boltless&prefix=../../evil": "prefix escapes the archive root",
17
+
} {
18
+
rec := httptest.NewRecorder()
19
+
(&Xrpc{}).GetArchive(rec, httptest.NewRequest(http.MethodGet, "/xrpc/sh.tangled.git.temp.getArchive?"+query, nil))
20
+
21
+
body := rec.Body.String()
22
+
if rec.Code != http.StatusBadRequest || !strings.Contains(body, "InvalidRequest") || !strings.Contains(body, wantMessage) {
23
+
t.Errorf("%s: status %d with body %s, want 400 InvalidRequest mentioning %q", query, rec.Code, body, wantMessage)
24
+
}
25
+
}
26
+
}
-23
knotserver/git/cmd.go
-23
knotserver/git/cmd.go
···
1
1
package git
2
2
3
3
import (
4
-
"bytes"
5
4
"fmt"
6
-
"io"
7
5
"os/exec"
8
-
"strings"
9
6
"syscall"
10
7
)
11
8
···
61
58
func (g *GitRepo) mergeBase(extraArgs ...string) ([]byte, error) {
62
59
return g.runGitCmd("merge-base", extraArgs...)
63
60
}
64
-
65
-
func (g *GitRepo) WriteArchive(w io.Writer, format string, prefix string) error {
66
-
args := []string{"archive", "--format=" + format}
67
-
if prefix != "" {
68
-
args = append(args, "--prefix="+strings.TrimRight(prefix, "/")+"/")
69
-
}
70
-
args = append(args, g.h.String())
71
-
72
-
cmd := exec.Command("git", args...)
73
-
cmd.Dir = g.path
74
-
cmd.Stdout = w
75
-
stderr := new(bytes.Buffer)
76
-
cmd.Stderr = stderr
77
-
78
-
if err := cmd.Run(); err != nil {
79
-
return fmt.Errorf("%w, stderr: %s", err, stderr.String())
80
-
}
81
-
82
-
return nil
83
-
}
+22
-73
knotserver/xrpc/repo_archive.go
+22
-73
knotserver/xrpc/repo_archive.go
···
3
3
import (
4
4
"fmt"
5
5
"net/http"
6
-
"net/url"
7
-
"strings"
8
-
9
-
"github.com/go-git/go-git/v5/plumbing"
10
6
11
7
"tangled.org/core/api/tangled"
8
+
"tangled.org/core/gitutil"
12
9
"tangled.org/core/knotserver/git"
13
10
xrpcerr "tangled.org/core/xrpc/errors"
14
11
)
15
12
16
13
func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) {
17
-
repo := r.URL.Query().Get("repo")
18
-
repoPath, err := x.parseRepoParam(repo)
14
+
params, err := gitutil.ParseArchiveParams(r.URL.Query())
19
15
if err != nil {
20
-
writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest)
21
-
return
22
-
}
23
-
24
-
ref := r.URL.Query().Get("ref")
25
-
// ref can be empty (git.Open handles this)
26
-
27
-
format := r.URL.Query().Get("format")
28
-
if format == "" {
29
-
format = "tar.gz" // default
30
-
}
31
-
32
-
prefix := r.URL.Query().Get("prefix")
33
-
34
-
if format != "tar.gz" && format != "zip" {
35
16
writeError(w, xrpcerr.NewXrpcError(
36
17
xrpcerr.WithTag("InvalidRequest"),
37
-
xrpcerr.WithMessage("only tar.gz and zip formats are supported"),
18
+
xrpcerr.WithMessage(err.Error()),
38
19
), http.StatusBadRequest)
39
20
return
40
21
}
41
22
42
-
gr, err := git.Open(repoPath, ref)
23
+
repo := r.URL.Query().Get("repo")
24
+
resolved, err := x.resolveRepo(repo)
43
25
if err != nil {
44
-
writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound)
26
+
writeError(w, err.(xrpcerr.XrpcError), http.StatusBadRequest)
45
27
return
46
28
}
47
29
48
-
repoParts := strings.Split(repo, "/")
49
-
repoName := repoParts[len(repoParts)-1]
50
-
51
-
immutableLink, err := x.buildImmutableLink(repo, format, gr.Hash().String(), prefix)
30
+
// ref can be empty (git.Open handles this)
31
+
gr, err := git.Open(resolved.path, params.Rev.String())
52
32
if err != nil {
53
-
x.Logger.Error(
54
-
"failed to build immutable link",
55
-
"err", err.Error(),
56
-
"repo", repo,
57
-
"format", format,
58
-
"ref", gr.Hash().String(),
59
-
"prefix", prefix,
60
-
)
33
+
writeError(w, xrpcerr.RefNotFoundError, http.StatusNotFound)
34
+
return
61
35
}
62
36
63
-
safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-")
64
-
65
-
var archivePrefix string
66
-
if prefix != "" {
67
-
archivePrefix = prefix
68
-
} else {
69
-
archivePrefix = fmt.Sprintf("%s-%s", repoName, safeRefFilename)
70
-
}
37
+
hash := gr.Hash()
38
+
params.Rev = params.Rev.OrHash(hash)
39
+
params.Prefix = params.Prefix.OrDefault(resolved.name, params.Rev)
71
40
72
-
filename := fmt.Sprintf("%s-%s.%s", repoName, safeRefFilename, format)
73
-
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
74
-
w.Header().Set("Content-Type", archiveContentType(format))
75
-
w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink))
41
+
params.SetHeaders(w.Header(), resolved.name)
42
+
w.Header().Set("Link", gitutil.ImmutableLink(
43
+
x.archiveURL(repo, params.WithRev(gitutil.RevFromHash(hash))),
44
+
))
76
45
77
-
err = gr.WriteArchive(w, format, archivePrefix)
78
-
if err != nil {
46
+
if err := gitutil.WriteArchive(r.Context(), w, resolved.path, gitutil.RevFromHash(hash), params.Format, params.Prefix); err != nil {
79
47
// once we start writing to the body we can't report error anymore
80
48
// so we are only left with logging the error
81
-
x.Logger.Error("writing archive", "error", err.Error(), "format", format)
82
-
return
49
+
x.Logger.Error("writing archive", "error", err.Error(), "format", params.Format)
83
50
}
84
51
}
85
52
86
-
func archiveContentType(format string) string {
87
-
if format == "zip" {
88
-
return "application/zip"
89
-
}
90
-
return "application/gzip"
91
-
}
92
-
93
-
func (x *Xrpc) buildImmutableLink(repo string, format string, ref string, prefix string) (string, error) {
53
+
func (x *Xrpc) archiveURL(repo string, params gitutil.ArchiveParams) string {
94
54
scheme := "https"
95
55
if x.Config.Server.Dev {
96
56
scheme = "http"
97
57
}
98
-
99
-
u, err := url.Parse(scheme + "://" + x.Config.Server.Hostname + "/xrpc/" + tangled.RepoArchiveNSID)
100
-
if err != nil {
101
-
return "", err
102
-
}
103
-
104
-
params := url.Values{}
105
-
params.Set("repo", repo)
106
-
params.Set("format", format)
107
-
params.Set("ref", ref)
108
-
params.Set("prefix", prefix)
109
-
110
-
return fmt.Sprintf("%s?%s", u.String(), params.Encode()), nil
58
+
return fmt.Sprintf("%s://%s/xrpc/%s?%s",
59
+
scheme, x.Config.Server.Hostname, tangled.RepoArchiveNSID, params.Query(repo).Encode())
111
60
}
+25
knotserver/xrpc/repo_archive_test.go
+25
knotserver/xrpc/repo_archive_test.go
···
1
+
package xrpc
2
+
3
+
import (
4
+
"net/http"
5
+
"net/http/httptest"
6
+
"strings"
7
+
"testing"
8
+
)
9
+
10
+
func TestRepoArchiveChecksParamsBeforeResolvingTheRepo(t *testing.T) {
11
+
for query, wantMessage := range map[string]string{
12
+
"format=tar.xz": "only tar.gz and zip formats are supported",
13
+
"ref=--output=/tmp/evil": "ref starts with a dash",
14
+
"prefix=../../evil": "prefix escapes the archive root",
15
+
"ref=refs/heads/main&format=zip": "repo parameter",
16
+
} {
17
+
rec := httptest.NewRecorder()
18
+
(&Xrpc{}).RepoArchive(rec, httptest.NewRequest(http.MethodGet, "/xrpc/sh.tangled.repo.archive?"+query, nil))
19
+
20
+
body := rec.Body.String()
21
+
if rec.Code != http.StatusBadRequest || !strings.Contains(body, "InvalidRequest") || !strings.Contains(body, wantMessage) {
22
+
t.Errorf("%s: status %d with body %s, want 400 InvalidRequest mentioning %q", query, rec.Code, body, wantMessage)
23
+
}
24
+
}
25
+
}
+5
knotserver/xrpc/repo_get_default_branch.go
+5
knotserver/xrpc/repo_get_default_branch.go
+19
-8
knotserver/xrpc/xrpc.go
+19
-8
knotserver/xrpc/xrpc.go
···
13
13
securejoin "github.com/cyphar/filepath-securejoin"
14
14
"github.com/go-chi/chi/v5"
15
15
"tangled.org/core/api/tangled"
16
+
"tangled.org/core/gitutil"
16
17
"tangled.org/core/idresolver"
17
18
"tangled.org/core/knotserver/config"
18
19
"tangled.org/core/knotserver/db"
···
97
98
return r
98
99
}
99
100
101
+
type resolvedRepo struct {
102
+
path string
103
+
name gitutil.RepoName
104
+
}
105
+
100
106
func (x *Xrpc) parseRepoParam(repo string) (string, error) {
107
+
resolved, err := x.resolveRepo(repo)
108
+
return resolved.path, err
109
+
}
110
+
111
+
func (x *Xrpc) resolveRepo(repo string) (resolvedRepo, error) {
101
112
if repo == "" || !strings.HasPrefix(repo, "did:") {
102
-
return "", xrpcerr.NewXrpcError(
113
+
return resolvedRepo{}, xrpcerr.NewXrpcError(
103
114
xrpcerr.WithTag("InvalidRequest"),
104
115
xrpcerr.WithMessage("missing or invalid repo parameter, expected a repo DID"),
105
116
)
106
117
}
107
118
108
119
if !strings.Contains(repo, "/") {
109
-
repoPath, _, _, err := x.Db.ResolveRepoDIDOnDisk(x.Config.Repo.ScanPath, repo)
120
+
repoPath, _, repoName, err := x.Db.ResolveRepoDIDOnDisk(x.Config.Repo.ScanPath, repo)
110
121
if err != nil {
111
-
return "", xrpcerr.RepoNotFoundError
122
+
return resolvedRepo{}, xrpcerr.RepoNotFoundError
112
123
}
113
-
return repoPath, nil
124
+
return resolvedRepo{path: repoPath, name: gitutil.RepoName(repoName)}, nil
114
125
}
115
126
116
127
parts := strings.SplitN(repo, "/", 2)
···
120
131
if err == nil {
121
132
repoPath, _, _, resolveErr := x.Db.ResolveRepoDIDOnDisk(x.Config.Repo.ScanPath, repoDid)
122
133
if resolveErr == nil {
123
-
return repoPath, nil
134
+
return resolvedRepo{path: repoPath, name: gitutil.RepoName(repoName)}, nil
124
135
}
125
136
}
126
137
127
138
repoPath, joinErr := securejoin.SecureJoin(x.Config.Repo.ScanPath, filepath.Join(ownerDid, repoName))
128
139
if joinErr != nil {
129
-
return "", xrpcerr.RepoNotFoundError
140
+
return resolvedRepo{}, xrpcerr.RepoNotFoundError
130
141
}
131
142
if _, statErr := os.Stat(repoPath); statErr != nil {
132
-
return "", xrpcerr.RepoNotFoundError
143
+
return resolvedRepo{}, xrpcerr.RepoNotFoundError
133
144
}
134
-
return repoPath, nil
145
+
return resolvedRepo{path: repoPath, name: gitutil.RepoName(repoName)}, nil
135
146
}
136
147
137
148
func (x *Xrpc) resolveRepoDID(repo *string, ownerDid, name string) (repoident.RepoDid, string, error) {