alice.pds.demo.boltless.dev / core
This repository has no description
0
Fork 0

Configure Feed

Select the types of activity you want to include in your feed.

appview: basic repository migrator

Signed-off-by: Seongmin Lee <git@boltless.me>

author
Seongmin Lee date (Jul 23, 2026, 8:56 PM +0900) commit b0b755ea parent 6e92c96c change-id wnvprzqs
+1098
+298
appview/accountmigration/accountmigration.go
··· 1 + package accountmigration 2 + 3 + import ( 4 + "encoding/json" 5 + "fmt" 6 + "log/slog" 7 + "net/http" 8 + "net/url" 9 + "slices" 10 + "strconv" 11 + "strings" 12 + 13 + "github.com/bluesky-social/indigo/atproto/syntax" 14 + "github.com/go-chi/chi/v5" 15 + "tangled.org/core/appview/db" 16 + "tangled.org/core/appview/middleware" 17 + "tangled.org/core/appview/models" 18 + "tangled.org/core/appview/oauth" 19 + "tangled.org/core/appview/pages" 20 + "tangled.org/core/rbac" 21 + "tangled.org/core/sets" 22 + ) 23 + 24 + type AccountMigration struct { 25 + logger *slog.Logger 26 + oauth *oauth.OAuth 27 + pages *pages.Pages 28 + db *db.DB 29 + enforcer *rbac.Enforcer 30 + } 31 + 32 + func New( 33 + logger *slog.Logger, 34 + o *oauth.OAuth, 35 + p *pages.Pages, 36 + d *db.DB, 37 + enforcer *rbac.Enforcer, 38 + ) *AccountMigration { 39 + return &AccountMigration{ 40 + logger: logger, 41 + oauth: o, 42 + pages: p, 43 + db: d, 44 + enforcer: enforcer, 45 + } 46 + } 47 + 48 + func (s *AccountMigration) Router() http.Handler { 49 + r := chi.NewRouter() 50 + r.Use(middleware.AuthMiddleware(s.oauth)) 51 + 52 + r.Get("/", s.migratePage) 53 + r.Post("/listGitHubRepos", s.listGitHubRepos) 54 + r.Post("/start", s.startMigration) 55 + r.Get("/progress/rows", s.progressRows) 56 + return r 57 + } 58 + 59 + func (s *AccountMigration) migratePage(w http.ResponseWriter, r *http.Request) { 60 + user := s.oauth.GetMultiAccountUser(r) 61 + 62 + service := r.URL.Query().Get("service") 63 + switch service { 64 + case "github": 65 + knots, err := s.enforcer.GetKnotsForUser(user.Did) 66 + if err != nil { 67 + s.logger.Error("knots lookup failed", "did", user.Did, "err", err) 68 + knots = nil 69 + } 70 + s.pages.AccountMigrateFromGitHub(w, pages.AccountMigrateFromGitHubParams{ 71 + BaseParams: pages.BaseParamsFromContext(r.Context()), 72 + Knots: knots, 73 + }) 74 + default: 75 + s.pages.AccountMigrate(w, pages.AccountMigrateParams{ 76 + BaseParams: pages.BaseParamsFromContext(r.Context()), 77 + }) 78 + } 79 + } 80 + 81 + type githubUserRepo struct { 82 + Name string `json:"name"` 83 + CloneUrl string `json:"clone_url"` 84 + DefaultBranch string `json:"default_branch"` 85 + Description string `json:"description"` 86 + Homepage string `json:"homepage"` 87 + Topics []string `json:"topics"` 88 + Fork bool `json:"fork"` 89 + Private bool `json:"private"` 90 + Archived bool `json:"archived"` 91 + Disabled bool `json:"disabled"` 92 + } 93 + 94 + func (s *AccountMigration) listGitHubRepos(w http.ResponseWriter, r *http.Request) { 95 + user := s.oauth.GetMultiAccountUser(r) 96 + 97 + username := strings.TrimSpace(r.FormValue("username")) 98 + if username == "" { 99 + s.pages.Notice(w, "migrate-error", "GitHub username is required.") 100 + return 101 + } 102 + 103 + query := url.Values{} 104 + query.Set("sort", "updated") 105 + query.Set("per_page", "80") 106 + 107 + endpoint := fmt.Sprintf("https://api.github.com/users/%s/repos?%s", url.PathEscape(username), query.Encode()) 108 + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, endpoint, nil) 109 + if err != nil { 110 + s.pages.Notice(w, "migrate-error", "Failed to build GitHub request.") 111 + return 112 + } 113 + req.Header.Set("Accept", "application/vnd.github+json") 114 + 115 + resp, err := http.DefaultClient.Do(req) 116 + if err != nil { 117 + s.logger.Error("github list repos failed", "username", username, "err", err) 118 + s.pages.Notice(w, "migrate-error", "Failed to reach GitHub. Try again.") 119 + return 120 + } 121 + defer resp.Body.Close() 122 + 123 + if resp.StatusCode != http.StatusOK { 124 + s.pages.Notice(w, "migrate-error", fmt.Sprintf("GitHub returned %d. Check the username.", resp.StatusCode)) 125 + return 126 + } 127 + 128 + var githubRepos []githubUserRepo 129 + if err := json.NewDecoder(resp.Body).Decode(&githubRepos); err != nil { 130 + s.logger.Error("decode github response failed", "err", err) 131 + s.pages.Notice(w, "migrate-error", "Failed to parse GitHub response.") 132 + return 133 + } 134 + 135 + enqueued, err := db.ListEnqueuedGitRepoNames(r.Context(), s.db, user.Did) 136 + if err != nil { 137 + s.logger.Error("list enqueued names failed", "err", err) 138 + enqueued = map[string]struct{}{} 139 + } 140 + 141 + knots, err := s.enforcer.GetKnotsForUser(user.Did) 142 + if err != nil { 143 + s.logger.Error("knots lookup failed", "did", user.Did, "err", err) 144 + knots = nil 145 + } 146 + 147 + importRepos := make([]pages.RepoImportParams, 0, len(githubRepos)) 148 + for _, ghrepo := range githubRepos { 149 + if ghrepo.Fork || ghrepo.Private || ghrepo.Archived || ghrepo.Disabled { 150 + continue 151 + } 152 + name := strings.ToLower(ghrepo.Name) 153 + _, migrated := enqueued[name] 154 + importRepos = append(importRepos, pages.RepoImportParams{ 155 + SourceKind: pages.RepoImportSourceGitHub, 156 + CloneUrl: ghrepo.CloneUrl, 157 + Name: name, 158 + Description: ghrepo.Description, 159 + Website: ghrepo.Homepage, 160 + Topics: ghrepo.Topics, 161 + Selected: !migrated, 162 + }) 163 + } 164 + 165 + if err := s.pages.AccountMigrateRepoListFragment(w, pages.AccountMigrateRepoListParams{ 166 + Repos: importRepos, 167 + Knots: knots, 168 + }); err != nil { 169 + s.logger.Error("render repo-list fragment failed", "err", err) 170 + } 171 + } 172 + 173 + func (s *AccountMigration) startMigration(w http.ResponseWriter, r *http.Request) { 174 + user := s.oauth.GetMultiAccountUser(r) 175 + 176 + if err := r.ParseForm(); err != nil { 177 + s.pages.Notice(w, "migrate-error", "Invalid form submission.") 178 + return 179 + } 180 + 181 + knots, err := s.enforcer.GetKnotsForUser(user.Did) 182 + if err != nil { 183 + s.logger.Error("knots lookup failed", "did", user.Did, "err", err) 184 + s.pages.Notice(w, "migrate-error", "Failed to look up your knots.") 185 + return 186 + } 187 + allowed := sets.Collect(slices.Values(knots)) 188 + 189 + sessionId := s.oauth.GetSessIdFromCookie(r) 190 + if sessionId == "" { 191 + s.pages.Notice(w, "migrate-error", "Session expired. Log in again.") 192 + return 193 + } 194 + 195 + count, err := strconv.Atoi(r.FormValue("count")) 196 + if err != nil || count <= 0 { 197 + s.pages.Notice(w, "migrate-error", "Invalid form submission.") 198 + return 199 + } 200 + 201 + s.logger.Debug("migrating repos", "count", count) 202 + 203 + rows := make([]db.GitRepoMigration, 0, count) 204 + seen := sets.New[string]() 205 + for i := range count { 206 + if r.FormValue(fmt.Sprintf("selected_%d", i)) == "" { 207 + s.logger.Warn("can't find selected repo", "i", i) 208 + continue 209 + } 210 + s.logger.Info("found selected repo", "i", i) 211 + 212 + cloneUrl := strings.TrimSpace(r.FormValue(fmt.Sprintf("clone_url_%d", i))) 213 + knot := strings.TrimSpace(r.FormValue(fmt.Sprintf("knot_%d", i))) 214 + name := strings.ToLower(strings.TrimSpace(r.FormValue(fmt.Sprintf("name_%d", i)))) 215 + desc := strings.TrimSpace(r.FormValue(fmt.Sprintf("description_%d", i))) 216 + website := strings.TrimSpace(r.FormValue(fmt.Sprintf("website_%d", i))) 217 + topics := r.FormValue(fmt.Sprintf("topics_%d", i)) 218 + 219 + if cloneUrl == "" || knot == "" || name == "" { 220 + s.pages.Notice(w, "migrate-error", "Each selected row needs a name, clone URL, and knot.") 221 + return 222 + } 223 + if err := models.ValidateRepoName(name); err != nil { 224 + s.pages.Notice(w, "migrate-error", fmt.Sprintf("Row %d: %s", i+1, err.Error())) 225 + return 226 + } 227 + if len([]rune(desc)) > 140 { 228 + s.pages.Notice(w, "migrate-error", fmt.Sprintf("Row %d: description must be 140 characters or fewer.", i+1)) 229 + return 230 + } 231 + if !allowed.Contains(knot) { 232 + s.pages.Notice(w, "migrate-error", fmt.Sprintf("You are not a member of knot %q.", knot)) 233 + return 234 + } 235 + if seen.Contains(name) { 236 + s.pages.Notice(w, "migrate-error", fmt.Sprintf("Duplicate repository name %q in selection.", name)) 237 + return 238 + } 239 + seen.Insert(name) 240 + 241 + rows = append(rows, db.GitRepoMigration{ 242 + OwnerDid: syntax.DID(user.Did), 243 + SourceKind: db.GitRepoMigrationSourceGitHub, 244 + CloneUrl: cloneUrl, 245 + Name: name, 246 + Knot: knot, 247 + Description: desc, 248 + Website: website, 249 + SessionID: sessionId, 250 + Topics: func(s string) []string { 251 + if s == "" { 252 + return nil 253 + } 254 + parts := strings.Split(s, ",") 255 + out := parts[:0] 256 + for _, p := range parts { 257 + p = strings.TrimSpace(p) 258 + if p != "" { 259 + out = append(out, p) 260 + } 261 + } 262 + return out 263 + }(topics), 264 + }) 265 + } 266 + 267 + if len(rows) == 0 { 268 + s.pages.Notice(w, "migrate-error", "Pick at least one repo.") 269 + return 270 + } 271 + 272 + s.logger.Info("inserting migrations", "len", len(rows)) 273 + 274 + if err := db.InsertGitRepoMigrations(r.Context(), s.db, rows); err != nil { 275 + s.logger.Error("insert migrations failed", "err", err) 276 + s.pages.Notice(w, "migrate-error", "Failed to enqueue migrations.") 277 + return 278 + } 279 + 280 + w.Header().Set("HX-Redirect", "/settings/migration") 281 + http.Redirect(w, r, "/settings/migration", http.StatusSeeOther) 282 + } 283 + 284 + func (s *AccountMigration) progressRows(w http.ResponseWriter, r *http.Request) { 285 + user := s.oauth.GetMultiAccountUser(r) 286 + 287 + migrations, err := db.ListGitRepoMigrationsForOwner(r.Context(), s.db, user.Did) 288 + if err != nil { 289 + s.logger.Error("list migrations failed", "did", user.Did, "err", err) 290 + } 291 + 292 + if err := s.pages.AccountMigrateProgressRowsFragment(w, pages.AccountMigrateProgressParams{ 293 + BaseParams: pages.BaseParamsFromContext(r.Context()), 294 + Migrations: migrations, 295 + }); err != nil { 296 + s.logger.Error("render progress rows failed", "err", err) 297 + } 298 + }
+193
appview/accountmigration/worker.go
··· 1 + package accountmigration 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "log/slog" 7 + "sync" 8 + "time" 9 + 10 + comatproto "github.com/bluesky-social/indigo/api/atproto" 11 + "github.com/bluesky-social/indigo/atproto/atclient" 12 + "github.com/bluesky-social/indigo/atproto/syntax" 13 + lexutil "github.com/bluesky-social/indigo/lex/util" 14 + "github.com/bluesky-social/indigo/xrpc" 15 + 16 + "tangled.org/core/api/tangled" 17 + "tangled.org/core/appview/db" 18 + "tangled.org/core/appview/models" 19 + "tangled.org/core/appview/oauth" 20 + "tangled.org/core/xrpc/xrpcclient" 21 + "tangled.org/core/orm" 22 + ) 23 + 24 + const defaultConcurrency = 4 25 + 26 + type Worker struct { 27 + db *db.DB 28 + oauth *oauth.OAuth 29 + dev bool 30 + logger *slog.Logger 31 + concurrency int 32 + 33 + startMu sync.Mutex 34 + started bool 35 + } 36 + 37 + func NewWorker(d *db.DB, o *oauth.OAuth, dev bool, logger *slog.Logger) *Worker { 38 + return &Worker{ 39 + db: d, 40 + oauth: o, 41 + dev: dev, 42 + logger: logger, 43 + concurrency: defaultConcurrency, 44 + } 45 + } 46 + 47 + func (w *Worker) Start(ctx context.Context) { 48 + w.startMu.Lock() 49 + defer w.startMu.Unlock() 50 + if w.started { 51 + return 52 + } 53 + w.started = true 54 + 55 + for i := 0; i < w.concurrency; i++ { 56 + go w.runWorker(ctx, i) 57 + } 58 + } 59 + 60 + func (w *Worker) runWorker(ctx context.Context, id int) { 61 + l := w.logger.With("worker", id) 62 + for { 63 + select { 64 + case <-ctx.Done(): 65 + return 66 + default: 67 + } 68 + 69 + row, ok, err := db.ClaimNextPending(ctx, w.db) 70 + if err != nil { 71 + l.Error("claim failed", "err", err) 72 + time.Sleep(time.Second) 73 + continue 74 + } 75 + if !ok { 76 + time.Sleep(time.Second) 77 + continue 78 + } 79 + l.Info("migrating repo", "row", row) 80 + 81 + w.importRepo(ctx, l, row) 82 + } 83 + } 84 + 85 + func (w *Worker) importRepo(ctx context.Context, l *slog.Logger, row *db.GitRepoMigration) { 86 + l = l.With("id", row.ID, "owner", row.OwnerDid, "name", row.Name, "knot", row.Knot) 87 + 88 + err := w.doImportRepo(ctx, row) 89 + if err == nil { 90 + if err := db.MarkGitRepoMigrationDone(ctx, w.db, row.ID); err != nil { 91 + l.Error("mark done failed", "err", err) 92 + } 93 + return 94 + } 95 + 96 + l.Warn("job failed", "err", err) 97 + 98 + if uerr := db.MarkGitRepoMigrationFailed(ctx, w.db, row.ID, err.Error()); uerr != nil { 99 + l.Error("mark failed failed", "err", uerr) 100 + } 101 + } 102 + 103 + func (w *Worker) doImportRepo(ctx context.Context, row *db.GitRepoMigration) error { 104 + atpClient, err := w.oauth.ClientForDidSession(ctx, row.OwnerDid, row.SessionID) 105 + if err != nil { 106 + return fmt.Errorf("resume session: %w", err) 107 + } 108 + 109 + existing, _ := db.GetRepo(w.db, 110 + orm.FilterEq("did", row.OwnerDid), 111 + orm.FilterEq("rkey", row.Name), 112 + ) 113 + if existing != nil || rkeyOccupied(ctx, atpClient, row.OwnerDid.String(), row.Name) { 114 + return fmt.Errorf("rkey %q already exists", row.Name) 115 + } 116 + 117 + sc, err := w.oauth.ServiceClientForDidSession(ctx, row.OwnerDid, row.SessionID, 118 + oauth.WithService(row.Knot), 119 + oauth.WithLxm(tangled.RepoCreateNSID), 120 + oauth.WithDev(w.dev), 121 + oauth.WithTimeout(10*time.Minute), 122 + ) 123 + if err != nil { 124 + return fmt.Errorf("knot service client: %w", err) 125 + } 126 + 127 + defaultBranch := "main" 128 + createResp, err := tangled.RepoCreate(ctx, sc, &tangled.RepoCreate_Input{ 129 + Rkey: row.Name, 130 + Name: row.Name, 131 + DefaultBranch: &defaultBranch, 132 + Source: &row.CloneUrl, 133 + }) 134 + if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 135 + return fmt.Errorf("knot RepoCreate: %w", err) 136 + } 137 + if err != nil { 138 + return fmt.Errorf("knot RepoCreate: %w", err) 139 + } 140 + var repoDid string 141 + if createResp != nil && createResp.RepoDid != nil { 142 + repoDid = *createResp.RepoDid 143 + } 144 + if repoDid == "" { 145 + return fmt.Errorf("knot returned empty repo DID") 146 + } 147 + 148 + repo := &models.Repo{ 149 + Did: row.OwnerDid.String(), 150 + Name: row.Name, 151 + Knot: row.Knot, 152 + Rkey: row.Name, 153 + Description: row.Description, 154 + Created: time.Now(), 155 + RepoDid: repoDid, 156 + } 157 + record := repo.AsRecord() 158 + 159 + rkey := row.Name 160 + _, err = comatproto.RepoCreateRecord(ctx, atpClient, &comatproto.RepoCreateRecord_Input{ 161 + Collection: tangled.RepoNSID, 162 + Repo: row.OwnerDid.String(), 163 + Rkey: &rkey, 164 + Record: &lexutil.LexiconTypeDecoder{ 165 + Val: &record, 166 + }, 167 + }) 168 + if err != nil { 169 + w.cleanupKnotRepo(ctx, sc, row.OwnerDid, row.Knot, row.Name) 170 + return fmt.Errorf("PDS PutRecord: %w", err) 171 + } 172 + 173 + return nil 174 + } 175 + 176 + func (w *Worker) cleanupKnotRepo(ctx context.Context, sc *xrpc.Client, did syntax.DID, knot, name string) { 177 + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) 178 + defer cancel() 179 + if err := tangled.RepoDelete(cctx, sc, &tangled.RepoDelete_Input{ 180 + Did: did.String(), 181 + Name: name, 182 + Rkey: name, 183 + }); err != nil { 184 + w.logger.Error("cleanup: RepoDelete failed", "knot", knot, "name", name, "err", err) 185 + } 186 + } 187 + 188 + func rkeyOccupied(ctx context.Context, client *atclient.APIClient, did, rkey string) bool { 189 + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) 190 + defer cancel() 191 + resp, err := comatproto.RepoGetRecord(probeCtx, client, "", tangled.RepoNSID, did, rkey) 192 + return err == nil && resp != nil 193 + }
+224
appview/db/account_migration.go
··· 1 + package db 2 + 3 + import ( 4 + "context" 5 + "database/sql" 6 + "errors" 7 + "time" 8 + 9 + "github.com/bluesky-social/indigo/atproto/syntax" 10 + ) 11 + 12 + type GitRepoMigrationStatus string 13 + 14 + const ( 15 + GitRepoMigrationStatusPending GitRepoMigrationStatus = "pending" 16 + GitRepoMigrationStatusRunning GitRepoMigrationStatus = "running" 17 + GitRepoMigrationStatusDone GitRepoMigrationStatus = "done" 18 + GitRepoMigrationStatusFailed GitRepoMigrationStatus = "failed" 19 + ) 20 + 21 + const ( 22 + GitRepoMigrationSourceGitHub = "github" 23 + ) 24 + 25 + type GitRepoMigrations []GitRepoMigration 26 + 27 + func (m GitRepoMigrations) AnyActive() bool { 28 + for _, r := range m { 29 + if r.Status == GitRepoMigrationStatusPending || r.Status == GitRepoMigrationStatusRunning { 30 + return true 31 + } 32 + } 33 + return false 34 + } 35 + 36 + type GitRepoMigration struct { 37 + ID int64 38 + OwnerDid syntax.DID 39 + SourceKind string 40 + CloneUrl string 41 + Name string 42 + Knot string 43 + Description string 44 + Website string 45 + Topics []string 46 + SessionID string 47 + Status GitRepoMigrationStatus 48 + ErrorMsg string 49 + UpdatedAt time.Time 50 + } 51 + 52 + func InsertGitRepoMigrations(ctx context.Context, e *DB, rows []GitRepoMigration) error { 53 + if len(rows) == 0 { 54 + return nil 55 + } 56 + txx, err := e.BeginTx(ctx, nil) 57 + if err != nil { 58 + return err 59 + } 60 + defer txx.Rollback() 61 + 62 + stmt, err := txx.PrepareContext(ctx, ` 63 + insert into gitrepo_migrations 64 + (owner_did, source_kind, clone_url, name, knot, description, session_id, 65 + status, error_msg, updated_at) 66 + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 67 + on conflict(owner_did, name) do update set 68 + source_kind = excluded.source_kind, 69 + clone_url = excluded.clone_url, 70 + knot = excluded.knot, 71 + description = excluded.description, 72 + session_id = excluded.session_id, 73 + status = 'pending', 74 + error_msg = '', 75 + updated_at = excluded.updated_at 76 + `) 77 + if err != nil { 78 + return err 79 + } 80 + defer stmt.Close() 81 + 82 + now := time.Now().UTC().Format(time.RFC3339) 83 + for _, r := range rows { 84 + status := r.Status 85 + if status == "" { 86 + status = GitRepoMigrationStatusPending 87 + } 88 + if _, err := stmt.ExecContext(ctx, 89 + r.OwnerDid, r.SourceKind, r.CloneUrl, r.Name, r.Knot, r.Description, r.SessionID, 90 + status, r.ErrorMsg, now, 91 + ); err != nil { 92 + return err 93 + } 94 + } 95 + return txx.Commit() 96 + } 97 + 98 + func ClaimNextPending(ctx context.Context, e Execer) (*GitRepoMigration, bool, error) { 99 + var ( 100 + m GitRepoMigration 101 + updatedAt string 102 + ) 103 + err := e.QueryRowContext(ctx, ` 104 + update gitrepo_migrations 105 + set status = 'running', 106 + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') 107 + where id = ( 108 + select id from gitrepo_migrations gm 109 + where status = 'pending' 110 + and not exists ( 111 + select 1 from gitrepo_migrations gm2 112 + where gm2.owner_did = gm.owner_did 113 + and gm2.status = 'running' 114 + ) 115 + order by id 116 + limit 1 117 + ) 118 + returning id, owner_did, source_kind, clone_url, name, knot, 119 + description, session_id, status, error_msg, updated_at 120 + `).Scan( 121 + &m.ID, &m.OwnerDid, &m.SourceKind, &m.CloneUrl, &m.Name, &m.Knot, 122 + &m.Description, &m.SessionID, &m.Status, &m.ErrorMsg, 123 + &updatedAt, 124 + ) 125 + if errors.Is(err, sql.ErrNoRows) { 126 + return nil, false, nil 127 + } 128 + if err != nil { 129 + return nil, false, err 130 + } 131 + if t, err := time.Parse(time.RFC3339, updatedAt); err == nil { 132 + m.UpdatedAt = t 133 + } 134 + return &m, true, nil 135 + } 136 + 137 + func MarkGitRepoMigrationDone(ctx context.Context, e Execer, id int64) error { 138 + _, err := e.ExecContext(ctx, ` 139 + update gitrepo_migrations 140 + set status = 'done', 141 + error_msg = '', 142 + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') 143 + where id = ? 144 + `, id) 145 + return err 146 + } 147 + 148 + func MarkGitRepoMigrationFailed(ctx context.Context, e Execer, id int64, errMsg string) error { 149 + _, err := e.ExecContext(ctx, ` 150 + update gitrepo_migrations 151 + set status = 'failed', 152 + error_msg = ?, 153 + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') 154 + where id = ? 155 + `, errMsg, id) 156 + return err 157 + } 158 + 159 + func ListGitRepoMigrationsForOwner(ctx context.Context, e Execer, owner string) (GitRepoMigrations, error) { 160 + rows, err := e.QueryContext(ctx, ` 161 + select id, owner_did, source_kind, clone_url, name, knot, 162 + description, session_id, status, coalesce(error_msg, ''), 163 + updated_at 164 + from gitrepo_migrations 165 + where owner_did = ? 166 + order by id desc 167 + `, owner) 168 + if err != nil { 169 + return nil, err 170 + } 171 + defer rows.Close() 172 + 173 + var out GitRepoMigrations 174 + for rows.Next() { 175 + var ( 176 + m GitRepoMigration 177 + updatedAt string 178 + ) 179 + if err := rows.Scan( 180 + &m.ID, &m.OwnerDid, &m.SourceKind, &m.CloneUrl, &m.Name, &m.Knot, 181 + &m.Description, &m.SessionID, &m.Status, &m.ErrorMsg, 182 + &updatedAt, 183 + ); err != nil { 184 + return nil, err 185 + } 186 + if t, err := time.Parse(time.RFC3339, updatedAt); err == nil { 187 + m.UpdatedAt = t 188 + } 189 + out = append(out, m) 190 + } 191 + return out, rows.Err() 192 + } 193 + 194 + func ListEnqueuedGitRepoNames(ctx context.Context, e Execer, owner string) (map[string]struct{}, error) { 195 + rows, err := e.QueryContext(ctx, ` 196 + select name from gitrepo_migrations 197 + where owner_did = ? 198 + and status in ('pending', 'running', 'done') 199 + `, owner) 200 + if err != nil { 201 + return nil, err 202 + } 203 + defer rows.Close() 204 + 205 + out := map[string]struct{}{} 206 + for rows.Next() { 207 + var n string 208 + if err := rows.Scan(&n); err != nil { 209 + return nil, err 210 + } 211 + out[n] = struct{}{} 212 + } 213 + return out, rows.Err() 214 + } 215 + 216 + func ReapStaleRunningGitRepoMigrations(ctx context.Context, e Execer) error { 217 + _, err := e.ExecContext(ctx, ` 218 + update gitrepo_migrations 219 + set status = 'pending', 220 + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') 221 + where status = 'running' 222 + `) 223 + return err 224 + }
+24
appview/db/db.go
··· 2487 2487 return err 2488 2488 }) 2489 2489 2490 + orm.RunMigration(conn, logger, "add-gitrepo_migrations", func(tx *sql.Tx) error { 2491 + _, err := tx.Exec(` 2492 + create table gitrepo_migrations ( 2493 + id integer primary key autoincrement, 2494 + owner_did text not null, 2495 + source_kind text not null, 2496 + clone_url text not null, 2497 + name text not null, 2498 + knot text not null, 2499 + description text not null, 2500 + session_id text not null default '', 2501 + 2502 + status text not null default 'pending', 2503 + error_msg text not null default '', 2504 + updated_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), 2505 + 2506 + unique(owner_did, name) 2507 + ); 2508 + create index if not exists idx_gitrepo_migrations_status 2509 + on gitrepo_migrations(status); 2510 + `) 2511 + return err 2512 + }) 2513 + 2490 2514 return &DB{ 2491 2515 db, 2492 2516 logger,
+40
appview/oauth/oauth.go
··· 350 350 return session.APIClient(), nil 351 351 } 352 352 353 + func (o *OAuth) ClientForDidSession(ctx context.Context, did syntax.DID, sessionId string) (*atclient.APIClient, error) { 354 + sess, err := o.resumeSession(ctx, did, sessionId) 355 + if err != nil { 356 + return nil, fmt.Errorf("error getting session: %w", err) 357 + } 358 + return sess.APIClient(), nil 359 + } 360 + 353 361 // this is a higher level abstraction on ServerGetServiceAuth 354 362 type ServiceClientOpts struct { 355 363 service string ··· 458 466 WithDev(noTLS), 459 467 WithTimeout(time.Second*30), 460 468 ) 469 + } 470 + 471 + func (o *OAuth) ServiceClientForDidSession(ctx context.Context, did syntax.DID, sessionId string, os ...ServiceClientOpt) (*xrpc.Client, error) { 472 + opts := DefaultServiceClientOpts() 473 + for _, o := range os { 474 + o(&opts) 475 + } 476 + 477 + client, err := o.ClientForDidSession(ctx, did, sessionId) 478 + if err != nil { 479 + return nil, err 480 + } 481 + 482 + sixty := time.Now().Unix() + 60 483 + if opts.exp < sixty { 484 + opts.exp = sixty 485 + } 486 + 487 + resp, err := comatproto.ServerGetServiceAuth(ctx, client, opts.Audience(), opts.exp, opts.lxm) 488 + if err != nil { 489 + return nil, err 490 + } 491 + 492 + return &xrpc.Client{ 493 + Auth: &xrpc.AuthInfo{ 494 + AccessJwt: resp.Token, 495 + }, 496 + Host: opts.Host(), 497 + Client: &http.Client{ 498 + Timeout: opts.timeout, 499 + }, 500 + }, nil 461 501 } 462 502 463 503 func (o *OAuth) StartElevatedAuthFlow(ctx context.Context, w http.ResponseWriter, r *http.Request, did string, extraScopes []string, returnURL string) (string, error) {
+1
appview/pages/funcmap.go
··· 599 599 {"Name": "knots", "Label": "Knots", "Icon": "volleyball"}, 600 600 {"Name": "spindles", "Label": "Spindles", "Icon": "spool"}, 601 601 {"Name": "sites", "Label": "Sites", "Icon": "globe"}, 602 + {"Name": "migration", "Label": "Migration", "Icon": "book-down"}, 602 603 }, 603 604 "RepoSettingsTabs": []tab{ 604 605 {"Name": "general", "Label": "General", "Icon": "sliders-horizontal"},
+63
appview/pages/pages.go
··· 2047 2047 return p.executePlain("fragments/comment/replyPlaceholder", w, params) 2048 2048 } 2049 2049 2050 + type AccountMigrateParams struct { 2051 + BaseParams 2052 + } 2053 + 2054 + func (p *Pages) AccountMigrate(w io.Writer, params AccountMigrateParams) error { 2055 + return p.execute("account/migrate/index", w, params) 2056 + } 2057 + 2058 + type AccountMigrateFromGitHubParams struct { 2059 + BaseParams 2060 + Knots []string 2061 + } 2062 + 2063 + type RepoImportSource string 2064 + 2065 + const ( 2066 + RepoImportSourceGitHub RepoImportSource = "github" 2067 + ) 2068 + 2069 + type RepoImportParams struct { 2070 + SourceKind RepoImportSource 2071 + CloneUrl string 2072 + Name string 2073 + Knot string 2074 + Description string 2075 + Website string 2076 + Topics []string 2077 + 2078 + Selected bool 2079 + } 2080 + 2081 + func (p *Pages) AccountMigrateFromGitHub(w io.Writer, params AccountMigrateFromGitHubParams) error { 2082 + return p.execute("account/migrate/fromGitHub", w, params) 2083 + } 2084 + 2085 + type AccountMigrateRepoListParams struct { 2086 + Repos []RepoImportParams 2087 + Knots []string 2088 + } 2089 + 2090 + func (p *Pages) AccountMigrateRepoListFragment(w io.Writer, params AccountMigrateRepoListParams) error { 2091 + return p.executePlain("account/migrate/fragments/repoList", w, params) 2092 + } 2093 + 2094 + type AccountMigrateProgressParams struct { 2095 + BaseParams 2096 + Migrations db.GitRepoMigrations 2097 + } 2098 + 2099 + func (p *Pages) AccountMigrateProgressRowsFragment(w io.Writer, params AccountMigrateProgressParams) error { 2100 + return p.executePlain("account/migrate/fragments/progressRows", w, params) 2101 + } 2102 + 2103 + type UserMigrationSettingsParams struct { 2104 + BaseParams 2105 + Tab string 2106 + Migrations db.GitRepoMigrations 2107 + } 2108 + 2109 + func (p *Pages) UserMigrationSettings(w io.Writer, params UserMigrationSettingsParams) error { 2110 + return p.execute("user/settings/migration", w, params) 2111 + } 2112 + 2050 2113 func (p *Pages) Static() http.Handler { 2051 2114 if p.dev { 2052 2115 return http.StripPrefix("/static/", http.FileServer(http.Dir("appview/pages/static")))
+35
appview/pages/templates/account/migrate/fragments/progressRows.html
··· 1 + {{ define "account/migrate/fragments/progressRows" }} 2 + <tbody 3 + id="migration-rows" 4 + {{ if .Migrations.AnyActive }} 5 + hx-get="/account/migrate/progress/rows" 6 + hx-trigger="every 3s" 7 + hx-target="this" 8 + hx-swap="outerHTML" 9 + {{ end }} 10 + > 11 + {{ range .Migrations }} 12 + <tr class="border-t border-gray-100 dark:border-gray-700"> 13 + <td class="px-3 py-2 truncate">{{ .Name }}</td> 14 + <td class="px-3 py-2 text-gray-500 dark:text-gray-400 max-w-[8rem] sm:max-w-none truncate">{{ .Knot }}</td> 15 + <td class="px-3 py-2"> 16 + {{ if eq .Status "done" }} 17 + <a href="/{{ resolve .OwnerDid.String }}/{{ .Name }}">done</a> 18 + {{ else if eq .Status "failed" }} 19 + <span class="error" title="{{ .ErrorMsg }}">failed</span> 20 + {{ else if eq .Status "running" }} 21 + <span class="text-blue-600 dark:text-blue-400">running…</span> 22 + {{ else }} 23 + <span class="text-gray-500 dark:text-gray-400">pending</span> 24 + {{ end }} 25 + {{ if .ErrorMsg }} 26 + <span class="block sm:inline mt-1 sm:mt-0 text-xs text-gray-500 dark:text-gray-400 break-words">({{ .ErrorMsg }})</span> 27 + {{ end }} 28 + </td> 29 + <td class="px-3 py-2 text-gray-500 dark:text-gray-400 text-xs hidden sm:table-cell"> 30 + {{ if not .UpdatedAt.IsZero }}{{ .UpdatedAt.Format "2006-01-02 15:04:05" }}{{ end }} 31 + </td> 32 + </tr> 33 + {{ end }} 34 + </tbody> 35 + {{ end }}
+68
appview/pages/templates/account/migrate/fragments/repoList.html
··· 1 + {{ define "account/migrate/fragments/repoList" }} 2 + {{ if not .Repos }} 3 + <p class="text-gray-500 dark:text-gray-400">No importable repositories found.</p> 4 + {{ else }} 5 + <form action="/account/migrate/start" method="post" class="flex flex-col gap-3"> 6 + <input type="hidden" name="count" value="{{ len .Repos }}" /> 7 + 8 + <div class="flex items-center gap-2 pb-2 border-b border-gray-200 dark:border-gray-700"> 9 + <button type="button" 10 + onclick="document.querySelectorAll('input[name^=selected_]').forEach(c => c.checked = true)" 11 + class="btn-flat"> 12 + select all 13 + </button> 14 + <button type="button" 15 + onclick="document.querySelectorAll('input[name^=selected_]').forEach(c => c.checked = false)" 16 + class="btn-flat"> 17 + clear 18 + </button> 19 + </div> 20 + 21 + {{ range $i, $repo := .Repos }} 22 + <details class="border border-gray-200 dark:border-gray-700 rounded"> 23 + <summary class="flex flex-col gap-2 sm:flex-row sm:items-center sm:gap-3 px-3 py-2 cursor-pointer list-none"> 24 + <div class="flex items-center gap-3 flex-1 min-w-0"> 25 + <input type="checkbox" name="selected_{{ $i }}" value="1" id="selected_{{ $i }}" 26 + onclick="event.stopPropagation()" 27 + {{ if $repo.Selected }}checked{{ end }} /> 28 + <div class="flex-1 min-w-0"> 29 + <div class="font-medium truncate">{{ $repo.Name }}</div> 30 + <div class="text-xs text-gray-500 dark:text-gray-400 truncate">{{ $repo.CloneUrl }}</div> 31 + </div> 32 + </div> 33 + <select name="knot_{{ $i }}" 34 + onclick="event.stopPropagation()" 35 + class="w-full sm:w-auto rounded border border-gray-300 bg-white dark:bg-gray-800 dark:text-white dark:border-gray-600"> 36 + {{ range $.Knots }} 37 + <option value="{{ . }}">{{ . }}</option> 38 + {{ end }} 39 + </select> 40 + <span class="hidden sm:inline text-gray-400 text-xs select-none">edit</span> 41 + </summary> 42 + 43 + <div class="border-t border-gray-200 dark:border-gray-700 p-3 flex flex-col gap-2"> 44 + <input type="hidden" name="clone_url_{{ $i }}" value="{{ $repo.CloneUrl }}" /> 45 + <input type="hidden" name="website_{{ $i }}" value="{{ $repo.Website }}" /> 46 + <input type="hidden" name="topics_{{ $i }}" value="{{ join $repo.Topics "," }}" /> 47 + 48 + <label class="flex flex-col gap-1 text-sm"> 49 + <span class="text-xs uppercase text-gray-500">Repository name</span> 50 + <input type="text" name="name_{{ $i }}" value="{{ $repo.Name }}" required 51 + class="border border-gray-300 dark:border-gray-700 rounded px-2 py-1 bg-transparent" /> 52 + </label> 53 + 54 + <label class="flex flex-col gap-1 text-sm"> 55 + <span class="text-xs uppercase text-gray-500">Description</span> 56 + <input type="text" name="description_{{ $i }}" value="{{ $repo.Description }}" maxlength="140" 57 + class="border border-gray-300 dark:border-gray-700 rounded px-2 py-1 bg-transparent" /> 58 + </label> 59 + </div> 60 + </details> 61 + {{ end }} 62 + 63 + <div class="pt-3"> 64 + <button type="submit" class="btn">Start migration</button> 65 + </div> 66 + </form> 67 + {{ end }} 68 + {{ end }}
+37
appview/pages/templates/account/migrate/fromGitHub.html
··· 1 + {{ define "title" }}Migrate from GitHub{{ end }} 2 + 3 + {{ define "content" }} 4 + <div class="p-6"> 5 + <p class="text-xl font-bold dark:text-white">Migrate from GitHub</p> 6 + <p class="text-gray-500 dark:text-gray-400 pb-2"> 7 + Enter a GitHub username to list their public repositories. 8 + </p> 9 + </div> 10 + <div class="bg-white dark:bg-gray-800 p-6 rounded w-full mx-auto drop-shadow-sm dark:text-white flex flex-col gap-4"> 11 + <form 12 + hx-post="/account/migrate/listGitHubRepos" 13 + hx-target="#source-repos" 14 + hx-swap="innerHTML" 15 + class="flex flex-wrap gap-2 items-center" 16 + > 17 + <input 18 + type="text" 19 + name="username" 20 + required 21 + placeholder="GitHub username" 22 + class="border border-gray-300 dark:border-gray-700 rounded px-2 py-1 bg-transparent flex-1 min-w-0" 23 + /> 24 + <button type="submit" class="btn">Load repos</button> 25 + </form> 26 + 27 + <span id="migrate-error" class="error"></span> 28 + 29 + {{ if not .Knots }} 30 + <p class="text-gray-500 dark:text-gray-400"> 31 + You aren't a member of any knot yet. Join or register a knot first. 32 + </p> 33 + {{ end }} 34 + 35 + <div id="source-repos"></div> 36 + </div> 37 + {{ end }}
+15
appview/pages/templates/account/migrate/index.html
··· 1 + {{ define "title" }}Migrate account{{ end }} 2 + 3 + {{ define "content" }} 4 + <div class="p-6"> 5 + <p class="text-xl font-bold dark:text-white">Migrate repositories</p> 6 + <p class="text-gray-500 dark:text-gray-400 pb-2"> 7 + Pull repositories from another forge into Tangled. 8 + </p> 9 + </div> 10 + <div class="bg-white dark:bg-gray-800 p-6 rounded w-full mx-auto drop-shadow-sm dark:text-white"> 11 + <a href="/account/migrate?service=github" class="btn"> 12 + Migrate from GitHub 13 + </a> 14 + </div> 15 + {{ end }}
+55
appview/pages/templates/user/settings/migration.html
··· 1 + {{ define "title" }}{{ .Tab }} settings{{ end }} 2 + 3 + {{ define "content" }} 4 + <div class="p-6"> 5 + <p class="text-xl font-bold dark:text-white">Settings</p> 6 + </div> 7 + <div class="bg-white dark:bg-gray-800 p-6 rounded relative w-full mx-auto drop-shadow-sm dark:text-white"> 8 + <section class="w-full grid grid-cols-1 md:grid-cols-4 gap-6"> 9 + <div class="col-span-1"> 10 + {{ template "user/settings/fragments/sidebar" . }} 11 + </div> 12 + <div class="col-span-1 md:col-span-3 flex flex-col gap-6"> 13 + {{ template "migrationSettings" . }} 14 + </div> 15 + </section> 16 + </div> 17 + {{ end }} 18 + 19 + {{ define "migrationSettings" }} 20 + <div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between"> 21 + <div> 22 + <h2 class="text-sm pb-2 uppercase font-bold">Repository imports</h2> 23 + <p class="text-gray-500 dark:text-gray-400"> 24 + Track the status of repositories pulled in from another forge. 25 + Successful imports stay; failed ones can be retried by starting a new migration with the same name. 26 + </p> 27 + </div> 28 + <div class="sm:justify-self-end"> 29 + <a href="/account/migrate?service=github" class="btn flex items-center gap-2 whitespace-nowrap"> 30 + {{ i "plus" "size-4" }} 31 + new migration 32 + </a> 33 + </div> 34 + </div> 35 + 36 + <div class="flex flex-col rounded border border-gray-200 dark:border-gray-700 w-full"> 37 + {{ if not .Migrations }} 38 + <div class="flex items-center justify-center p-4 text-gray-500"> 39 + no migrations yet 40 + </div> 41 + {{ else }} 42 + <table class="w-full text-left table-fixed"> 43 + <thead class="text-xs uppercase text-gray-500"> 44 + <tr> 45 + <th class="px-3 py-2">Repo</th> 46 + <th class="px-3 py-2">Knot</th> 47 + <th class="px-3 py-2">Status</th> 48 + <th class="px-3 py-2 hidden sm:table-cell">Updated</th> 49 + </tr> 50 + </thead> 51 + {{ template "account/migrate/fragments/progressRows" . }} 52 + </table> 53 + {{ end }} 54 + </div> 55 + {{ end }}
+19
appview/settings/settings.go
··· 79 79 r.Delete("/", s.releaseSitesDomain) 80 80 }) 81 81 82 + r.Route("/migration", func(r chi.Router) { 83 + r.Get("/", s.migrationSettings) 84 + }) 85 + 82 86 r.Post("/password/request", s.requestPasswordReset) 83 87 r.Post("/password/reset", s.resetPassword) 84 88 r.Post("/deactivate", s.deactivateAccount) ··· 108 112 return false, err 109 113 } 110 114 return strings.TrimRight(userIdent.PDSEndpoint(), "/") == strings.TrimRight(s.Config.Pds.Host, "/"), nil 115 + } 116 + 117 + func (s *Settings) migrationSettings(w http.ResponseWriter, r *http.Request) { 118 + user := s.OAuth.GetMultiAccountUser(r) 119 + 120 + migrations, err := db.ListGitRepoMigrationsForOwner(r.Context(), s.Db, user.Did) 121 + if err != nil { 122 + s.Logger.Error("list migrations failed", "did", user.Did, "err", err) 123 + } 124 + 125 + s.Pages.UserMigrationSettings(w, pages.UserMigrationSettingsParams{ 126 + BaseParams: pages.BaseParamsFromContext(r.Context()), 127 + Tab: "migration", 128 + Migrations: migrations, 129 + }) 111 130 } 112 131 113 132 func (s *Settings) sitesSettings(w http.ResponseWriter, r *http.Request) {
+14
appview/state/router.go
··· 8 8 "strings" 9 9 10 10 "github.com/go-chi/chi/v5" 11 + "tangled.org/core/appview/accountmigration" 11 12 "tangled.org/core/appview/db" 12 13 "tangled.org/core/appview/focus" 13 14 "tangled.org/core/appview/issues" ··· 262 263 }) 263 264 r.Get("/profile/popover", s.ProfilePopover) 264 265 266 + r.Mount("/account/migrate", s.AccountMigrationRouter()) 267 + 265 268 r.Route("/profile", func(r chi.Router) { 266 269 r.Use(middleware.AuthMiddleware(s.oauth)) 267 270 r.Get("/edit-bio", s.EditBioFragment) ··· 320 323 321 324 http.Redirect(w, r, "/@tangled.org/core", http.StatusFound) 322 325 } 326 + } 327 + 328 + func (s *State) AccountMigrationRouter() http.Handler { 329 + am := accountmigration.New( 330 + log.SubLogger(s.logger, "accountmigration"), 331 + s.oauth, 332 + s.pages, 333 + s.db, 334 + s.enforcer, 335 + ) 336 + return am.Router() 323 337 } 324 338 325 339 func (s *State) SettingsRouter() http.Handler {
+12
appview/state/state.go
··· 13 13 14 14 "tangled.org/core/api/tangled" 15 15 "tangled.org/core/appview" 16 + "tangled.org/core/appview/accountmigration" 16 17 "tangled.org/core/appview/bsky" 17 18 "tangled.org/core/appview/cache" 18 19 "tangled.org/core/appview/cloudflare" ··· 75 76 logger *slog.Logger 76 77 cfClient *cloudflare.Client 77 78 codesearch *codesearch.CodeSearch 79 + 80 + accountMigrationWorker *accountmigration.Worker 78 81 } 79 82 80 83 func Make(ctx context.Context, config *config.Config) (*State, error) { ··· 224 227 } 225 228 knotstream.Start(ctx) 226 229 230 + 231 + if err := db.ReapStaleRunningGitRepoMigrations(ctx, d); err != nil { 232 + logger.Warn("failed to reap stale gitrepo migrations", "err", err) 233 + } 234 + amWorker := accountmigration.NewWorker(d, oauth, config.Core.Dev, log.SubLogger(logger, "accountmigration")) 235 + amWorker.Start(ctx) 236 + 227 237 state := &State{ 228 238 db: d, 229 239 notifier: notifier, ··· 243 253 logger: logger, 244 254 cfClient: cfClient, 245 255 codesearch: &codesearch.CodeSearch{Host: config.CodeSearch.ZoektUrl}, 256 + 257 + accountMigrationWorker: amWorker, 246 258 } 247 259 248 260 // fetch initial bluesky posts if configured