alice.pds.demo.boltless.dev
This repository has no description
1// heavily inspired by gitea's model (basically copy-pasted)
2package pulls_indexer
3
4import (
5 "context"
6 "errors"
7 "log"
8 "os"
9
10 "github.com/blevesearch/bleve/v2"
11 "github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
12 "github.com/blevesearch/bleve/v2/analysis/token/camelcase"
13 "github.com/blevesearch/bleve/v2/analysis/token/lowercase"
14 "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
15 "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
16 "github.com/blevesearch/bleve/v2/index/upsidedown"
17 "github.com/blevesearch/bleve/v2/mapping"
18 "github.com/blevesearch/bleve/v2/search/query"
19 "tangled.org/core/appview/db"
20 "tangled.org/core/appview/indexer/base36"
21 bleveutil "tangled.org/core/appview/indexer/bleve"
22 "tangled.org/core/appview/models"
23 "tangled.org/core/appview/pagination"
24 tlog "tangled.org/core/log"
25)
26
27const (
28 pullIndexerAnalyzer = "pullIndexer"
29 pullIndexerDocType = "pullIndexerDocType"
30
31 unicodeNormalizeName = "uicodeNormalize"
32
33 // Bump this when the index mapping changes to trigger a rebuild.
34 pullIndexerVersion = 4
35)
36
37type Indexer struct {
38 indexer bleve.Index
39 path string
40}
41
42func NewIndexer(indexDir string) *Indexer {
43 return &Indexer{
44 path: indexDir,
45 }
46}
47
48// Init initializes the indexer
49func (ix *Indexer) Init(ctx context.Context, e db.Execer) {
50 l := tlog.FromContext(ctx)
51 existed, err := ix.initialize(ctx)
52 if err != nil {
53 log.Fatalln("failed to initialize pull indexer", err)
54 }
55 if !existed {
56 l.Debug("Populating the pull indexer")
57 err := PopulateIndexer(ctx, ix, e)
58 if err != nil {
59 log.Fatalln("failed to populate pull indexer", err)
60 }
61 }
62
63 count, _ := ix.indexer.DocCount()
64 l.Info("Initialized the pull indexer", "docCount", count)
65}
66
67func generatePullIndexMapping() (mapping.IndexMapping, error) {
68 mapping := bleve.NewIndexMapping()
69 docMapping := bleve.NewDocumentMapping()
70
71 textFieldMapping := bleve.NewTextFieldMapping()
72 textFieldMapping.Store = false
73 textFieldMapping.IncludeInAll = false
74
75 keywordFieldMapping := bleve.NewKeywordFieldMapping()
76 keywordFieldMapping.Store = false
77 keywordFieldMapping.IncludeInAll = false
78
79 // numericFieldMapping := bleve.NewNumericFieldMapping()
80
81 docMapping.AddFieldMappingsAt("title", textFieldMapping)
82 docMapping.AddFieldMappingsAt("body", textFieldMapping)
83
84 docMapping.AddFieldMappingsAt("repo_did", keywordFieldMapping)
85 docMapping.AddFieldMappingsAt("state", keywordFieldMapping)
86 docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping)
87 docMapping.AddFieldMappingsAt("labels", keywordFieldMapping)
88 docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping)
89
90 err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{
91 "type": unicodenorm.Name,
92 "form": unicodenorm.NFC,
93 })
94 if err != nil {
95 return nil, err
96 }
97
98 err = mapping.AddCustomAnalyzer(pullIndexerAnalyzer, map[string]any{
99 "type": custom.Name,
100 "char_filters": []string{},
101 "tokenizer": unicode.Name,
102 "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name},
103 })
104 if err != nil {
105 return nil, err
106 }
107
108 mapping.DefaultAnalyzer = pullIndexerAnalyzer
109 mapping.AddDocumentMapping(pullIndexerDocType, docMapping)
110 mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
111 mapping.DefaultMapping = bleve.NewDocumentDisabledMapping()
112
113 return mapping, nil
114}
115
116func (ix *Indexer) initialize(ctx context.Context) (bool, error) {
117 if ix.indexer != nil {
118 return false, errors.New("indexer is already initialized")
119 }
120
121 indexer, err := openIndexer(ctx, ix.path, pullIndexerVersion)
122 if err != nil {
123 return false, err
124 }
125 if indexer != nil {
126 ix.indexer = indexer
127 return true, nil
128 }
129
130 mapping, err := generatePullIndexMapping()
131 if err != nil {
132 return false, err
133 }
134 indexer, err = bleve.New(ix.path, mapping)
135 if err != nil {
136 return false, err
137 }
138 indexer.SetInternal([]byte("mapping_version"), []byte{byte(pullIndexerVersion)})
139
140 ix.indexer = indexer
141
142 return false, nil
143}
144
145func openIndexer(ctx context.Context, path string, version int) (bleve.Index, error) {
146 l := tlog.FromContext(ctx)
147 indexer, err := bleve.Open(path)
148 if err != nil {
149 if errors.Is(err, upsidedown.IncompatibleVersion) {
150 l.Info("Indexer was built with a previous version of bleve, deleting and rebuilding")
151 return nil, os.RemoveAll(path)
152 }
153 return nil, nil
154 }
155
156 storedVersion, _ := indexer.GetInternal([]byte("mapping_version"))
157 if storedVersion == nil || int(storedVersion[0]) != version {
158 l.Info("Indexer mapping version changed, deleting and rebuilding")
159 indexer.Close()
160 return nil, os.RemoveAll(path)
161 }
162
163 return indexer, nil
164}
165
166func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error {
167 l := tlog.FromContext(ctx)
168 count := 0
169 err := pagination.IterateAll(
170 func(page pagination.Page) ([]*models.Pull, error) {
171 return db.GetPullsPaginated(ctx, e, page)
172 },
173 func(pulls []*models.Pull) error {
174 count += len(pulls)
175 return ix.Index(ctx, pulls...)
176 },
177 )
178 l.Info("pulls indexed", "count", count)
179 return err
180}
181
182type pullData struct {
183 ID int64 `json:"id"`
184 RepoDid string `json:"repo_did"`
185 PullID int `json:"pull_id"`
186 Title string `json:"title"`
187 Body string `json:"body"`
188 State string `json:"state"`
189 AuthorDid string `json:"author_did"`
190 Labels []string `json:"labels"`
191 LabelValues []string `json:"label_values"`
192
193 Comments []pullCommentData `json:"comments"`
194}
195
196func makePullData(pull *models.Pull) *pullData {
197 return &pullData{
198 ID: pull.ID,
199 RepoDid: pull.RepoDid.String(),
200 PullID: int(pull.PullId),
201 Title: pull.Title,
202 Body: pull.Body,
203 State: pull.State.String(),
204 AuthorDid: pull.OwnerDid.String(),
205 Labels: pull.Labels.LabelNames(),
206 LabelValues: pull.Labels.LabelNameValues(),
207 }
208}
209
210// Type returns the document type, for bleve's mapping.Classifier interface.
211func (i *pullData) Type() string {
212 return pullIndexerDocType
213}
214
215type pullCommentData struct {
216 Body string `json:"body"`
217}
218
219type searchResult struct {
220 Hits []int64
221 Total uint64
222}
223
224const maxBatchSize = 20
225
226func (ix *Indexer) Index(ctx context.Context, pulls ...*models.Pull) error {
227 batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize)
228 for _, pull := range pulls {
229 pullData := makePullData(pull)
230 if err := batch.Index(base36.Encode(pullData.ID), pullData); err != nil {
231 return err
232 }
233 }
234 return batch.Flush()
235}
236
237func (ix *Indexer) Delete(ctx context.Context, pullID int64) error {
238 return ix.indexer.Delete(base36.Encode(pullID))
239}
240
241func (ix *Indexer) Search(ctx context.Context, opts models.PullSearchOptions) (*searchResult, error) {
242 var musts []query.Query
243 var mustNots []query.Query
244
245 // TODO(boltless): remove this after implementing pulls page pagination
246 limit := opts.Page.Limit
247 if limit == 0 {
248 limit = 500
249 }
250
251 for _, keyword := range opts.Keywords {
252 musts = append(musts, bleve.NewDisjunctionQuery(
253 bleveutil.MatchAndQuery("title", keyword, pullIndexerAnalyzer, 0),
254 bleveutil.MatchAndQuery("body", keyword, pullIndexerAnalyzer, 0),
255 ))
256 }
257
258 for _, phrase := range opts.Phrases {
259 musts = append(musts, bleve.NewDisjunctionQuery(
260 bleveutil.MatchPhraseQuery("title", phrase, pullIndexerAnalyzer),
261 bleveutil.MatchPhraseQuery("body", phrase, pullIndexerAnalyzer),
262 ))
263 }
264
265 for _, keyword := range opts.NegatedKeywords {
266 mustNots = append(mustNots, bleve.NewDisjunctionQuery(
267 bleveutil.MatchAndQuery("title", keyword, pullIndexerAnalyzer, 0),
268 bleveutil.MatchAndQuery("body", keyword, pullIndexerAnalyzer, 0),
269 ))
270 }
271
272 for _, phrase := range opts.NegatedPhrases {
273 mustNots = append(mustNots, bleve.NewDisjunctionQuery(
274 bleveutil.MatchPhraseQuery("title", phrase, pullIndexerAnalyzer),
275 bleveutil.MatchPhraseQuery("body", phrase, pullIndexerAnalyzer),
276 ))
277 }
278
279 musts = append(musts, bleveutil.KeywordFieldQuery("repo_did", opts.RepoDid))
280 if opts.State != nil {
281 musts = append(musts, bleveutil.KeywordFieldQuery("state", opts.State.String()))
282 }
283
284 if opts.AuthorDid != "" {
285 musts = append(musts, bleveutil.KeywordFieldQuery("author_did", opts.AuthorDid))
286 }
287
288 for _, label := range opts.Labels {
289 musts = append(musts, bleveutil.KeywordFieldQuery("labels", label))
290 }
291
292 for _, did := range opts.NegatedAuthorDids {
293 mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did))
294 }
295
296 for _, label := range opts.NegatedLabels {
297 mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label))
298 }
299
300 for _, lv := range opts.LabelValues {
301 musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv))
302 }
303
304 for _, lv := range opts.NegatedLabelValues {
305 mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv))
306 }
307
308 indexerQuery := bleve.NewBooleanQuery()
309 indexerQuery.AddMust(musts...)
310 indexerQuery.AddMustNot(mustNots...)
311 searchReq := bleve.NewSearchRequestOptions(indexerQuery, limit, opts.Page.Offset, false)
312 res, err := ix.indexer.SearchInContext(ctx, searchReq)
313 if err != nil {
314 return nil, err
315 }
316 ret := &searchResult{
317 Total: res.Total,
318 Hits: make([]int64, len(res.Hits)),
319 }
320 for i, hit := range res.Hits {
321 id, err := base36.Decode(hit.ID)
322 if err != nil {
323 return nil, err
324 }
325 ret.Hits[i] = id
326 }
327 return ret, nil
328}