Skip to content

Commit 7eba450

Browse files
silvanocerzasilvanocerza
authored
Change core and lib search commands to use fuzzy search (#1193)
* Change lib search command to use fuzzy search * Change core search command to use fuzzy search * Avoid splitting search arguments when doing fuzzy search * Check ranking when running fuzzy search * Some other enhancements to fuzzy search * Fix duplicated results in lib search command
1 parent b8c9e89 commit 7eba450

File tree

8 files changed

+235
-121
lines changed

8 files changed

+235
-121
lines changed

‎commands/core/search.go‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,16 @@ import (
2323
"github.com/arduino/arduino-cli/arduino/cores"
2424
"github.com/arduino/arduino-cli/commands"
2525
rpc "github.com/arduino/arduino-cli/rpc/commands"
26+
"github.com/lithammer/fuzzysearch/fuzzy"
2627
)
2728

28-
func match(line, searchArgs string) bool {
29-
return strings.Contains(strings.ToLower(line), strings.ToLower(searchArgs))
30-
}
31-
32-
func exactMatch(line, searchArgs string) bool {
33-
return strings.Compare(strings.ToLower(line), strings.ToLower(searchArgs)) == 0
34-
}
29+
// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
30+
// This value is completely arbitrary and picked randomly.
31+
const maximumSearchDistance = 20
3532

3633
// PlatformSearch FIXMEDOC
3734
func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error) {
38-
searchArgs := req.SearchArgs
35+
searchArgs := strings.Trim(req.SearchArgs, " ")
3936
allVersions := req.AllVersions
4037
pm := commands.GetPackageManager(req.Instance.Id)
4138
if pm == nil {
@@ -63,29 +60,54 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error)
6360
continue
6461
}
6562

66-
// platform has a valid release, check if it matches the search arguments
67-
if match(platform.Name, searchArgs) || match(platform.Architecture, searchArgs) ||
68-
exactMatch(platform.String(), searchArgs) || match(targetPackage.Name, searchArgs) ||
69-
match(targetPackage.Maintainer, searchArgs) || match(targetPackage.WebsiteURL, searchArgs) {
63+
if searchArgs == "" {
7064
if allVersions {
7165
res = append(res, platform.GetAllReleases()...)
7266
} else {
7367
res = append(res, platformRelease)
7468
}
75-
} else {
76-
// if we didn't find a match in the platform data, search for
77-
// a match in the boards manifest
78-
for _, board := range platformRelease.BoardsManifest {
79-
if match(board.Name, searchArgs) {
69+
continue
70+
}
71+
72+
// Gather all strings that can be used for searching
73+
toTest := []string{
74+
platform.String(),
75+
platform.Name,
76+
platform.Architecture,
77+
targetPackage.Name,
78+
targetPackage.Maintainer,
79+
targetPackage.WebsiteURL,
80+
}
81+
for _, board := range platformRelease.BoardsManifest {
82+
toTest = append(toTest, board.Name)
83+
}
84+
85+
// Removes some chars from query strings to enhance results
86+
cleanSearchArgs := strings.Map(func(r rune) rune {
87+
switch r {
88+
case '_':
89+
case '-':
90+
case ' ':
91+
return -1
92+
}
93+
return r
94+
}, searchArgs)
95+
96+
// Fuzzy search
97+
for _, arg := range []string{searchArgs, cleanSearchArgs} {
98+
for _, rank := range fuzzy.RankFindNormalizedFold(arg, toTest) {
99+
// Accepts only results that close to the searched terms
100+
if rank.Distance < maximumSearchDistance {
80101
if allVersions {
81102
res = append(res, platform.GetAllReleases()...)
82103
} else {
83104
res = append(res, platformRelease)
84105
}
85-
break
106+
goto nextPlatform
86107
}
87108
}
88109
}
110+
nextPlatform:
89111
}
90112
}
91113
}

‎commands/core/search_test.go‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,9 @@ import (
2424
"github.com/arduino/arduino-cli/rpc/commands"
2525
rpc "github.com/arduino/arduino-cli/rpc/commands"
2626
"github.com/arduino/go-paths-helper"
27-
"github.com/stretchr/testify/assert"
2827
"github.com/stretchr/testify/require"
2928
)
3029

31-
func TestMatch(t *testing.T) {
32-
assert.True(t, match("this is platform Foo", "foo"))
33-
assert.True(t, match("this is platform Foo", "FOO"))
34-
assert.True(t, match("this is platform Foo", ""))
35-
assert.False(t, match("this is platform Foo", "Bar"))
36-
}
37-
3830
func TestPlatformSearch(t *testing.T) {
3931

4032
dataDir := paths.TempDir().Join("test", "data_dir")
@@ -238,4 +230,50 @@ func TestPlatformSearch(t *testing.T) {
238230
{Name: "Linino One"},
239231
},
240232
})
233+
234+
res, err = PlatformSearch(&rpc.PlatformSearchReq{
235+
Instance: inst,
236+
SearchArgs: "yun",
237+
AllVersions: true,
238+
})
239+
require.Nil(t, err)
240+
require.NotNil(t, res)
241+
require.Len(t, res.SearchOutput, 1)
242+
require.Contains(t, res.SearchOutput, &commands.Platform{
243+
ID: "arduino:avr",
244+
Installed: "",
245+
Latest: "1.8.3",
246+
Name: "Arduino AVR Boards",
247+
Maintainer: "Arduino",
248+
Website: "https://www.arduino.cc/",
249+
Email: "packages@arduino.cc",
250+
Boards: []*commands.Board{
251+
{Name: "Arduino Yún"},
252+
{Name: "Arduino Uno"},
253+
{Name: "Arduino Uno WiFi"},
254+
{Name: "Arduino Diecimila"},
255+
{Name: "Arduino Nano"},
256+
{Name: "Arduino Mega"},
257+
{Name: "Arduino MegaADK"},
258+
{Name: "Arduino Leonardo"},
259+
{Name: "Arduino Leonardo Ethernet"},
260+
{Name: "Arduino Micro"},
261+
{Name: "Arduino Esplora"},
262+
{Name: "Arduino Mini"},
263+
{Name: "Arduino Ethernet"},
264+
{Name: "Arduino Fio"},
265+
{Name: "Arduino BT"},
266+
{Name: "Arduino LilyPadUSB"},
267+
{Name: "Arduino Lilypad"},
268+
{Name: "Arduino Pro"},
269+
{Name: "Arduino ATMegaNG"},
270+
{Name: "Arduino Robot Control"},
271+
{Name: "Arduino Robot Motor"},
272+
{Name: "Arduino Gemma"},
273+
{Name: "Adafruit Circuit Playground"},
274+
{Name: "Arduino Yún Mini"},
275+
{Name: "Arduino Industrial 101"},
276+
{Name: "Linino One"},
277+
},
278+
})
241279
}

‎commands/lib/search.go‎

Lines changed: 52 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,10 @@ import (
2424
"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
2525
"github.com/arduino/arduino-cli/commands"
2626
rpc "github.com/arduino/arduino-cli/rpc/commands"
27-
"github.com/imjasonmiller/godice"
27+
"github.com/lithammer/fuzzysearch/fuzzy"
2828
semver "go.bug.st/relaxed-semver"
2929
)
3030

31-
var similarityThreshold = 0.7
32-
3331
// LibrarySearch FIXMEDOC
3432
func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchReq) (*rpc.LibrarySearchResp, error) {
3533
lm := commands.GetLibraryManager(req.GetInstance().GetId())
@@ -41,45 +39,70 @@ func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchReq) (*rpc.Library
4139
}
4240

4341
func searchLibrary(req *rpc.LibrarySearchReq, lm *librariesmanager.LibrariesManager) (*rpc.LibrarySearchResp, error) {
42+
query := req.GetQuery()
4443
res := []*rpc.SearchedLibrary{}
4544
status := rpc.LibrarySearchStatus_success
4645

47-
for _, lib := range lm.Index.Libraries {
48-
qry := strings.ToLower(req.GetQuery())
49-
if strings.Contains(strings.ToLower(lib.Name), qry) ||
50-
strings.Contains(strings.ToLower(lib.Latest.Paragraph), qry) ||
51-
strings.Contains(strings.ToLower(lib.Latest.Sentence), qry) {
52-
releases := map[string]*rpc.LibraryRelease{}
53-
for str, rel := range lib.Releases {
54-
releases[str] = GetLibraryParameters(rel)
55-
}
56-
latest := GetLibraryParameters(lib.Latest)
57-
58-
searchedLib := &rpc.SearchedLibrary{
59-
Name: lib.Name,
60-
Releases: releases,
61-
Latest: latest,
62-
}
63-
res = append(res, searchedLib)
46+
// If the query is empty all libraries are returned
47+
if strings.Trim(query, " ") == "" {
48+
for _, lib := range lm.Index.Libraries {
49+
res = append(res, indexLibraryToRPCSearchLibrary(lib))
6450
}
51+
return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
6552
}
6653

67-
if len(res) == 0 {
68-
status = rpc.LibrarySearchStatus_failed
69-
for _, lib := range lm.Index.Libraries {
70-
if godice.CompareString(req.GetQuery(), lib.Name) > similarityThreshold {
71-
res = append(res, &rpc.SearchedLibrary{
72-
Name: lib.Name,
73-
})
54+
// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
55+
// This value is completely arbitrary and picked randomly.
56+
maximumSearchDistance := 150
57+
// Use a lower distance for shorter query or the user might be flooded with unrelated results
58+
if len(query) <= 4 {
59+
maximumSearchDistance = 40
60+
}
61+
62+
// Removes some chars from query strings to enhance results
63+
cleanQuery := strings.Map(func(r rune) rune {
64+
switch r {
65+
case '_':
66+
case '-':
67+
case ' ':
68+
return -1
69+
}
70+
return r
71+
}, query)
72+
for _, lib := range lm.Index.Libraries {
73+
// Use both uncleaned and cleaned query
74+
for _, q := range []string{query, cleanQuery} {
75+
toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence}
76+
for _, rank := range fuzzy.RankFindNormalizedFold(q, toTest) {
77+
if rank.Distance < maximumSearchDistance {
78+
res = append(res, indexLibraryToRPCSearchLibrary(lib))
79+
goto nextLib
80+
}
7481
}
7582
}
83+
nextLib:
7684
}
7785

7886
return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
7987
}
8088

81-
// GetLibraryParameters FIXMEDOC
82-
func GetLibraryParameters(rel *librariesindex.Release) *rpc.LibraryRelease {
89+
// indexLibraryToRPCSearchLibrary converts a librariindex.Library to rpc.SearchLibrary
90+
func indexLibraryToRPCSearchLibrary(lib *librariesindex.Library) *rpc.SearchedLibrary {
91+
releases := map[string]*rpc.LibraryRelease{}
92+
for str, rel := range lib.Releases {
93+
releases[str] = getLibraryParameters(rel)
94+
}
95+
latest := getLibraryParameters(lib.Latest)
96+
97+
return &rpc.SearchedLibrary{
98+
Name: lib.Name,
99+
Releases: releases,
100+
Latest: latest,
101+
}
102+
}
103+
104+
// getLibraryParameters FIXMEDOC
105+
func getLibraryParameters(rel *librariesindex.Release) *rpc.LibraryRelease {
83106
return &rpc.LibraryRelease{
84107
Author: rel.Author,
85108
Version: rel.Version.String(),

‎commands/lib/search_test.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,12 @@ func TestSearchLibrarySimilar(t *testing.T) {
4848
}
4949

5050
assert := assert.New(t)
51-
assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_failed)
52-
assert.Equal(len(resp.GetLibraries()), 1)
53-
assert.Equal(resp.GetLibraries()[0].Name, "Arduino")
51+
assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_success)
52+
assert.Equal(len(resp.GetLibraries()), 2)
53+
libs := map[string]*rpc.SearchedLibrary{}
54+
for _, l := range resp.GetLibraries() {
55+
libs[l.Name] = l
56+
}
57+
assert.Contains(libs, "ArduinoTestPackage")
58+
assert.Contains(libs, "Arduino")
5459
}

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ require (
1919
github.com/gofrs/uuid v3.2.0+incompatible
2020
github.com/golang/protobuf v1.4.2
2121
github.com/h2non/filetype v1.0.8 // indirect
22-
github.com/imjasonmiller/godice v0.1.2
2322
github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
2423
github.com/kr/text v0.2.0 // indirect
2524
github.com/leonelquinteros/gotext v1.4.0
25+
github.com/lithammer/fuzzysearch v1.1.1
2626
github.com/mattn/go-colorable v0.1.2
2727
github.com/mattn/go-isatty v0.0.8
2828
github.com/mattn/go-runewidth v0.0.9 // indirect

‎go.sum‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
22
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
33
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
4-
github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
54
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
65
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
76
github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
@@ -117,8 +116,6 @@ github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
117116
github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
118117
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
119118
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
120-
github.com/imjasonmiller/godice v0.1.2 h1:T1/sW/HoDzFeuwzOOuQjmeMELz9CzZ53I2CnD+08zD4=
121-
github.com/imjasonmiller/godice v0.1.2/go.mod h1:8cTkdnVI+NglU2d6sv+ilYcNaJ5VSTBwvMbFULJd/QQ=
122119
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
123120
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
124121
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -155,6 +152,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
155152
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
156153
github.com/leonelquinteros/gotext v1.4.0 h1:2NHPCto5IoMXbrT0bldPrxj0qM5asOCwtb1aUQZ1tys=
157154
github.com/leonelquinteros/gotext v1.4.0/go.mod h1:yZGXREmoGTtBvZHNcc+Yfug49G/2spuF/i/Qlsvz1Us=
155+
github.com/lithammer/fuzzysearch v1.1.1 h1:8F9OAV2xPuYblToVohjanztdnPjbtA0MLgMvDKQ0Z08=
156+
github.com/lithammer/fuzzysearch v1.1.1/go.mod h1:H2bng+w5gsR7NlfIJM8ElGZI0sX6C/9uzGqicVXGU6c=
158157
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
159158
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
160159
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -215,7 +214,6 @@ github.com/segmentio/objconv v1.0.1 h1:QjfLzwriJj40JibCV3MGSEiAoXixbp4ybhwfTB8RX
215214
github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys=
216215
github.com/segmentio/stats/v4 v4.5.3 h1:Y/DSUWZ4c8ICgqJ9rQohzKvGqGWbLPWad5zmxVoKN+Y=
217216
github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo=
218-
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
219217
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
220218
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
221219
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -281,7 +279,6 @@ golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnf
281279
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
282280
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
283281
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
284-
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
285282
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
286283
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y=
287284
golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -334,7 +331,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
334331
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
335332
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
336333
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
337-
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
338334
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
339335
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
340336
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

0 commit comments

Comments
 (0)