mirror of
https://github.com/grafana/grafana.git
synced 2025-12-21 12:04:45 +08:00
Compare commits
1 Commits
docs/add-t
...
selectable
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
584287dc61 |
@@ -8,6 +8,12 @@ foldersV1beta1: {
|
|||||||
spec: {
|
spec: {
|
||||||
title: string
|
title: string
|
||||||
description?: string
|
description?: string
|
||||||
|
foo: bool
|
||||||
|
bar: int
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
selectableFields: [
|
||||||
|
"spec.title",
|
||||||
|
]
|
||||||
}
|
}
|
||||||
@@ -5,13 +5,26 @@
|
|||||||
package v1beta1
|
package v1beta1
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
"github.com/grafana/grafana-app-sdk/resource"
|
"github.com/grafana/grafana-app-sdk/resource"
|
||||||
)
|
)
|
||||||
|
|
||||||
// schema is unexported to prevent accidental overwrites
|
// schema is unexported to prevent accidental overwrites
|
||||||
var (
|
var (
|
||||||
schemaFolder = resource.NewSimpleSchema("folder.grafana.app", "v1beta1", NewFolder(), &FolderList{}, resource.WithKind("Folder"),
|
schemaFolder = resource.NewSimpleSchema("folder.grafana.app", "v1beta1", NewFolder(), &FolderList{}, resource.WithKind("Folder"),
|
||||||
resource.WithPlural("folders"), resource.WithScope(resource.NamespacedScope))
|
resource.WithPlural("folders"), resource.WithScope(resource.NamespacedScope), resource.WithSelectableFields([]resource.SelectableField{resource.SelectableField{
|
||||||
|
FieldSelector: "spec.title",
|
||||||
|
FieldValueFunc: func(o resource.Object) (string, error) {
|
||||||
|
cast, ok := o.(*Folder)
|
||||||
|
if !ok {
|
||||||
|
return "", errors.New("provided object must be of type *Folder")
|
||||||
|
}
|
||||||
|
|
||||||
|
return cast.Spec.Title, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}))
|
||||||
kindFolder = resource.Kind{
|
kindFolder = resource.Kind{
|
||||||
Schema: schemaFolder,
|
Schema: schemaFolder,
|
||||||
Codecs: map[resource.KindEncoding]resource.Codec{
|
Codecs: map[resource.KindEncoding]resource.Codec{
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ import (
|
|||||||
v1beta1 "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
v1beta1 "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var ()
|
||||||
|
|
||||||
var appManifestData = app.ManifestData{
|
var appManifestData = app.ManifestData{
|
||||||
AppName: "folder",
|
AppName: "folder",
|
||||||
Group: "folder.grafana.app",
|
Group: "folder.grafana.app",
|
||||||
@@ -32,6 +34,9 @@ var appManifestData = app.ManifestData{
|
|||||||
Plural: "Folders",
|
Plural: "Folders",
|
||||||
Scope: "Namespaced",
|
Scope: "Namespaced",
|
||||||
Conversion: false,
|
Conversion: false,
|
||||||
|
SelectableFields: []string{
|
||||||
|
"spec.title",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
Routes: app.ManifestVersionRoutes{
|
Routes: app.ManifestVersionRoutes{
|
||||||
|
|||||||
4
apps/iam/pkg/apis/iam_manifest.go
generated
4
apps/iam/pkg/apis/iam_manifest.go
generated
@@ -74,6 +74,10 @@ var appManifestData = app.ManifestData{
|
|||||||
Plural: "Users",
|
Plural: "Users",
|
||||||
Scope: "Namespaced",
|
Scope: "Namespaced",
|
||||||
Conversion: false,
|
Conversion: false,
|
||||||
|
SelectableFields: []string{
|
||||||
|
"spec.email",
|
||||||
|
"spec.login",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -4,11 +4,24 @@ import (
|
|||||||
"k8s.io/apimachinery/pkg/runtime"
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
"k8s.io/apiserver/pkg/registry/generic"
|
"k8s.io/apiserver/pkg/registry/generic"
|
||||||
"k8s.io/apiserver/pkg/registry/generic/registry"
|
"k8s.io/apiserver/pkg/registry/generic/registry"
|
||||||
|
"k8s.io/apiserver/pkg/storage"
|
||||||
|
|
||||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter) (*registry.Store, error) {
|
type registryStoreOptions struct {
|
||||||
|
attrFunc storage.AttrFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
type OptionFn func(*registryStoreOptions)
|
||||||
|
|
||||||
|
func WithAttrFunc(attrFunc storage.AttrFunc) OptionFn {
|
||||||
|
return func(opts *registryStoreOptions) {
|
||||||
|
opts.attrFunc = attrFunc
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter, options ...OptionFn) (*registry.Store, error) {
|
||||||
gv := resourceInfo.GroupVersion()
|
gv := resourceInfo.GroupVersion()
|
||||||
gv.Version = runtime.APIVersionInternal
|
gv.Version = runtime.APIVersionInternal
|
||||||
strategy := NewStrategy(scheme, gv)
|
strategy := NewStrategy(scheme, gv)
|
||||||
@@ -20,7 +33,7 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o
|
|||||||
NewListFunc: resourceInfo.NewListFunc,
|
NewListFunc: resourceInfo.NewListFunc,
|
||||||
KeyRootFunc: KeyRootFunc(resourceInfo.GroupResource()),
|
KeyRootFunc: KeyRootFunc(resourceInfo.GroupResource()),
|
||||||
KeyFunc: NamespaceKeyFunc(resourceInfo.GroupResource()),
|
KeyFunc: NamespaceKeyFunc(resourceInfo.GroupResource()),
|
||||||
PredicateFunc: Matcher,
|
//PredicateFunc: Matcher,
|
||||||
DefaultQualifiedResource: resourceInfo.GroupResource(),
|
DefaultQualifiedResource: resourceInfo.GroupResource(),
|
||||||
SingularQualifiedResource: resourceInfo.SingularGroupResource(),
|
SingularQualifiedResource: resourceInfo.SingularGroupResource(),
|
||||||
TableConvertor: resourceInfo.TableConverter(),
|
TableConvertor: resourceInfo.TableConverter(),
|
||||||
@@ -28,8 +41,16 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o
|
|||||||
UpdateStrategy: strategy,
|
UpdateStrategy: strategy,
|
||||||
DeleteStrategy: strategy,
|
DeleteStrategy: strategy,
|
||||||
}
|
}
|
||||||
options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs}
|
|
||||||
if err := store.CompleteWithOptions(options); err != nil {
|
opts := ®istryStoreOptions{
|
||||||
|
attrFunc: GetAttrs,
|
||||||
|
}
|
||||||
|
for _, opt := range options {
|
||||||
|
opt(opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
o := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: opts.attrFunc}
|
||||||
|
if err := store.CompleteWithOptions(o); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return store, nil
|
return store, nil
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
"k8s.io/apimachinery/pkg/fields"
|
||||||
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
"k8s.io/apimachinery/pkg/runtime"
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
"k8s.io/apiserver/pkg/admission"
|
"k8s.io/apiserver/pkg/admission"
|
||||||
@@ -22,6 +24,8 @@ import (
|
|||||||
authlib "github.com/grafana/authlib/types"
|
authlib "github.com/grafana/authlib/types"
|
||||||
"github.com/grafana/grafana-app-sdk/logging"
|
"github.com/grafana/grafana-app-sdk/logging"
|
||||||
|
|
||||||
|
sdkres "github.com/grafana/grafana-app-sdk/resource"
|
||||||
|
|
||||||
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||||
"github.com/grafana/grafana/apps/iam/pkg/reconcilers"
|
"github.com/grafana/grafana/apps/iam/pkg/reconcilers"
|
||||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||||
@@ -129,6 +133,29 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
|
|||||||
Version: runtime.APIVersionInternal,
|
Version: runtime.APIVersionInternal,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
kinds := []sdkres.Kind{folders.FolderKind()}
|
||||||
|
for _, kind := range kinds {
|
||||||
|
gvk := gv.WithKind(kind.Kind())
|
||||||
|
err := scheme.AddFieldLabelConversionFunc(
|
||||||
|
gvk,
|
||||||
|
func(label, value string) (string, string, error) {
|
||||||
|
if label == "metadata.name" || label == "metadata.namespace" {
|
||||||
|
return label, value, nil
|
||||||
|
}
|
||||||
|
fields := kind.SelectableFields()
|
||||||
|
for _, field := range fields {
|
||||||
|
if field.FieldSelector == label {
|
||||||
|
return label, value, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", "", fmt.Errorf("field label not supported for %s: %s", gvk, label)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// If multiple versions exist, then register conversions from zz_generated.conversion.go
|
// If multiple versions exist, then register conversions from zz_generated.conversion.go
|
||||||
// if err := playlist.RegisterConversions(scheme); err != nil {
|
// if err := playlist.RegisterConversions(scheme); err != nil {
|
||||||
// return err
|
// return err
|
||||||
@@ -137,6 +164,26 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
|
|||||||
return scheme.SetVersionPriority(gv)
|
return scheme.SetVersionPriority(gv)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: work with all kinds from schema, not just one.
|
||||||
|
func (b *FolderAPIBuilder) BuildGetAttrsFn(k sdkres.Kind) func(obj runtime.Object) (labels.Set, fields.Set, error) {
|
||||||
|
return func(obj runtime.Object) (labels.Set, fields.Set, error) {
|
||||||
|
if robj, ok := obj.(sdkres.Object); !ok {
|
||||||
|
return nil, nil, fmt.Errorf("not a resource.Object")
|
||||||
|
} else {
|
||||||
|
fieldsSet := fields.Set{}
|
||||||
|
|
||||||
|
for _, f := range k.SelectableFields() {
|
||||||
|
v, err := f.FieldValueFunc(robj)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
fieldsSet[f.FieldSelector] = v
|
||||||
|
}
|
||||||
|
return robj.GetLabels(), fieldsSet, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (b *FolderAPIBuilder) AllowedV0Alpha1Resources() []string {
|
func (b *FolderAPIBuilder) AllowedV0Alpha1Resources() []string {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -148,10 +195,11 @@ func (b *FolderAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.API
|
|||||||
Permissions: b.setDefaultFolderPermissions,
|
Permissions: b.setDefaultFolderPermissions,
|
||||||
})
|
})
|
||||||
|
|
||||||
unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter)
|
unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter, grafanaregistry.WithAttrFunc(b.BuildGetAttrsFn(folders.FolderKind())))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
b.registerPermissionHooks(unified)
|
b.registerPermissionHooks(unified)
|
||||||
b.storage = unified
|
b.storage = unified
|
||||||
|
|
||||||
|
|||||||
@@ -3,9 +3,11 @@ package resource
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/grafana/grafana-app-sdk/app"
|
||||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
|
||||||
@@ -101,6 +103,9 @@ type IndexableDocument struct {
|
|||||||
// metadata, annotations, or external data linked at index time
|
// metadata, annotations, or external data linked at index time
|
||||||
Fields map[string]any `json:"fields,omitempty"`
|
Fields map[string]any `json:"fields,omitempty"`
|
||||||
|
|
||||||
|
// Automatically indexed selectable fields, used for field-based filtering when listing.
|
||||||
|
SelectableFields map[string]string `json:"selectable_fields,omitempty"`
|
||||||
|
|
||||||
// Maintain a list of resource references.
|
// Maintain a list of resource references.
|
||||||
// Someday this will likely be part of https://github.com/grafana/gamma
|
// Someday this will likely be part of https://github.com/grafana/gamma
|
||||||
References ResourceReferences `json:"references,omitempty"`
|
References ResourceReferences `json:"references,omitempty"`
|
||||||
@@ -175,7 +180,7 @@ func (m ResourceReferences) Less(i, j int) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create a new indexable document based on a generic k8s resource
|
// Create a new indexable document based on a generic k8s resource
|
||||||
func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.GrafanaMetaAccessor) *IndexableDocument {
|
func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.GrafanaMetaAccessor, selectableFields map[string]string) *IndexableDocument {
|
||||||
title := obj.FindTitle(key.Name)
|
title := obj.FindTitle(key.Name)
|
||||||
if title == key.Name {
|
if title == key.Name {
|
||||||
// TODO: something wrong with FindTitle
|
// TODO: something wrong with FindTitle
|
||||||
@@ -199,6 +204,7 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa
|
|||||||
Folder: obj.GetFolder(),
|
Folder: obj.GetFolder(),
|
||||||
CreatedBy: obj.GetCreatedBy(),
|
CreatedBy: obj.GetCreatedBy(),
|
||||||
UpdatedBy: obj.GetUpdatedBy(),
|
UpdatedBy: obj.GetUpdatedBy(),
|
||||||
|
SelectableFields: selectableFields,
|
||||||
}
|
}
|
||||||
m, ok := obj.GetManagerProperties()
|
m, ok := obj.GetManagerProperties()
|
||||||
if ok {
|
if ok {
|
||||||
@@ -220,11 +226,14 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa
|
|||||||
return doc.UpdateCopyFields()
|
return doc.UpdateCopyFields()
|
||||||
}
|
}
|
||||||
|
|
||||||
func StandardDocumentBuilder() DocumentBuilder {
|
func StandardDocumentBuilder(manifests []app.Manifest) DocumentBuilder {
|
||||||
return &standardDocumentBuilder{}
|
return &standardDocumentBuilder{selectableFields: SelectableFieldsForManifests(manifests)}
|
||||||
}
|
}
|
||||||
|
|
||||||
type standardDocumentBuilder struct{}
|
type standardDocumentBuilder struct {
|
||||||
|
// Maps "group/resource" (in lowercase) to list of selectable fields.
|
||||||
|
selectableFields map[string][]string
|
||||||
|
}
|
||||||
|
|
||||||
func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*IndexableDocument, error) {
|
func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*IndexableDocument, error) {
|
||||||
tmp := &unstructured.Unstructured{}
|
tmp := &unstructured.Unstructured{}
|
||||||
@@ -238,10 +247,36 @@ func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resour
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
doc := NewIndexableDocument(key, rv, obj)
|
sfKey := strings.ToLower(key.GetGroup() + "/" + key.GetResource())
|
||||||
|
selectableFields := buildSelectableFields(tmp, s.selectableFields[sfKey])
|
||||||
|
|
||||||
|
doc := NewIndexableDocument(key, rv, obj, selectableFields)
|
||||||
return doc, nil
|
return doc, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildSelectableFields(tmp *unstructured.Unstructured, fields []string) map[string]string {
|
||||||
|
result := map[string]string{}
|
||||||
|
|
||||||
|
for _, field := range fields {
|
||||||
|
path := strings.Split(field, ".")
|
||||||
|
val, ok, err := unstructured.NestedFieldNoCopy(tmp.Object, path...)
|
||||||
|
if err != nil || !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch v := val.(type) {
|
||||||
|
case string:
|
||||||
|
result[field] = v
|
||||||
|
case bool:
|
||||||
|
result[field] = strconv.FormatBool(v)
|
||||||
|
case int, float64:
|
||||||
|
result[field] = fmt.Sprintf("%v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
type searchableDocumentFields struct {
|
type searchableDocumentFields struct {
|
||||||
names []string
|
names []string
|
||||||
fields map[string]*resourceTableColumn
|
fields map[string]*resourceTableColumn
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
|
|
||||||
func TestStandardDocumentBuilder(t *testing.T) {
|
func TestStandardDocumentBuilder(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
builder := StandardDocumentBuilder()
|
builder := StandardDocumentBuilder(nil)
|
||||||
|
|
||||||
body, err := os.ReadFile("testdata/playlist-resource.json")
|
body, err := os.ReadFile("testdata/playlist-resource.json")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
45
pkg/storage/unified/resource/selectable_fields.go
Normal file
45
pkg/storage/unified/resource/selectable_fields.go
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
package resource
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/grafana/grafana-app-sdk/app"
|
||||||
|
|
||||||
|
folder "github.com/grafana/grafana/apps/folder/pkg/apis/manifestdata"
|
||||||
|
iam "github.com/grafana/grafana/apps/iam/pkg/apis"
|
||||||
|
)
|
||||||
|
|
||||||
|
func AppManifests() []app.Manifest {
|
||||||
|
return []app.Manifest{
|
||||||
|
iam.LocalManifest(),
|
||||||
|
folder.LocalManifest(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func SelectableFields() map[string][]string {
|
||||||
|
return SelectableFieldsForManifests(AppManifests())
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectableFieldsForManifests returns map of <group/kind> to list of selectable fields.
|
||||||
|
// Also <group/plural> is included as a key, pointing to the same fields.
|
||||||
|
func SelectableFieldsForManifests(manifests []app.Manifest) map[string][]string {
|
||||||
|
fields := map[string][]string{}
|
||||||
|
|
||||||
|
for _, m := range manifests {
|
||||||
|
group := m.ManifestData.Group
|
||||||
|
|
||||||
|
for _, version := range m.ManifestData.Versions {
|
||||||
|
for _, kind := range version.Kinds {
|
||||||
|
key := strings.ToLower(group + "/" + kind.Kind)
|
||||||
|
keyPlural := strings.ToLower(group + "/" + kind.Plural)
|
||||||
|
|
||||||
|
if len(kind.SelectableFields) > 0 {
|
||||||
|
fields[key] = kind.SelectableFields
|
||||||
|
fields[keyPlural] = kind.SelectableFields
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fields
|
||||||
|
}
|
||||||
@@ -12,6 +12,8 @@ import (
|
|||||||
|
|
||||||
"github.com/Masterminds/semver"
|
"github.com/Masterminds/semver"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
claims "github.com/grafana/authlib/types"
|
||||||
|
"github.com/grafana/dskit/backoff"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"go.opentelemetry.io/otel"
|
"go.opentelemetry.io/otel"
|
||||||
"go.opentelemetry.io/otel/attribute"
|
"go.opentelemetry.io/otel/attribute"
|
||||||
@@ -20,9 +22,6 @@ import (
|
|||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
|
||||||
claims "github.com/grafana/authlib/types"
|
|
||||||
"github.com/grafana/dskit/backoff"
|
|
||||||
|
|
||||||
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
"github.com/grafana/grafana/pkg/apimachinery/utils"
|
||||||
"github.com/grafana/grafana/pkg/apimachinery/validation"
|
"github.com/grafana/grafana/pkg/apimachinery/validation"
|
||||||
"github.com/grafana/grafana/pkg/infra/log"
|
"github.com/grafana/grafana/pkg/infra/log"
|
||||||
@@ -1043,6 +1042,68 @@ func (s *server) List(ctx context.Context, req *resourcepb.ListRequest) (*resour
|
|||||||
return rsp, nil
|
return rsp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Remove metadata.namespace filter from requirement fields, if it's present.
|
||||||
|
for ix := 0; ix < len(req.Options.Fields); {
|
||||||
|
v := req.Options.Fields[ix]
|
||||||
|
if v.Key == "metadata.namespace" && v.Operator == "=" {
|
||||||
|
if len(v.Values) == 1 && v.Values[0] == req.Options.Key.Namespace {
|
||||||
|
// Remove this requirement from fields, as it's implied by the key.namespace.
|
||||||
|
req.Options.Fields = append(req.Options.Fields[:ix], req.Options.Fields[ix+1:]...)
|
||||||
|
// Don't increment ix, as we're removing an element from the slice.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ix++
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: What to do about RV and version_match fields?
|
||||||
|
// If we get here, we're doing list with selectable fields. Let's do search instead, since
|
||||||
|
// we index all selectable fields, and fetch resulting documents one by one.
|
||||||
|
if s.search != nil && req.Source == resourcepb.ListRequest_STORE && (len(req.Options.Fields) > 0) {
|
||||||
|
if req.Options.Key.Namespace == "" {
|
||||||
|
return &resourcepb.ListResponse{
|
||||||
|
Error: NewBadRequestError("namespace must be specified for list with filter"),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
srq := &resourcepb.ResourceSearchRequest{
|
||||||
|
Options: req.Options,
|
||||||
|
//Federated: nil,
|
||||||
|
Limit: req.Limit,
|
||||||
|
// Offset: req.NextPageToken, // TODO
|
||||||
|
// Page: 0,
|
||||||
|
// Permission: 0, // Not needed, default is List
|
||||||
|
}
|
||||||
|
|
||||||
|
searchResp, err := s.search.Search(ctx, srq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
rsp := &resourcepb.ListResponse{}
|
||||||
|
// Using searchResp.GetResults().GetRows() will not panic if anything is nil on the path.
|
||||||
|
for _, row := range searchResp.GetResults().GetRows() {
|
||||||
|
// TODO: use batch reading
|
||||||
|
val, err := s.Read(ctx, &resourcepb.ReadRequest{
|
||||||
|
Key: row.Key,
|
||||||
|
ResourceVersion: row.ResourceVersion,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return &resourcepb.ListResponse{Error: AsErrorResult(err)}, nil
|
||||||
|
}
|
||||||
|
if len(val.Value) > 0 {
|
||||||
|
rsp.Items = append(rsp.Items, &resourcepb.ResourceWrapper{
|
||||||
|
Value: val.Value,
|
||||||
|
ResourceVersion: val.ResourceVersion,
|
||||||
|
})
|
||||||
|
if val.ResourceVersion > rsp.ResourceVersion {
|
||||||
|
rsp.ResourceVersion = val.ResourceVersion
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rsp, nil
|
||||||
|
}
|
||||||
|
|
||||||
if req.Limit < 1 {
|
if req.Limit < 1 {
|
||||||
req.Limit = 500 // default max 500 items in a page
|
req.Limit = 500 // default max 500 items in a page
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -59,7 +59,6 @@ type kvStorageBackend struct {
|
|||||||
dataStore *dataStore
|
dataStore *dataStore
|
||||||
eventStore *eventStore
|
eventStore *eventStore
|
||||||
notifier *notifier
|
notifier *notifier
|
||||||
builder DocumentBuilder
|
|
||||||
log logging.Logger
|
log logging.Logger
|
||||||
withPruner bool
|
withPruner bool
|
||||||
eventRetentionPeriod time.Duration
|
eventRetentionPeriod time.Duration
|
||||||
@@ -109,7 +108,6 @@ func NewKVStorageBackend(opts KVBackendOptions) (StorageBackend, error) {
|
|||||||
eventStore: eventStore,
|
eventStore: eventStore,
|
||||||
notifier: newNotifier(eventStore, notifierOptions{}),
|
notifier: newNotifier(eventStore, notifierOptions{}),
|
||||||
snowflake: s,
|
snowflake: s,
|
||||||
builder: StandardDocumentBuilder(), // For now we use the standard document builder.
|
|
||||||
log: &logging.NoOpLogger{}, // Make this configurable
|
log: &logging.NoOpLogger{}, // Make this configurable
|
||||||
eventRetentionPeriod: eventRetentionPeriod,
|
eventRetentionPeriod: eventRetentionPeriod,
|
||||||
eventPruningInterval: eventPruningInterval,
|
eventPruningInterval: eventPruningInterval,
|
||||||
|
|||||||
@@ -93,6 +93,8 @@ type bleveBackend struct {
|
|||||||
|
|
||||||
indexMetrics *resource.BleveIndexMetrics
|
indexMetrics *resource.BleveIndexMetrics
|
||||||
|
|
||||||
|
selectableFields map[string][]string
|
||||||
|
|
||||||
bgTasksCancel func()
|
bgTasksCancel func()
|
||||||
bgTasksWg sync.WaitGroup
|
bgTasksWg sync.WaitGroup
|
||||||
}
|
}
|
||||||
@@ -140,6 +142,7 @@ func NewBleveBackend(opts BleveOptions, indexMetrics *resource.BleveIndexMetrics
|
|||||||
opts: opts,
|
opts: opts,
|
||||||
ownsIndexFn: ownFn,
|
ownsIndexFn: ownFn,
|
||||||
indexMetrics: indexMetrics,
|
indexMetrics: indexMetrics,
|
||||||
|
selectableFields: resource.SelectableFields(),
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
@@ -366,7 +369,9 @@ func (b *bleveBackend) BuildIndex(
|
|||||||
attribute.String("reason", indexBuildReason),
|
attribute.String("reason", indexBuildReason),
|
||||||
)
|
)
|
||||||
|
|
||||||
mapper, err := GetBleveMappings(fields)
|
selectableFields := b.selectableFields[fmt.Sprintf("%s/%s", key.Group, key.Resource)]
|
||||||
|
|
||||||
|
mapper, err := GetBleveMappings(fields, selectableFields)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -459,7 +464,7 @@ func (b *bleveBackend) BuildIndex(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Batch all the changes
|
// Batch all the changes
|
||||||
idx := b.newBleveIndex(key, index, newIndexType, fields, allFields, standardSearchFields, updater, b.log.New("namespace", key.Namespace, "group", key.Group, "resource", key.Resource))
|
idx := b.newBleveIndex(key, index, newIndexType, fields, allFields, standardSearchFields, selectableFields, updater, b.log.New("namespace", key.Namespace, "group", key.Group, "resource", key.Resource))
|
||||||
|
|
||||||
if build {
|
if build {
|
||||||
if b.indexMetrics != nil {
|
if b.indexMetrics != nil {
|
||||||
@@ -701,6 +706,7 @@ type bleveIndex struct {
|
|||||||
|
|
||||||
standard resource.SearchableDocumentFields
|
standard resource.SearchableDocumentFields
|
||||||
fields resource.SearchableDocumentFields
|
fields resource.SearchableDocumentFields
|
||||||
|
selectableFields []string
|
||||||
|
|
||||||
indexStorage string // memory or file, used when updating metrics
|
indexStorage string // memory or file, used when updating metrics
|
||||||
|
|
||||||
@@ -736,6 +742,7 @@ func (b *bleveBackend) newBleveIndex(
|
|||||||
fields resource.SearchableDocumentFields,
|
fields resource.SearchableDocumentFields,
|
||||||
allFields []*resourcepb.ResourceTableColumnDefinition,
|
allFields []*resourcepb.ResourceTableColumnDefinition,
|
||||||
standardSearchFields resource.SearchableDocumentFields,
|
standardSearchFields resource.SearchableDocumentFields,
|
||||||
|
selectableFields []string,
|
||||||
updaterFn resource.UpdateFn,
|
updaterFn resource.UpdateFn,
|
||||||
logger log.Logger,
|
logger log.Logger,
|
||||||
) *bleveIndex {
|
) *bleveIndex {
|
||||||
@@ -745,6 +752,7 @@ func (b *bleveBackend) newBleveIndex(
|
|||||||
indexStorage: newIndexType,
|
indexStorage: newIndexType,
|
||||||
fields: fields,
|
fields: fields,
|
||||||
allFields: allFields,
|
allFields: allFields,
|
||||||
|
selectableFields: selectableFields,
|
||||||
standard: standardSearchFields,
|
standard: standardSearchFields,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
updaterFn: updaterFn,
|
updaterFn: updaterFn,
|
||||||
@@ -1215,7 +1223,11 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R
|
|||||||
// filters
|
// filters
|
||||||
if len(req.Options.Fields) > 0 {
|
if len(req.Options.Fields) > 0 {
|
||||||
for _, v := range req.Options.Fields {
|
for _, v := range req.Options.Fields {
|
||||||
q, err := requirementQuery(v, "")
|
prefix := ""
|
||||||
|
if b.isSelectableField(v.Key) {
|
||||||
|
prefix = "selectable_fields."
|
||||||
|
}
|
||||||
|
q, err := requirementQuery(v, prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -1787,6 +1799,15 @@ func (b *bleveIndex) hitsToTable(ctx context.Context, selectFields []string, hit
|
|||||||
return table, nil
|
return table, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *bleveIndex) isSelectableField(key string) bool {
|
||||||
|
for _, f := range b.selectableFields {
|
||||||
|
if key == f {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
func getAllFields(standard resource.SearchableDocumentFields, custom resource.SearchableDocumentFields) ([]*resourcepb.ResourceTableColumnDefinition, error) {
|
func getAllFields(standard resource.SearchableDocumentFields, custom resource.SearchableDocumentFields) ([]*resourcepb.ResourceTableColumnDefinition, error) {
|
||||||
fields := []*resourcepb.ResourceTableColumnDefinition{
|
fields := []*resourcepb.ResourceTableColumnDefinition{
|
||||||
standard.Field(resource.SEARCH_FIELD_ID),
|
standard.Field(resource.SEARCH_FIELD_ID),
|
||||||
|
|||||||
@@ -10,19 +10,19 @@ import (
|
|||||||
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
|
||||||
)
|
)
|
||||||
|
|
||||||
func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
|
func GetBleveMappings(fields resource.SearchableDocumentFields, selectableFields []string) (mapping.IndexMapping, error) {
|
||||||
mapper := bleve.NewIndexMapping()
|
mapper := bleve.NewIndexMapping()
|
||||||
|
|
||||||
err := RegisterCustomAnalyzers(mapper)
|
err := RegisterCustomAnalyzers(mapper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
mapper.DefaultMapping = getBleveDocMappings(fields)
|
mapper.DefaultMapping = getBleveDocMappings(fields, selectableFields)
|
||||||
|
|
||||||
return mapper, nil
|
return mapper, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getBleveDocMappings(fields resource.SearchableDocumentFields) *mapping.DocumentMapping {
|
func getBleveDocMappings(fields resource.SearchableDocumentFields, selectableFields []string) *mapping.DocumentMapping {
|
||||||
mapper := bleve.NewDocumentStaticMapping()
|
mapper := bleve.NewDocumentStaticMapping()
|
||||||
|
|
||||||
nameMapping := &mapping.FieldMapping{
|
nameMapping := &mapping.FieldMapping{
|
||||||
@@ -165,5 +165,73 @@ func getBleveDocMappings(fields resource.SearchableDocumentFields) *mapping.Docu
|
|||||||
|
|
||||||
mapper.AddSubDocumentMapping("fields", fieldMapper)
|
mapper.AddSubDocumentMapping("fields", fieldMapper)
|
||||||
|
|
||||||
|
selectableFieldsMapper := bleve.NewDocumentStaticMapping()
|
||||||
|
for _, field := range selectableFields {
|
||||||
|
selectableFieldsMapper.AddFieldMappingsAt(field, &mapping.FieldMapping{
|
||||||
|
Name: field,
|
||||||
|
Type: "text",
|
||||||
|
Analyzer: keyword.Name,
|
||||||
|
Store: false,
|
||||||
|
Index: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
mapper.AddSubDocumentMapping("selectable_fields", selectableFieldsMapper)
|
||||||
|
|
||||||
return mapper
|
return mapper
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
Here's a tree representation of the field mappings in pkg/storage/unified/search/bleve_mappings.go:
|
||||||
|
|
||||||
|
Document Root (DefaultMapping)
|
||||||
|
│
|
||||||
|
├── name [text, keyword analyzer]
|
||||||
|
│
|
||||||
|
├── title_phrase [keyword, not stored]
|
||||||
|
│
|
||||||
|
├── title [3 mappings]
|
||||||
|
│ ├── [1] standard analyzer, stored
|
||||||
|
│ ├── [2] TITLE_ANALYZER (edge ngram), not stored
|
||||||
|
│ └── [3] keyword, not stored
|
||||||
|
│
|
||||||
|
├── description [text, stored]
|
||||||
|
│
|
||||||
|
├── tags [text, keyword analyzer, stored, includeInAll]
|
||||||
|
│
|
||||||
|
├── folder [text, keyword analyzer, stored, includeInAll, docValues]
|
||||||
|
│
|
||||||
|
├── managedBy [text, keyword analyzer, not stored]
|
||||||
|
│
|
||||||
|
├── source/ [sub-document]
|
||||||
|
│ ├── path [text, keyword analyzer, stored]
|
||||||
|
│ ├── checksum [text, keyword analyzer, stored]
|
||||||
|
│ └── timestampMillis [numeric]
|
||||||
|
│
|
||||||
|
├── manager/ [sub-document]
|
||||||
|
│ ├── kind [text, keyword analyzer, stored, includeInAll]
|
||||||
|
│ └── id [text, keyword analyzer, stored, includeInAll]
|
||||||
|
│
|
||||||
|
├── reference/ [sub-document, default analyzer: keyword]
|
||||||
|
│ └── (dynamic fields inherit keyword analyzer)
|
||||||
|
│
|
||||||
|
├── labels/ [sub-document]
|
||||||
|
│ └── (dynamic fields)
|
||||||
|
│
|
||||||
|
└── fields/ [sub-document]
|
||||||
|
└── (conditional mappings)
|
||||||
|
├── {filterable string fields} [keyword, stored]
|
||||||
|
└── {other fields} [dynamically mapped by Bleve]
|
||||||
|
|
||||||
|
Key observations:
|
||||||
|
|
||||||
|
- Root level has standard searchable fields (name, title, description, tags, folder)
|
||||||
|
- title has 3 analyzers applied: standard (for word search), edge ngram (for prefix search), and keyword (for phrase sorting)
|
||||||
|
- source/, manager/: Static sub-documents with explicitly mapped fields
|
||||||
|
- reference/: Dynamic sub-document with keyword default analyzer (line 142)
|
||||||
|
- labels/, fields/: Dynamic sub-documents where Bleve auto-detects field types at index time
|
||||||
|
|
||||||
|
References:
|
||||||
|
- Main mapping function: pkg/storage/unified/search/bleve_mappings.go:25-169
|
||||||
|
- Sub-document mappings: lines 88-143
|
||||||
|
- Dynamic fields handling: lines 148-166
|
||||||
|
*/
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func TestDocumentMapping(t *testing.T) {
|
func TestDocumentMapping(t *testing.T) {
|
||||||
mappings, err := search.GetBleveMappings(nil)
|
mappings, err := search.GetBleveMappings(nil, nil)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
data := resource.IndexableDocument{
|
data := resource.IndexableDocument{
|
||||||
Title: "title",
|
Title: "title",
|
||||||
|
|||||||
@@ -264,7 +264,7 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
|
|||||||
summary.UID = obj.GetName()
|
summary.UID = obj.GetName()
|
||||||
summary.ID = obj.GetDeprecatedInternalID() // nolint:staticcheck
|
summary.ID = obj.GetDeprecatedInternalID() // nolint:staticcheck
|
||||||
|
|
||||||
doc := resource.NewIndexableDocument(key, rv, obj)
|
doc := resource.NewIndexableDocument(key, rv, obj, nil)
|
||||||
doc.Title = summary.Title
|
doc.Title = summary.Title
|
||||||
doc.Description = summary.Description
|
doc.Description = summary.Description
|
||||||
doc.Tags = summary.Tags
|
doc.Tags = summary.Tags
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ func TestDashboardDocumentBuilder(t *testing.T) {
|
|||||||
"aaa",
|
"aaa",
|
||||||
})
|
})
|
||||||
|
|
||||||
builder = resource.StandardDocumentBuilder()
|
builder = resource.StandardDocumentBuilder(nil)
|
||||||
doSnapshotTests(t, builder, "folder", &resourcepb.ResourceKey{
|
doSnapshotTests(t, builder, "folder", &resourcepb.ResourceKey{
|
||||||
Namespace: "default",
|
Namespace: "default",
|
||||||
Group: "folder.grafana.app",
|
Group: "folder.grafana.app",
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ func (u *extGroupMappingDocumentBuilder) BuildDocument(ctx context.Context, key
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
doc := resource.NewIndexableDocument(key, rv, obj)
|
doc := resource.NewIndexableDocument(key, rv, obj, nil)
|
||||||
|
|
||||||
doc.Fields = make(map[string]any)
|
doc.Fields = make(map[string]any)
|
||||||
if extGroupMapping.Spec.TeamRef.Name != "" {
|
if extGroupMapping.Spec.TeamRef.Name != "" {
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ func (t *teamSearchBuilder) BuildDocument(ctx context.Context, key *resourcepb.R
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
doc := resource.NewIndexableDocument(key, rv, obj)
|
doc := resource.NewIndexableDocument(key, rv, obj, nil)
|
||||||
|
|
||||||
doc.Fields = make(map[string]any)
|
doc.Fields = make(map[string]any)
|
||||||
if team.Spec.Email != "" {
|
if team.Spec.Email != "" {
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ func (u *userDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
doc := resource.NewIndexableDocument(key, rv, obj)
|
doc := resource.NewIndexableDocument(key, rv, obj, nil)
|
||||||
|
|
||||||
doc.Fields = make(map[string]any)
|
doc.Fields = make(map[string]any)
|
||||||
if user.Spec.Email != "" {
|
if user.Spec.Email != "" {
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBui
|
|||||||
|
|
||||||
result := []resource.DocumentBuilderInfo{
|
result := []resource.DocumentBuilderInfo{
|
||||||
{
|
{
|
||||||
Builder: resource.StandardDocumentBuilder(),
|
Builder: resource.StandardDocumentBuilder(resource.AppManifests()),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return append(result, all...), nil
|
return append(result, all...), nil
|
||||||
|
|||||||
Reference in New Issue
Block a user