Gives resource server a search client so storage-api can call search

apps/folder/kinds/folder.cue

@@ -8,6 +8,12 @@ foldersV1beta1: {
 		spec: {
 			title:    string
 			description?: string
+			foo: bool
+			bar: int
 		}
 	}
-}
+
+	selectableFields: [
+		"spec.title",
+	]
+}

apps/folder/pkg/apis/folder/v1beta1/folder_schema_gen.go

@@ -5,13 +5,26 @@
 package v1beta1
 
 import (
+	"errors"
+
 	"github.com/grafana/grafana-app-sdk/resource"
 )
 
 // schema is unexported to prevent accidental overwrites
 var (
 	schemaFolder = resource.NewSimpleSchema("folder.grafana.app", "v1beta1", NewFolder(), &FolderList{}, resource.WithKind("Folder"),
-		resource.WithPlural("folders"), resource.WithScope(resource.NamespacedScope))
+		resource.WithPlural("folders"), resource.WithScope(resource.NamespacedScope), resource.WithSelectableFields([]resource.SelectableField{resource.SelectableField{
+			FieldSelector: "spec.title",
+			FieldValueFunc: func(o resource.Object) (string, error) {
+				cast, ok := o.(*Folder)
+				if !ok {
+					return "", errors.New("provided object must be of type *Folder")
+				}
+
+				return cast.Spec.Title, nil
+			},
+		},
+		}))
 	kindFolder = resource.Kind{
 		Schema: schemaFolder,
 		Codecs: map[resource.KindEncoding]resource.Codec{

apps/folder/pkg/apis/manifestdata/folder_manifest.go

@@ -18,6 +18,8 @@ import (
 	v1beta1 "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
 )
 
+var ()
+
 var appManifestData = app.ManifestData{
 	AppName:          "folder",
 	Group:            "folder.grafana.app",
@@ -32,6 +34,9 @@ var appManifestData = app.ManifestData{
 					Plural:     "Folders",
 					Scope:      "Namespaced",
 					Conversion: false,
+					SelectableFields: []string{
+						"spec.title",
+					},
 				},
 			},
 			Routes: app.ManifestVersionRoutes{

apps/iam/kinds/user.cue

@@ -14,7 +14,7 @@ userKind: {
 }
 
 userv0alpha1: userKind & {
-	// TODO: Uncomment this when User will be added to ManagedKinds 
+	// TODO: Uncomment this when User will be added to ManagedKinds
 	// validation: {
 	// 	operations: [
 	// 		"CREATE",

apps/iam/pkg/apis/iam_manifest.go

@@ -74,6 +74,10 @@ var appManifestData = app.ManifestData{
 					Plural:     "Users",
 					Scope:      "Namespaced",
 					Conversion: false,
+					SelectableFields: []string{
+						"spec.email",
+						"spec.login",
+					},
 				},
 
 				{

pkg/apiserver/registry/generic/storage.go

@@ -4,11 +4,24 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/generic/registry"
+	"k8s.io/apiserver/pkg/storage"
 
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
 )
 
-func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter) (*registry.Store, error) {
+type registryStoreOptions struct {
+	attrFunc storage.AttrFunc
+}
+
+type OptionFn func(*registryStoreOptions)
+
+func WithAttrFunc(attrFunc storage.AttrFunc) OptionFn {
+	return func(opts *registryStoreOptions) {
+		opts.attrFunc = attrFunc
+	}
+}
+
+func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, optsGetter generic.RESTOptionsGetter, options ...OptionFn) (*registry.Store, error) {
 	gv := resourceInfo.GroupVersion()
 	gv.Version = runtime.APIVersionInternal
 	strategy := NewStrategy(scheme, gv)
@@ -20,7 +33,7 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o
 		NewListFunc:               resourceInfo.NewListFunc,
 		KeyRootFunc:               KeyRootFunc(resourceInfo.GroupResource()),
 		KeyFunc:                   NamespaceKeyFunc(resourceInfo.GroupResource()),
-		PredicateFunc:             Matcher,
+		//PredicateFunc:             Matcher,
 		DefaultQualifiedResource:  resourceInfo.GroupResource(),
 		SingularQualifiedResource: resourceInfo.SingularGroupResource(),
 		TableConvertor:            resourceInfo.TableConverter(),
@@ -28,8 +41,16 @@ func NewRegistryStore(scheme *runtime.Scheme, resourceInfo utils.ResourceInfo, o
 		UpdateStrategy:            strategy,
 		DeleteStrategy:            strategy,
 	}
-	options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs}
-	if err := store.CompleteWithOptions(options); err != nil {
+
+	opts := &registryStoreOptions{
+		attrFunc: GetAttrs,
+	}
+	for _, opt := range options {
+		opt(opts)
+	}
+
+	o := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: opts.attrFunc}
+	if err := store.CompleteWithOptions(o); err != nil {
 		return nil, err
 	}
 	return store, nil

pkg/registry/apis/folders/register.go

@@ -8,6 +8,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
@@ -22,6 +24,8 @@ import (
 	authlib "github.com/grafana/authlib/types"
 	"github.com/grafana/grafana-app-sdk/logging"
 
+	sdkres "github.com/grafana/grafana-app-sdk/resource"
+
 	folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
 	"github.com/grafana/grafana/apps/iam/pkg/reconcilers"
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
@@ -129,6 +133,29 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
 		Version: runtime.APIVersionInternal,
 	})
 
+	kinds := []sdkres.Kind{folders.FolderKind()}
+	for _, kind := range kinds {
+		gvk := gv.WithKind(kind.Kind())
+		err := scheme.AddFieldLabelConversionFunc(
+			gvk,
+			func(label, value string) (string, string, error) {
+				if label == "metadata.name" || label == "metadata.namespace" {
+					return label, value, nil
+				}
+				fields := kind.SelectableFields()
+				for _, field := range fields {
+					if field.FieldSelector == label {
+						return label, value, nil
+					}
+				}
+				return "", "", fmt.Errorf("field label not supported for %s: %s", gvk, label)
+			},
+		)
+		if err != nil {
+			return err
+		}
+	}
+
 	// If multiple versions exist, then register conversions from zz_generated.conversion.go
 	// if err := playlist.RegisterConversions(scheme); err != nil {
 	//   return err
@@ -137,6 +164,26 @@ func (b *FolderAPIBuilder) InstallSchema(scheme *runtime.Scheme) error {
 	return scheme.SetVersionPriority(gv)
 }
 
+// TODO: work with all kinds from schema, not just one.
+func (b *FolderAPIBuilder) BuildGetAttrsFn(k sdkres.Kind) func(obj runtime.Object) (labels.Set, fields.Set, error) {
+	return func(obj runtime.Object) (labels.Set, fields.Set, error) {
+		if robj, ok := obj.(sdkres.Object); !ok {
+			return nil, nil, fmt.Errorf("not a resource.Object")
+		} else {
+			fieldsSet := fields.Set{}
+
+			for _, f := range k.SelectableFields() {
+				v, err := f.FieldValueFunc(robj)
+				if err != nil {
+					return nil, nil, err
+				}
+				fieldsSet[f.FieldSelector] = v
+			}
+			return robj.GetLabels(), fieldsSet, nil
+		}
+	}
+}
+
 func (b *FolderAPIBuilder) AllowedV0Alpha1Resources() []string {
 	return nil
 }
@@ -148,10 +195,11 @@ func (b *FolderAPIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.API
 		Permissions:                 b.setDefaultFolderPermissions,
 	})
 
-	unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter)
+	unified, err := grafanaregistry.NewRegistryStore(opts.Scheme, resourceInfo, opts.OptsGetter, grafanaregistry.WithAttrFunc(b.BuildGetAttrsFn(folders.FolderKind())))
 	if err != nil {
 		return err
 	}
+
 	b.registerPermissionHooks(unified)
 	b.storage = unified
 

pkg/server/module_server.go

@@ -13,6 +13,9 @@ import (
 	"github.com/grafana/dskit/kv"
 	"github.com/grafana/dskit/ring"
 	ringclient "github.com/grafana/dskit/ring/client"
+	"github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/grafana/grafana/pkg/storage/unified"
+	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/urfave/cli/v2"
 
@@ -76,6 +79,22 @@ func newModuleServer(opts Options,
 ) (*ModuleServer, error) {
 	rootCtx, shutdownFn := context.WithCancel(context.Background())
 
+	// TODO should inject this with Wire
+	apiserverCfg := cfg.SectionWithEnvOverrides("grafana-apiserver")
+	searchServerAddress := apiserverCfg.Key("search_server_address").MustString("")
+	var searchClient resourcepb.ResourceIndexClient
+	var err error
+	if searchServerAddress != "" {
+		storageOptions := options.StorageOptions{
+			SearchServerAddress: searchServerAddress,
+		}
+		searchClient, err = unified.NewSearchClient(storageOptions, features)
+		if err != nil {
+			shutdownFn()
+			return nil, fmt.Errorf("failed to create search client: %w", err)
+		}
+	}
+
 	s := &ModuleServer{
 		opts:             opts,
 		apiOpts:          apiOpts,
@@ -96,6 +115,7 @@ func newModuleServer(opts Options,
 		license:          license,
 		moduleRegisterer: moduleRegisterer,
 		storageBackend:   storageBackend,
+		searchClient:     searchClient,
 		hooksService:     hooksService,
 	}
 
@@ -119,6 +139,7 @@ type ModuleServer struct {
 	isInitialized    bool
 	mtx              sync.Mutex
 	storageBackend   resource.StorageBackend
+	searchClient     resourcepb.ResourceIndexClient
 	storageMetrics   *resource.StorageMetrics
 	indexMetrics     *resource.BleveIndexMetrics
 	license          licensing.Licensing
@@ -202,7 +223,7 @@ func (s *ModuleServer) Run() error {
 		if err != nil {
 			return nil, err
 		}
-		return sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, nil, s.log, s.registerer, docBuilders, s.storageMetrics, s.indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend)
+		return sql.ProvideUnifiedStorageGrpcService(s.cfg, s.features, nil, s.log, s.registerer, docBuilders, s.storageMetrics, s.indexMetrics, s.searchServerRing, s.MemberlistKVConfig, s.httpServerRouter, s.storageBackend, s.searchClient)
 	})
 
 	m.RegisterModule(modules.ZanzanaServer, func() (services.Service, error) {

pkg/storage/unified/client.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	badger "github.com/dgraph-io/badger/v4"
+	"github.com/fullstorydev/grpchan"
+	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
 	otgrpc "github.com/opentracing-contrib/go-grpc"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
@@ -31,6 +33,7 @@ import (
 	"github.com/grafana/grafana/pkg/storage/legacysql"
 	"github.com/grafana/grafana/pkg/storage/unified/federated"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
+	grpcUtils "github.com/grafana/grafana/pkg/storage/unified/resource/grpc"
 	"github.com/grafana/grafana/pkg/storage/unified/search"
 	"github.com/grafana/grafana/pkg/storage/unified/sql"
 	"github.com/grafana/grafana/pkg/util/scheduler"
@@ -91,6 +94,27 @@ func ProvideUnifiedStorageClient(opts *Options,
 	return client, err
 }
 
+// TODO use wire to provide to module server
+func NewSearchClient(opts options.StorageOptions, features featuremgmt.FeatureToggles) (resourcepb.ResourceIndexClient, error) {
+	if opts.SearchServerAddress == "" {
+		return nil, fmt.Errorf("expecting address for search server")
+	}
+
+	var (
+		conn    grpc.ClientConnInterface
+		err     error
+		metrics = newClientMetrics(prometheus.NewRegistry())
+	)
+
+	conn, err = newGrpcConn(opts.SearchServerAddress, metrics, features, opts.GrpcClientKeepaliveTime)
+	if err != nil {
+		return nil, err
+	}
+
+	cc := grpchan.InterceptClientConn(conn, grpcUtils.UnaryClientInterceptor, grpcUtils.StreamClientInterceptor)
+	return resourcepb.NewResourceIndexClient(cc), nil
+}
+
 func newClient(opts options.StorageOptions,
 	cfg *setting.Cfg,
 	features featuremgmt.FeatureToggles,

pkg/storage/unified/resource/document.go

@@ -3,9 +3,11 @@ package resource
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 	"sync"
 
+	"github.com/grafana/grafana-app-sdk/app"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 
@@ -101,6 +103,9 @@ type IndexableDocument struct {
 	// metadata, annotations, or external data linked at index time
 	Fields map[string]any `json:"fields,omitempty"`
 
+	// Automatically indexed selectable fields, used for field-based filtering when listing.
+	SelectableFields map[string]string `json:"selectable_fields,omitempty"`
+
 	// Maintain a list of resource references.
 	// Someday this will likely be part of https://github.com/grafana/gamma
 	References ResourceReferences `json:"references,omitempty"`
@@ -175,7 +180,7 @@ func (m ResourceReferences) Less(i, j int) bool {
 }
 
 // Create a new indexable document based on a generic k8s resource
-func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.GrafanaMetaAccessor) *IndexableDocument {
+func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.GrafanaMetaAccessor, selectableFields map[string]string) *IndexableDocument {
 	title := obj.FindTitle(key.Name)
 	if title == key.Name {
 		// TODO: something wrong with FindTitle
@@ -191,14 +196,15 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa
 		}
 	}
 	doc := &IndexableDocument{
-		Key:       key,
-		RV:        rv,
-		Name:      key.Name,
-		Title:     title, // We always want *something* to display
-		Labels:    obj.GetLabels(),
-		Folder:    obj.GetFolder(),
-		CreatedBy: obj.GetCreatedBy(),
-		UpdatedBy: obj.GetUpdatedBy(),
+		Key:              key,
+		RV:               rv,
+		Name:             key.Name,
+		Title:            title, // We always want *something* to display
+		Labels:           obj.GetLabels(),
+		Folder:           obj.GetFolder(),
+		CreatedBy:        obj.GetCreatedBy(),
+		UpdatedBy:        obj.GetUpdatedBy(),
+		SelectableFields: selectableFields,
 	}
 	m, ok := obj.GetManagerProperties()
 	if ok {
@@ -220,11 +226,14 @@ func NewIndexableDocument(key *resourcepb.ResourceKey, rv int64, obj utils.Grafa
 	return doc.UpdateCopyFields()
 }
 
-func StandardDocumentBuilder() DocumentBuilder {
-	return &standardDocumentBuilder{}
+func StandardDocumentBuilder(manifests []app.Manifest) DocumentBuilder {
+	return &standardDocumentBuilder{selectableFields: SelectableFieldsForManifests(manifests)}
 }
 
-type standardDocumentBuilder struct{}
+type standardDocumentBuilder struct {
+	// Maps "group/resource" (in lowercase) to list of selectable fields.
+	selectableFields map[string][]string
+}
 
 func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb.ResourceKey, rv int64, value []byte) (*IndexableDocument, error) {
 	tmp := &unstructured.Unstructured{}
@@ -238,10 +247,36 @@ func (s *standardDocumentBuilder) BuildDocument(ctx context.Context, key *resour
 		return nil, err
 	}
 
-	doc := NewIndexableDocument(key, rv, obj)
+	sfKey := strings.ToLower(key.GetGroup() + "/" + key.GetResource())
+	selectableFields := buildSelectableFields(tmp, s.selectableFields[sfKey])
+
+	doc := NewIndexableDocument(key, rv, obj, selectableFields)
 	return doc, nil
 }
 
+func buildSelectableFields(tmp *unstructured.Unstructured, fields []string) map[string]string {
+	result := map[string]string{}
+
+	for _, field := range fields {
+		path := strings.Split(field, ".")
+		val, ok, err := unstructured.NestedFieldNoCopy(tmp.Object, path...)
+		if err != nil || !ok {
+			continue
+		}
+
+		switch v := val.(type) {
+		case string:
+			result[field] = v
+		case bool:
+			result[field] = strconv.FormatBool(v)
+		case int, float64:
+			result[field] = fmt.Sprintf("%v", v)
+		}
+	}
+
+	return result
+}
+
 type searchableDocumentFields struct {
 	names  []string
 	fields map[string]*resourceTableColumn

pkg/storage/unified/resource/document_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestStandardDocumentBuilder(t *testing.T) {
 	ctx := context.Background()
-	builder := StandardDocumentBuilder()
+	builder := StandardDocumentBuilder(nil)
 
 	body, err := os.ReadFile("testdata/playlist-resource.json")
 	require.NoError(t, err)

pkg/storage/unified/resource/selectable_fields.go

@@ -0,0 +1,45 @@
+package resource
+
+import (
+	"strings"
+
+	"github.com/grafana/grafana-app-sdk/app"
+
+	folder "github.com/grafana/grafana/apps/folder/pkg/apis/manifestdata"
+	iam "github.com/grafana/grafana/apps/iam/pkg/apis"
+)
+
+func AppManifests() []app.Manifest {
+	return []app.Manifest{
+		iam.LocalManifest(),
+		folder.LocalManifest(),
+	}
+}
+
+func SelectableFields() map[string][]string {
+	return SelectableFieldsForManifests(AppManifests())
+}
+
+// SelectableFieldsForManifests returns map of <group/kind> to list of selectable fields.
+// Also <group/plural> is included as a key, pointing to the same fields.
+func SelectableFieldsForManifests(manifests []app.Manifest) map[string][]string {
+	fields := map[string][]string{}
+
+	for _, m := range manifests {
+		group := m.ManifestData.Group
+
+		for _, version := range m.ManifestData.Versions {
+			for _, kind := range version.Kinds {
+				key := strings.ToLower(group + "/" + kind.Kind)
+				keyPlural := strings.ToLower(group + "/" + kind.Plural)
+
+				if len(kind.SelectableFields) > 0 {
+					fields[key] = kind.SelectableFields
+					fields[keyPlural] = kind.SelectableFields
+				}
+			}
+		}
+	}
+
+	return fields
+}

pkg/storage/unified/resource/server.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/Masterminds/semver"
 	"github.com/google/uuid"
+	claims "github.com/grafana/authlib/types"
+	"github.com/grafana/dskit/backoff"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -20,9 +22,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
-	claims "github.com/grafana/authlib/types"
-	"github.com/grafana/dskit/backoff"
-
 	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	"github.com/grafana/grafana/pkg/apimachinery/validation"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -223,6 +222,9 @@ type ResourceServerOptions struct {
 	// Search options
 	Search SearchOptions
 
+	// to be used by storage
+	SearchClient resourcepb.ResourceIndexClient
+
 	// Quota service
 	OverridesService *OverridesService
 
@@ -348,6 +350,7 @@ func NewResourceServer(opts ResourceServerOptions) (*server, error) {
 		queue:            opts.QOSQueue,
 		queueConfig:      opts.QOSConfig,
 		overridesService: opts.OverridesService,
+		searchClient:     opts.SearchClient,
 
 		artificialSuccessfulWriteDelay: opts.Search.IndexMinUpdateInterval,
 	}
@@ -387,6 +390,9 @@ type server struct {
 	indexMetrics     *BleveIndexMetrics
 	overridesService *OverridesService
 
+	// only to be used with storage server for field selector search
+	searchClient resourcepb.ResourceIndexClient
+
 	// Background watch task -- this has permissions for everything
 	ctx         context.Context
 	cancel      context.CancelFunc
@@ -1043,6 +1049,75 @@ func (s *server) List(ctx context.Context, req *resourcepb.ListRequest) (*resour
 		return rsp, nil
 	}
 
+	// Remove metadata.namespace filter from requirement fields, if it's present.
+	for ix := 0; ix < len(req.Options.Fields); {
+		v := req.Options.Fields[ix]
+		if v.Key == "metadata.namespace" && v.Operator == "=" {
+			if len(v.Values) == 1 && v.Values[0] == req.Options.Key.Namespace {
+				// Remove this requirement from fields, as it's implied by the key.namespace.
+				req.Options.Fields = append(req.Options.Fields[:ix], req.Options.Fields[ix+1:]...)
+				// Don't increment ix, as we're removing an element from the slice.
+				continue
+			}
+		}
+		ix++
+	}
+
+	// TODO: What to do about RV and version_match fields?
+	// If we get here, we're doing list with selectable fields. Let's do search instead, since
+	// we index all selectable fields, and fetch resulting documents one by one.
+
+	if (s.search != nil || s.searchClient != nil) && req.Source == resourcepb.ListRequest_STORE && (len(req.Options.Fields) > 0) {
+		if req.Options.Key.Namespace == "" {
+			return &resourcepb.ListResponse{
+				Error: NewBadRequestError("namespace must be specified for list with filter"),
+			}, nil
+		}
+
+		srq := &resourcepb.ResourceSearchRequest{
+			Options: req.Options,
+			//Federated: nil,
+			Limit: req.Limit,
+			// Offset:     req.NextPageToken, // TODO
+			// Page:       0,
+			// Permission: 0, // Not needed, default is List
+		}
+
+		var searchResp *resourcepb.ResourceSearchResponse
+		var err error
+		if s.searchClient != nil {
+			searchResp, err = s.searchClient.Search(ctx, srq)
+		} else {
+			searchResp, err = s.search.Search(ctx, srq)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		rsp := &resourcepb.ListResponse{}
+		// Using searchResp.GetResults().GetRows() will not panic if anything is nil on the path.
+		for _, row := range searchResp.GetResults().GetRows() {
+			// TODO: use batch reading
+			val, err := s.Read(ctx, &resourcepb.ReadRequest{
+				Key:             row.Key,
+				ResourceVersion: row.ResourceVersion,
+			})
+			if err != nil {
+				return &resourcepb.ListResponse{Error: AsErrorResult(err)}, nil
+			}
+			if len(val.Value) > 0 {
+				rsp.Items = append(rsp.Items, &resourcepb.ResourceWrapper{
+					Value:           val.Value,
+					ResourceVersion: val.ResourceVersion,
+				})
+				if val.ResourceVersion > rsp.ResourceVersion {
+					rsp.ResourceVersion = val.ResourceVersion
+				}
+			}
+		}
+		return rsp, nil
+	}
+
 	if req.Limit < 1 {
 		req.Limit = 500 // default max 500 items in a page
 	}

pkg/storage/unified/resource/storage_backend.go

@@ -59,7 +59,6 @@ type kvStorageBackend struct {
 	dataStore                    *dataStore
 	eventStore                   *eventStore
 	notifier                     *notifier
-	builder                      DocumentBuilder
 	log                          logging.Logger
 	withPruner                   bool
 	eventRetentionPeriod         time.Duration
@@ -109,8 +108,7 @@ func NewKVStorageBackend(opts KVBackendOptions) (StorageBackend, error) {
 		eventStore:                   eventStore,
 		notifier:                     newNotifier(eventStore, notifierOptions{}),
 		snowflake:                    s,
-		builder:                      StandardDocumentBuilder(), // For now we use the standard document builder.
-		log:                          &logging.NoOpLogger{},     // Make this configurable
+		log:                          &logging.NoOpLogger{}, // Make this configurable
 		eventRetentionPeriod:         eventRetentionPeriod,
 		eventPruningInterval:         eventPruningInterval,
 		withExperimentalClusterScope: opts.WithExperimentalClusterScope,

pkg/storage/unified/search/bleve.go

@@ -93,6 +93,8 @@ type bleveBackend struct {
 
 	indexMetrics *resource.BleveIndexMetrics
 
+	selectableFields map[string][]string
+
 	bgTasksCancel func()
 	bgTasksWg     sync.WaitGroup
 }
@@ -135,11 +137,12 @@ func NewBleveBackend(opts BleveOptions, indexMetrics *resource.BleveIndexMetrics
 	}
 
 	be := &bleveBackend{
-		log:          l,
-		cache:        map[resource.NamespacedResource]*bleveIndex{},
-		opts:         opts,
-		ownsIndexFn:  ownFn,
-		indexMetrics: indexMetrics,
+		log:              l,
+		cache:            map[resource.NamespacedResource]*bleveIndex{},
+		opts:             opts,
+		ownsIndexFn:      ownFn,
+		indexMetrics:     indexMetrics,
+		selectableFields: resource.SelectableFields(),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -366,7 +369,9 @@ func (b *bleveBackend) BuildIndex(
 		attribute.String("reason", indexBuildReason),
 	)
 
-	mapper, err := GetBleveMappings(fields)
+	selectableFields := b.selectableFields[fmt.Sprintf("%s/%s", key.Group, key.Resource)]
+
+	mapper, err := GetBleveMappings(fields, selectableFields)
 	if err != nil {
 		return nil, err
 	}
@@ -459,7 +464,7 @@ func (b *bleveBackend) BuildIndex(
 	}
 
 	// Batch all the changes
-	idx := b.newBleveIndex(key, index, newIndexType, fields, allFields, standardSearchFields, updater, b.log.New("namespace", key.Namespace, "group", key.Group, "resource", key.Resource))
+	idx := b.newBleveIndex(key, index, newIndexType, fields, allFields, standardSearchFields, selectableFields, updater, b.log.New("namespace", key.Namespace, "group", key.Group, "resource", key.Resource))
 
 	if build {
 		if b.indexMetrics != nil {
@@ -699,8 +704,9 @@ type bleveIndex struct {
 	// Subsequent update requests only trigger new update if minUpdateInterval has elapsed.
 	nextUpdateTime time.Time
 
-	standard resource.SearchableDocumentFields
-	fields   resource.SearchableDocumentFields
+	standard         resource.SearchableDocumentFields
+	fields           resource.SearchableDocumentFields
+	selectableFields []string
 
 	indexStorage string // memory or file, used when updating metrics
 
@@ -736,6 +742,7 @@ func (b *bleveBackend) newBleveIndex(
 	fields resource.SearchableDocumentFields,
 	allFields []*resourcepb.ResourceTableColumnDefinition,
 	standardSearchFields resource.SearchableDocumentFields,
+	selectableFields []string,
 	updaterFn resource.UpdateFn,
 	logger log.Logger,
 ) *bleveIndex {
@@ -745,6 +752,7 @@ func (b *bleveBackend) newBleveIndex(
 		indexStorage:      newIndexType,
 		fields:            fields,
 		allFields:         allFields,
+		selectableFields:  selectableFields,
 		standard:          standardSearchFields,
 		logger:            logger,
 		updaterFn:         updaterFn,
@@ -1215,7 +1223,11 @@ func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resourcepb.R
 	// filters
 	if len(req.Options.Fields) > 0 {
 		for _, v := range req.Options.Fields {
-			q, err := requirementQuery(v, "")
+			prefix := ""
+			if b.isSelectableField(v.Key) {
+				prefix = "selectable_fields."
+			}
+			q, err := requirementQuery(v, prefix)
 			if err != nil {
 				return nil, err
 			}
@@ -1787,6 +1799,15 @@ func (b *bleveIndex) hitsToTable(ctx context.Context, selectFields []string, hit
 	return table, nil
 }
 
+func (b *bleveIndex) isSelectableField(key string) bool {
+	for _, f := range b.selectableFields {
+		if key == f {
+			return true
+		}
+	}
+	return false
+}
+
 func getAllFields(standard resource.SearchableDocumentFields, custom resource.SearchableDocumentFields) ([]*resourcepb.ResourceTableColumnDefinition, error) {
 	fields := []*resourcepb.ResourceTableColumnDefinition{
 		standard.Field(resource.SEARCH_FIELD_ID),

pkg/storage/unified/search/bleve_mappings.go

@@ -10,19 +10,19 @@ import (
 	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
 )
 
-func GetBleveMappings(fields resource.SearchableDocumentFields) (mapping.IndexMapping, error) {
+func GetBleveMappings(fields resource.SearchableDocumentFields, selectableFields []string) (mapping.IndexMapping, error) {
 	mapper := bleve.NewIndexMapping()
 
 	err := RegisterCustomAnalyzers(mapper)
 	if err != nil {
 		return nil, err
 	}
-	mapper.DefaultMapping = getBleveDocMappings(fields)
+	mapper.DefaultMapping = getBleveDocMappings(fields, selectableFields)
 
 	return mapper, nil
 }
 
-func getBleveDocMappings(fields resource.SearchableDocumentFields) *mapping.DocumentMapping {
+func getBleveDocMappings(fields resource.SearchableDocumentFields, selectableFields []string) *mapping.DocumentMapping {
 	mapper := bleve.NewDocumentStaticMapping()
 
 	nameMapping := &mapping.FieldMapping{
@@ -165,5 +165,73 @@ func getBleveDocMappings(fields resource.SearchableDocumentFields) *mapping.Docu
 
 	mapper.AddSubDocumentMapping("fields", fieldMapper)
 
+	selectableFieldsMapper := bleve.NewDocumentStaticMapping()
+	for _, field := range selectableFields {
+		selectableFieldsMapper.AddFieldMappingsAt(field, &mapping.FieldMapping{
+			Name:     field,
+			Type:     "text",
+			Analyzer: keyword.Name,
+			Store:    false,
+			Index:    true,
+		})
+	}
+	mapper.AddSubDocumentMapping("selectable_fields", selectableFieldsMapper)
+
 	return mapper
 }
+
+/*
+Here's a tree representation of the field mappings in pkg/storage/unified/search/bleve_mappings.go:
+
+  Document Root (DefaultMapping)
+  │
+  ├── name [text, keyword analyzer]
+  │
+  ├── title_phrase [keyword, not stored]
+  │
+  ├── title [3 mappings]
+  │   ├── [1] standard analyzer, stored
+  │   ├── [2] TITLE_ANALYZER (edge ngram), not stored
+  │   └── [3] keyword, not stored
+  │
+  ├── description [text, stored]
+  │
+  ├── tags [text, keyword analyzer, stored, includeInAll]
+  │
+  ├── folder [text, keyword analyzer, stored, includeInAll, docValues]
+  │
+  ├── managedBy [text, keyword analyzer, not stored]
+  │
+  ├── source/ [sub-document]
+  │   ├── path [text, keyword analyzer, stored]
+  │   ├── checksum [text, keyword analyzer, stored]
+  │   └── timestampMillis [numeric]
+  │
+  ├── manager/ [sub-document]
+  │   ├── kind [text, keyword analyzer, stored, includeInAll]
+  │   └── id [text, keyword analyzer, stored, includeInAll]
+  │
+  ├── reference/ [sub-document, default analyzer: keyword]
+  │   └── (dynamic fields inherit keyword analyzer)
+  │
+  ├── labels/ [sub-document]
+  │   └── (dynamic fields)
+  │
+  └── fields/ [sub-document]
+      └── (conditional mappings)
+          ├── {filterable string fields} [keyword, stored]
+          └── {other fields} [dynamically mapped by Bleve]
+
+  Key observations:
+
+  - Root level has standard searchable fields (name, title, description, tags, folder)
+  - title has 3 analyzers applied: standard (for word search), edge ngram (for prefix search), and keyword (for phrase sorting)
+  - source/, manager/: Static sub-documents with explicitly mapped fields
+  - reference/: Dynamic sub-document with keyword default analyzer (line 142)
+  - labels/, fields/: Dynamic sub-documents where Bleve auto-detects field types at index time
+
+  References:
+  - Main mapping function: pkg/storage/unified/search/bleve_mappings.go:25-169
+  - Sub-document mappings: lines 88-143
+  - Dynamic fields handling: lines 148-166
+*/

pkg/storage/unified/search/bleve_mappings_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestDocumentMapping(t *testing.T) {
-	mappings, err := search.GetBleveMappings(nil)
+	mappings, err := search.GetBleveMappings(nil, nil)
 	require.NoError(t, err)
 	data := resource.IndexableDocument{
 		Title:       "title",

pkg/storage/unified/search/builders/dashboard.go

@@ -264,7 +264,7 @@ func (s *DashboardDocumentBuilder) BuildDocument(ctx context.Context, key *resou
 	summary.UID = obj.GetName()
 	summary.ID = obj.GetDeprecatedInternalID() // nolint:staticcheck
 
-	doc := resource.NewIndexableDocument(key, rv, obj)
+	doc := resource.NewIndexableDocument(key, rv, obj, nil)
 	doc.Title = summary.Title
 	doc.Description = summary.Description
 	doc.Tags = summary.Tags

pkg/storage/unified/search/builders/document_test.go

@@ -115,7 +115,7 @@ func TestDashboardDocumentBuilder(t *testing.T) {
 		"aaa",
 	})
 
-	builder = resource.StandardDocumentBuilder()
+	builder = resource.StandardDocumentBuilder(nil)
 	doSnapshotTests(t, builder, "folder", &resourcepb.ResourceKey{
 		Namespace: "default",
 		Group:     "folder.grafana.app",

pkg/storage/unified/search/builders/external_group_mapping.go

@@ -66,7 +66,7 @@ func (u *extGroupMappingDocumentBuilder) BuildDocument(ctx context.Context, key
 		return nil, err
 	}
 
-	doc := resource.NewIndexableDocument(key, rv, obj)
+	doc := resource.NewIndexableDocument(key, rv, obj, nil)
 
 	doc.Fields = make(map[string]any)
 	if extGroupMapping.Spec.TeamRef.Name != "" {

pkg/storage/unified/search/builders/team_search.go

@@ -70,7 +70,7 @@ func (t *teamSearchBuilder) BuildDocument(ctx context.Context, key *resourcepb.R
 		return nil, err
 	}
 
-	doc := resource.NewIndexableDocument(key, rv, obj)
+	doc := resource.NewIndexableDocument(key, rv, obj, nil)
 
 	doc.Fields = make(map[string]any)
 	if team.Spec.Email != "" {

pkg/storage/unified/search/builders/user.go

@@ -66,7 +66,7 @@ func (u *userDocumentBuilder) BuildDocument(ctx context.Context, key *resourcepb
 		return nil, err
 	}
 
-	doc := resource.NewIndexableDocument(key, rv, obj)
+	doc := resource.NewIndexableDocument(key, rv, obj, nil)
 
 	doc.Fields = make(map[string]any)
 	if user.Spec.Email != "" {

pkg/storage/unified/search/document.go

@@ -25,7 +25,7 @@ func (s *StandardDocumentBuilders) GetDocumentBuilders() ([]resource.DocumentBui
 
 	result := []resource.DocumentBuilderInfo{
 		{
-			Builder: resource.StandardDocumentBuilder(),
+			Builder: resource.StandardDocumentBuilder(resource.AppManifests()),
 		},
 	}
 	return append(result, all...), nil

pkg/storage/unified/sql/server.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/grafana/grafana/pkg/storage/unified/resourcepb"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/trace"
 
@@ -37,6 +38,7 @@ type ServerOptions struct {
 	Tracer           trace.Tracer
 	Reg              prometheus.Registerer
 	AccessClient     types.AccessClient
+	SearchClient     resourcepb.ResourceIndexClient
 	SearchOptions    resource.SearchOptions
 	StorageMetrics   *resource.StorageMetrics
 	IndexMetrics     *resource.BleveIndexMetrics
@@ -115,6 +117,10 @@ func NewResourceServer(opts ServerOptions) (resource.ResourceServer, error) {
 		serverOptions.Lifecycle = backend
 	}
 
+	// use the search client when search isnt initialized. Dont need both.
+	if opts.SearchOptions.Resources == nil {
+		serverOptions.SearchClient = opts.SearchClient
+	}
 	serverOptions.Search = opts.SearchOptions
 	serverOptions.IndexMetrics = opts.IndexMetrics
 	serverOptions.QOSQueue = opts.QOSQueue

pkg/storage/unified/sql/service.go

@@ -59,12 +59,13 @@ type service struct {
 	subservicesWatcher *services.FailureWatcher
 	hasSubservices     bool
 
-	backend   resource.StorageBackend
-	cfg       *setting.Cfg
-	features  featuremgmt.FeatureToggles
-	db        infraDB.DB
-	stopCh    chan struct{}
-	stoppedCh chan error
+	backend      resource.StorageBackend
+	searchClient resourcepb.ResourceIndexClient
+	cfg          *setting.Cfg
+	features     featuremgmt.FeatureToggles
+	db           infraDB.DB
+	stopCh       chan struct{}
+	stoppedCh    chan error
 
 	handler grpcserver.Provider
 
@@ -99,6 +100,7 @@ func ProvideUnifiedStorageGrpcService(
 	memberlistKVConfig kv.Config,
 	httpServerRouter *mux.Router,
 	backend resource.StorageBackend,
+	searchClient resourcepb.ResourceIndexClient,
 ) (UnifiedStorageGrpcService, error) {
 	var err error
 	tracer := otel.Tracer("unified-storage")
@@ -112,6 +114,7 @@ func ProvideUnifiedStorageGrpcService(
 
 	s := &service{
 		backend:            backend,
+		searchClient:       searchClient,
 		cfg:                cfg,
 		features:           features,
 		stopCh:             make(chan struct{}),
@@ -272,6 +275,7 @@ func (s *service) starting(ctx context.Context) error {
 		Tracer:         s.tracing,
 		Reg:            s.reg,
 		AccessClient:   authzClient,
+		SearchClient:   s.searchClient,
 		SearchOptions:  searchOptions,
 		StorageMetrics: s.storageMetrics,
 		IndexMetrics:   s.indexMetrics,