Skip to content

.Net: Remove legacy filtering from MEVD providers #13636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
roji wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

.Net: Remove legacy filtering from MEVD providers #13636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions dotnet/src/VectorData/AzureAISearch/AzureAISearchCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,9 +442,6 @@ public async IAsyncEnumerable<VectorSearchResult<TRecord>> HybridSearchAsync<TIn
this._model,
new()
{
#pragma warning disable CS0618 // Type or member is obsolete
OldFilter = options.OldFilter,
#pragma warning restore CS0618 // Type or member is obsolete
Filter = options.Filter,
VectorProperty = options.VectorProperty,
Skip = options.Skip,
Expand Down Expand Up @@ -660,16 +657,10 @@ private static SearchOptions BuildSearchOptions(CollectionModel model, VectorSea
throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
}

#pragma warning disable CS0618 // VectorSearchFilter is obsolete
// Build filter object.
var filter = options switch
{
{ OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"),
{ OldFilter: VectorSearchFilter legacyFilter } => AzureAISearchCollectionSearchMapping.BuildLegacyFilterString(legacyFilter, model),
{ Filter: Expression<Func<TRecord, bool>> newFilter } => new AzureAISearchFilterTranslator().Translate(newFilter, model),
_ => null
};
#pragma warning restore CS0618
var filter = options.Filter is not null
? new AzureAISearchFilterTranslator().Translate(options.Filter, model)
: null;

// Build search options.
var searchOptions = new SearchOptions
Expand Down
78 changes: 0 additions & 78 deletions dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionSearchMapping.cs
View file
Open in desktop

This file was deleted.

12 changes: 3 additions & 9 deletions dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,15 +373,9 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
: throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType()))
};

#pragma warning disable CS0618 // VectorSearchFilter is obsolete
var filter = options switch
{
{ OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"),
{ OldFilter: VectorSearchFilter legacyFilter } => CosmosMongoCollectionSearchMapping.BuildFilter(legacyFilter, this._model),
{ Filter: Expression<Func<TRecord, bool>> newFilter } => new CosmosMongoFilterTranslator().Translate(newFilter, this._model),
_ => null
};
#pragma warning restore CS0618
var filter = options.Filter is not null
? new CosmosMongoFilterTranslator().Translate(options.Filter, this._model)
: null;

// Constructing a query to fetch "skip + top" total items
// to perform skip logic locally, since skip option is not part of API.
Expand Down
72 changes: 0 additions & 72 deletions dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollectionSearchMapping.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Linq;
using Microsoft.Extensions.VectorData;
using Microsoft.Extensions.VectorData.ProviderServices;
using Microsoft.SemanticKernel.Connectors.MongoDB;
using MongoDB.Bson;

Expand All @@ -20,75 +17,6 @@ internal static class CosmosMongoCollectionSearchMapping
/// <summary>Returns distance function specified on vector property or default <see cref="MongoConstants.DefaultDistanceFunction"/>.</summary>
public static string GetVectorPropertyDistanceFunction(string? distanceFunction) => !string.IsNullOrWhiteSpace(distanceFunction) ? distanceFunction! : MongoConstants.DefaultDistanceFunction;

#pragma warning disable CS0618 // VectorSearchFilter is obsolete
/// <summary>
/// Build Azure CosmosDB MongoDB filter from the provided <see cref="VectorSearchFilter"/>.
/// </summary>
/// <param name="vectorSearchFilter">The <see cref="VectorSearchFilter"/> to build Azure CosmosDB MongoDB filter from.</param>
/// <param name="model">The model.</param>
/// <exception cref="NotSupportedException">Thrown when the provided filter type is unsupported.</exception>
/// <exception cref="InvalidOperationException">Thrown when property name specified in filter doesn't exist.</exception>
public static BsonDocument? BuildFilter(VectorSearchFilter? vectorSearchFilter, CollectionModel model)
{
const string EqualOperator = "$eq";

var filterClauses = vectorSearchFilter?.FilterClauses.ToList();

if (filterClauses is not { Count: > 0 })
{
return null;
}

var filter = new BsonDocument();

foreach (var filterClause in filterClauses)
{
string propertyName;
BsonValue propertyValue;
string filterOperator;

if (filterClause is EqualToFilterClause equalToFilterClause)
{
propertyName = equalToFilterClause.FieldName;
propertyValue = BsonValueFactory.Create(equalToFilterClause.Value);
filterOperator = EqualOperator;
}
else
{
throw new NotSupportedException(
$"Unsupported filter clause type '{filterClause.GetType().Name}'. " +
$"Supported filter clause types are: {string.Join(", ", [
nameof(EqualToFilterClause)])}");
}

if (!model.PropertyMap.TryGetValue(propertyName, out var property))
{
throw new InvalidOperationException($"Property name '{propertyName}' provided as part of the filter clause is not a valid property name.");
}

var storageName = property.StorageName;

if (filter.Contains(storageName))
{
if (filter[storageName] is BsonDocument document && document.Contains(filterOperator))
{
throw new NotSupportedException(
$"Filter with operator '{filterOperator}' is already added to '{propertyName}' property. " +
"Multiple filters of the same type in the same property are not supported.");
}

filter[storageName][filterOperator] = propertyValue;
}
else
{
filter[storageName] = new BsonDocument() { [filterOperator] = propertyValue };
}
}

return filter;
}
#pragma warning restore CS0618 // VectorSearchFilter is obsolete

/// <summary>Returns search part of the search query for <see cref="IndexKind.Hnsw"/> index kind.</summary>
public static BsonDocument GetSearchQueryForHnswIndex<TVector>(
TVector vector,
Expand Down
6 changes: 0 additions & 6 deletions dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,7 +541,6 @@ public override async IAsyncEnumerable<VectorSearchResult<TRecord>> SearchAsync<
var vectorProperty = this._model.GetVectorPropertyOrSingle(options);
object vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false);

#pragma warning disable CS0618 // Type or member is obsolete
var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery(
vector,
null,
Expand All @@ -550,13 +549,11 @@ public override async IAsyncEnumerable<VectorSearchResult<TRecord>> SearchAsync<
vectorProperty.DistanceFunction,
textPropertyName: null,
ScorePropertyName,
options.OldFilter,
options.Filter,
options.ScoreThreshold,
top,
options.Skip,
options.IncludeVectors);
#pragma warning restore CS0618 // Type or member is obsolete

var searchResults = this.GetItemsAsync<JsonObject>(queryDefinition, OperationName, cancellationToken);

Expand Down Expand Up @@ -645,7 +642,6 @@ public async IAsyncEnumerable<VectorSearchResult<TRecord>> HybridSearchAsync<TIn
object vector = await GetSearchVectorAsync(searchValue, vectorProperty, cancellationToken).ConfigureAwait(false);
var textProperty = this._model.GetFullTextDataPropertyOrSingle(options.AdditionalProperty);

#pragma warning disable CS0618 // Type or member is obsolete
var queryDefinition = CosmosNoSqlCollectionQueryBuilder.BuildSearchQuery<TRecord>(
vector,
keywords,
Expand All @@ -654,13 +650,11 @@ public async IAsyncEnumerable<VectorSearchResult<TRecord>> HybridSearchAsync<TIn
vectorProperty.DistanceFunction,
textProperty.StorageName,
ScorePropertyName,
options.OldFilter,
options.Filter,
options.ScoreThreshold,
top,
options.Skip,
options.IncludeVectors);
#pragma warning restore CS0618 // Type or member is obsolete

var searchResults = this.GetItemsAsync<JsonObject>(queryDefinition, OperationName, cancellationToken);

Expand Down
77 changes: 3 additions & 74 deletions dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlCollectionQueryBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ public static QueryDefinition BuildSearchQuery<TRecord>(
string? distanceFunction,
string? textPropertyName,
string scorePropertyName,
#pragma warning disable CS0618 // Type or member is obsolete
VectorSearchFilter? oldFilter,
#pragma warning restore CS0618 // Type or member is obsolete
Expression<Func<TRecord, bool>>? filter,
double? scoreThreshold,
int top,
Expand All @@ -54,16 +51,10 @@ public static QueryDefinition BuildSearchQuery<TRecord>(

var selectClauseArguments = string.Join(",", [.. fieldsArgument, vectorDistanceArgumentWithAlias]);

#pragma warning disable CS0618 // VectorSearchFilter is obsolete
// Build filter object.
var (filterClause, filterParameters) = (OldFilter: oldFilter, Filter: filter) switch
{
{ OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"),
{ OldFilter: VectorSearchFilter legacyFilter } => BuildSearchFilter(legacyFilter, model),
{ Filter: Expression<Func<TRecord, bool>> newFilter } => new CosmosNoSqlFilterTranslator().Translate(newFilter, model),
_ => (null, [])
};
#pragma warning restore CS0618 // VectorSearchFilter is obsolete
var (filterClause, filterParameters) = filter is not null
? new CosmosNoSqlFilterTranslator().Translate(filter, model)
: ((string?)null, new Dictionary<string, object?>());

var queryParameters = new Dictionary<string, object?>
{
Expand Down Expand Up @@ -229,68 +220,6 @@ internal static QueryDefinition BuildSearchQuery<TRecord>(

#region private

#pragma warning disable CS0618 // VectorSearchFilter is obsolete
private static (string WhereClause, Dictionary<string, object?> Parameters) BuildSearchFilter(
VectorSearchFilter filter,
CollectionModel model)
{
const string ArrayContainsOperator = "ARRAY_CONTAINS";
const string ConditionValueVariableName = "@cv";

var tableVariableName = CosmosNoSqlConstants.ContainerAlias;

var filterClauses = filter.FilterClauses.ToList();

var whereClauseBuilder = new StringBuilder();
var queryParameters = new Dictionary<string, object?>();

for (var i = 0; i < filterClauses.Count; i++)
{
if (i > 0)
{
whereClauseBuilder.Append(" AND ");
}
var filterClause = filterClauses[i];

string queryParameterName = $"{ConditionValueVariableName}{i}";
object queryParameterValue;

if (filterClause is EqualToFilterClause equalToFilterClause)
{
var propertyName = GetStoragePropertyName(equalToFilterClause.FieldName, model);
whereClauseBuilder
.Append(GeneratePropertyAccess(tableVariableName, propertyName))
.Append(" = ")
.Append(queryParameterName);
queryParameterValue = equalToFilterClause.Value;
}
else if (filterClause is AnyTagEqualToFilterClause anyTagEqualToFilterClause)
{
var propertyName = GetStoragePropertyName(anyTagEqualToFilterClause.FieldName, model);
whereClauseBuilder.Append(ArrayContainsOperator)
.Append('(')
.Append(GeneratePropertyAccess(tableVariableName, propertyName))
.Append(", ")
.Append(queryParameterName)
.Append(')');
queryParameterValue = anyTagEqualToFilterClause.Value;
}
else
{
throw new NotSupportedException(
$"Unsupported filter clause type '{filterClause.GetType().Name}'. " +
$"Supported filter clause types are: {string.Join(", ", [
nameof(EqualToFilterClause),
nameof(AnyTagEqualToFilterClause)])}");
}

queryParameters.Add(queryParameterName, queryParameterValue);
}

return (whereClauseBuilder.ToString(), queryParameters);
}
#pragma warning restore CS0618 // VectorSearchFilter is obsolete

private static string GetStoragePropertyName(string propertyName, CollectionModel model)
{
if (!model.PropertyMap.TryGetValue(propertyName, out var property))
Expand Down
12 changes: 3 additions & 9 deletions dotnet/src/VectorData/InMemory/InMemoryCollection.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,17 +282,11 @@ _ when vectorProperty.EmbeddingGenerationDispatcher is not null
: throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(typeof(TInput), vectorProperty.EmbeddingGenerator.GetType()))
};

#pragma warning disable CS0618 // VectorSearchFilter is obsolete
// Filter records using the provided filter before doing the vector comparison.
var allValues = this.GetCollectionDictionary().Values.Cast<InMemoryRecordWrapper<TRecord>>();
var filteredRecords = options switch
{
{ OldFilter: not null, Filter: not null } => throw new ArgumentException("Either Filter or OldFilter can be specified, but not both"),
{ OldFilter: VectorSearchFilter legacyFilter } => InMemoryCollectionSearchMapping.FilterRecords(legacyFilter, allValues),
{ Filter: Expression<Func<TRecord, bool>> newFilter } => allValues.AsQueryable().Where(this.ConvertFilter(newFilter)),
_ => allValues
};
#pragma warning restore CS0618 // VectorSearchFilter is obsolete
var filteredRecords = options.Filter is not null
? allValues.AsQueryable().Where(this.ConvertFilter(options.Filter))
: allValues;

// Compare each vector in the filtered results with the provided vector.
var results = filteredRecords.Select<InMemoryRecordWrapper<TRecord>, (TRecord record, float score)?>(wrapper =>
Expand Down
Loading
Loading