lookup --> query

This commit is contained in:
shamoon 2024-09-23 13:35:36 -07:00
parent 9c09719dbd
commit 7a6d0c5b12
9 changed files with 54 additions and 54 deletions

View File

@ -277,39 +277,39 @@ attribute with various information about the search results:
### Filtering by custom fields ### Filtering by custom fields
You can filter documents by their custom field values by specifying the You can filter documents by their custom field values by specifying the
`custom_field_lookup` query parameter. Here are some recipes for common `custom_field_query` query parameter. Here are some recipes for common
use cases: use cases:
1. Documents with a custom field "due" (date) between Aug 1, 2024 and 1. Documents with a custom field "due" (date) between Aug 1, 2024 and
Sept 1, 2024 (inclusive): Sept 1, 2024 (inclusive):
`?custom_field_lookup=["due", "range", ["2024-08-01", "2024-09-01"]]` `?custom_field_query=["due", "range", ["2024-08-01", "2024-09-01"]]`
2. Documents with a custom field "customer" (text) that equals "bob" 2. Documents with a custom field "customer" (text) that equals "bob"
(case sensitive): (case sensitive):
`?custom_field_lookup=["customer", "exact", "bob"]` `?custom_field_query=["customer", "exact", "bob"]`
3. Documents with a custom field "answered" (boolean) set to `true`: 3. Documents with a custom field "answered" (boolean) set to `true`:
`?custom_field_lookup=["answered", "exact", true]` `?custom_field_query=["answered", "exact", true]`
4. Documents with a custom field "favorite animal" (select) set to either 4. Documents with a custom field "favorite animal" (select) set to either
"cat" or "dog": "cat" or "dog":
`?custom_field_lookup=["favorite animal", "in", ["cat", "dog"]]` `?custom_field_query=["favorite animal", "in", ["cat", "dog"]]`
5. Documents with a custom field "address" (text) that is empty: 5. Documents with a custom field "address" (text) that is empty:
`?custom_field_lookup=["OR", ["address", "isnull", true], ["address", "exact", ""]]` `?custom_field_query=["OR", ["address", "isnull", true], ["address", "exact", ""]]`
6. Documents that don't have a field called "foo": 6. Documents that don't have a field called "foo":
`?custom_field_lookup=["foo", "exists", false]` `?custom_field_query=["foo", "exists", false]`
7. Documents that have document links "references" to both document 3 and 7: 7. Documents that have document links "references" to both document 3 and 7:
`?custom_field_lookup=["references", "contains", [3, 7]]` `?custom_field_query=["references", "contains", [3, 7]]`
All field types support basic operations including `exact`, `in`, `isnull`, All field types support basic operations including `exact`, `in`, `isnull`,
and `exists`. String, URL, and monetary fields support case-insensitive and `exists`. String, URL, and monetary fields support case-insensitive

View File

@ -55,7 +55,7 @@ import {
FILTER_HAS_ANY_CUSTOM_FIELDS, FILTER_HAS_ANY_CUSTOM_FIELDS,
FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS, FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS,
FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ALL,
FILTER_CUSTOM_FIELDS_LOOKUP, FILTER_CUSTOM_FIELDS_QUERY,
} from 'src/app/data/filter-rule-type' } from 'src/app/data/filter-rule-type'
import { Correspondent } from 'src/app/data/correspondent' import { Correspondent } from 'src/app/data/correspondent'
import { DocumentType } from 'src/app/data/document-type' import { DocumentType } from 'src/app/data/document-type'
@ -887,11 +887,11 @@ describe('FilterEditorComponent', () => {
).toEqual(['42', CustomFieldQueryOperator.Exists, 'true']) ).toEqual(['42', CustomFieldQueryOperator.Exists, 'true'])
})) }))
it('should ingest filter rules for custom field lookup', fakeAsync(() => { it('should ingest filter rules for custom field queries', fakeAsync(() => {
expect(component.customFieldQueriesModel.isEmpty()).toBeTruthy() expect(component.customFieldQueriesModel.isEmpty()).toBeTruthy()
component.filterRules = [ component.filterRules = [
{ {
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: '["AND", [[42, "exists", "true"],[43, "exists", "true"]]]', value: '["AND", [[42, "exists", "true"],[43, "exists", "true"]]]',
}, },
] ]
@ -909,7 +909,7 @@ describe('FilterEditorComponent', () => {
// atom // atom
component.filterRules = [ component.filterRules = [
{ {
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: '[42, "exists", "true"]', value: '[42, "exists", "true"]',
}, },
] ]
@ -1459,7 +1459,7 @@ describe('FilterEditorComponent', () => {
expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(1) expect(component.customFieldQueriesModel.queries[0].value.length).toEqual(1)
expect(component.filterRules).toEqual([ expect(component.filterRules).toEqual([
{ {
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: JSON.stringify([ value: JSON.stringify([
CustomFieldQueryLogicalOperator.Or, CustomFieldQueryLogicalOperator.Or,
[[custom_fields[0].id, 'exists', 'true']], [[custom_fields[0].id, 'exists', 'true']],
@ -1876,7 +1876,7 @@ describe('FilterEditorComponent', () => {
component.filterRules = [ component.filterRules = [
{ {
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: '["AND",[["42","exists","true"],["43","exists","true"]]]', value: '["AND",[["42","exists","true"],["43","exists","true"]]]',
}, },
] ]

View File

@ -62,7 +62,7 @@ import {
FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_CUSTOM_FIELDS_ANY,
FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ALL,
FILTER_HAS_ANY_CUSTOM_FIELDS, FILTER_HAS_ANY_CUSTOM_FIELDS,
FILTER_CUSTOM_FIELDS_LOOKUP, FILTER_CUSTOM_FIELDS_QUERY,
} from 'src/app/data/filter-rule-type' } from 'src/app/data/filter-rule-type'
import { import {
FilterableDropdownSelectionModel, FilterableDropdownSelectionModel,
@ -234,7 +234,7 @@ export class FilterEditorComponent
return $localize`Without any tag` return $localize`Without any tag`
} }
case FILTER_CUSTOM_FIELDS_LOOKUP: case FILTER_CUSTOM_FIELDS_QUERY:
return $localize`Custom fields query` return $localize`Custom fields query`
case FILTER_TITLE: case FILTER_TITLE:
@ -525,7 +525,7 @@ export class FilterEditorComponent
false false
) )
break break
case FILTER_CUSTOM_FIELDS_LOOKUP: case FILTER_CUSTOM_FIELDS_QUERY:
try { try {
const query = JSON.parse(rule.value) const query = JSON.parse(rule.value)
if (Array.isArray(query)) { if (Array.isArray(query)) {
@ -786,7 +786,7 @@ export class FilterEditorComponent
) )
if (queries.length > 0) { if (queries.length > 0) {
filterRules.push({ filterRules.push({
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: JSON.stringify(queries[0]), value: JSON.stringify(queries[0]),
}) })
} }

View File

@ -55,7 +55,7 @@ export const FILTER_HAS_CUSTOM_FIELDS_ANY = 39
export const FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS = 40 export const FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS = 40
export const FILTER_HAS_ANY_CUSTOM_FIELDS = 41 export const FILTER_HAS_ANY_CUSTOM_FIELDS = 41
export const FILTER_CUSTOM_FIELDS_LOOKUP = 42 export const FILTER_CUSTOM_FIELDS_QUERY = 42
export const FILTER_RULE_TYPES: FilterRuleType[] = [ export const FILTER_RULE_TYPES: FilterRuleType[] = [
{ {
@ -320,8 +320,8 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [
default: true, default: true,
}, },
{ {
id: FILTER_CUSTOM_FIELDS_LOOKUP, id: FILTER_CUSTOM_FIELDS_QUERY,
filtervar: 'custom_field_lookup', filtervar: 'custom_field_query',
datatype: 'string', datatype: 'string',
multi: false, multi: false,
}, },

View File

@ -2,7 +2,7 @@ import { convertToParamMap } from '@angular/router'
import { FilterRule } from '../data/filter-rule' import { FilterRule } from '../data/filter-rule'
import { import {
FILTER_CORRESPONDENT, FILTER_CORRESPONDENT,
FILTER_CUSTOM_FIELDS_LOOKUP, FILTER_CUSTOM_FIELDS_QUERY,
FILTER_HAS_ANY_TAG, FILTER_HAS_ANY_TAG,
FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ALL,
FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_CUSTOM_FIELDS_ANY,
@ -214,7 +214,7 @@ describe('QueryParams Utils', () => {
expect(transformedFilterRules).toEqual([ expect(transformedFilterRules).toEqual([
{ {
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: JSON.stringify([ value: JSON.stringify([
CustomFieldQueryLogicalOperator.Or, CustomFieldQueryLogicalOperator.Or,
[ [
@ -240,7 +240,7 @@ describe('QueryParams Utils', () => {
expect(transformedFilterRules).toEqual([ expect(transformedFilterRules).toEqual([
{ {
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: JSON.stringify([ value: JSON.stringify([
CustomFieldQueryLogicalOperator.And, CustomFieldQueryLogicalOperator.And,
[ [

View File

@ -4,7 +4,7 @@ import {
FilterRuleType, FilterRuleType,
FILTER_RULE_TYPES, FILTER_RULE_TYPES,
FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_CUSTOM_FIELDS_ANY,
FILTER_CUSTOM_FIELDS_LOOKUP, FILTER_CUSTOM_FIELDS_QUERY,
FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ALL,
} from '../data/filter-rule-type' } from '../data/filter-rule-type'
import { ListViewState } from '../services/document-list-view.service' import { ListViewState } from '../services/document-list-view.service'
@ -83,7 +83,7 @@ export function transformLegacyFilterRules(
], ],
] ]
filterRules.push({ filterRules.push({
rule_type: FILTER_CUSTOM_FIELDS_LOOKUP, rule_type: FILTER_CUSTOM_FIELDS_QUERY,
value: JSON.stringify(customFieldQueryExpression), value: JSON.stringify(customFieldQueryExpression),
}) })
} }

View File

@ -36,8 +36,8 @@ ID_KWARGS = ["in", "exact"]
INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"] INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"]
DATE_KWARGS = ["year", "month", "day", "date__gt", "gt", "date__lt", "lt"] DATE_KWARGS = ["year", "month", "day", "date__gt", "gt", "date__lt", "lt"]
CUSTOM_FIELD_LOOKUP_MAX_DEPTH = 10 CUSTOM_FIELD_QUERY_MAX_DEPTH = 10
CUSTOM_FIELD_LOOKUP_MAX_ATOMS = 20 CUSTOM_FIELD_QUERY_MAX_ATOMS = 20
class CorrespondentFilterSet(FilterSet): class CorrespondentFilterSet(FilterSet):
@ -237,7 +237,7 @@ def handle_validation_prefix(func: Callable):
return wrapper return wrapper
class CustomFieldLookupParser: class CustomFieldQueryParser:
EXPR_BY_CATEGORY = { EXPR_BY_CATEGORY = {
"basic": ["exact", "in", "isnull", "exists"], "basic": ["exact", "in", "isnull", "exists"],
"string": [ "string": [
@ -351,7 +351,7 @@ class CustomFieldLookupParser:
elif len(expr) == 3: elif len(expr) == 3:
return self._parse_atom(*expr) return self._parse_atom(*expr)
raise serializers.ValidationError( raise serializers.ValidationError(
[_("Invalid custom field lookup expression")], [_("Invalid custom field query expression")],
) )
@handle_validation_prefix @handle_validation_prefix
@ -485,7 +485,7 @@ class CustomFieldLookupParser:
if not supported: if not supported:
raise serializers.ValidationError( raise serializers.ValidationError(
[ [
_("{data_type} does not support lookup expr {expr!r}.").format( _("{data_type} does not support query expr {expr!r}.").format(
data_type=custom_field.data_type, data_type=custom_field.data_type,
expr=raw_op, expr=raw_op,
), ),
@ -506,7 +506,7 @@ class CustomFieldLookupParser:
custom_field.data_type == CustomField.FieldDataType.DATE custom_field.data_type == CustomField.FieldDataType.DATE
and prefix in self.DATE_COMPONENTS and prefix in self.DATE_COMPONENTS
): ):
# DateField admits lookups in the form of `year__exact`, etc. These take integers. # DateField admits queries in the form of `year__exact`, etc. These take integers.
field = serializers.IntegerField() field = serializers.IntegerField()
elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK: elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
# We can be more specific here and make sure the value is a list. # We can be more specific here and make sure the value is a list.
@ -568,7 +568,7 @@ class CustomFieldLookupParser:
custom_fields__value_document_ids__isnull=False, custom_fields__value_document_ids__isnull=False,
) )
# First we lookup reverse links from the requested documents. # First we look up reverse links from the requested documents.
links = CustomFieldInstance.objects.filter( links = CustomFieldInstance.objects.filter(
document_id__in=value, document_id__in=value,
field__data_type=CustomField.FieldDataType.DOCUMENTLINK, field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
@ -600,7 +600,7 @@ class CustomFieldLookupParser:
self._current_depth -= 1 self._current_depth -= 1
class CustomFieldLookupFilter(Filter): class CustomFieldQueryFilter(Filter):
def __init__(self, validation_prefix): def __init__(self, validation_prefix):
""" """
A filter that filters documents based on custom field name and value. A filter that filters documents based on custom field name and value.
@ -615,10 +615,10 @@ class CustomFieldLookupFilter(Filter):
if not value: if not value:
return qs return qs
parser = CustomFieldLookupParser( parser = CustomFieldQueryParser(
self._validation_prefix, self._validation_prefix,
max_query_depth=CUSTOM_FIELD_LOOKUP_MAX_DEPTH, max_query_depth=CUSTOM_FIELD_QUERY_MAX_DEPTH,
max_atom_count=CUSTOM_FIELD_LOOKUP_MAX_ATOMS, max_atom_count=CUSTOM_FIELD_QUERY_MAX_ATOMS,
) )
q, annotations = parser.parse(value) q, annotations = parser.parse(value)
@ -672,7 +672,7 @@ class DocumentFilterSet(FilterSet):
exclude=True, exclude=True,
) )
custom_field_lookup = CustomFieldLookupFilter("custom_field_lookup") custom_field_query = CustomFieldQueryFilter("custom_field_query")
shared_by__id = SharedByUser() shared_by__id = SharedByUser()

View File

@ -507,7 +507,7 @@ class SavedViewFilterRule(models.Model):
(39, _("has custom field in")), (39, _("has custom field in")),
(40, _("does not have custom field in")), (40, _("does not have custom field in")),
(41, _("does not have custom field")), (41, _("does not have custom field")),
(42, _("custom fields lookup")), (42, _("custom fields query")),
] ]
saved_view = models.ForeignKey( saved_view = models.ForeignKey(

View File

@ -181,7 +181,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
"/api/documents/?" "/api/documents/?"
+ "&".join( + "&".join(
( (
f"custom_field_lookup={query_string}", f"custom_field_query={query_string}",
"ordering=archive_serial_number", "ordering=archive_serial_number",
"page=1", "page=1",
f"page_size={len(self.documents)}", f"page_size={len(self.documents)}",
@ -205,7 +205,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
"/api/documents/?" "/api/documents/?"
+ "&".join( + "&".join(
( (
f"custom_field_lookup={query_string}", f"custom_field_query={query_string}",
"ordering=archive_serial_number", "ordering=archive_serial_number",
"page=1", "page=1",
f"page_size={len(self.documents)}", f"page_size={len(self.documents)}",
@ -477,57 +477,57 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
def test_invalid_json(self): def test_invalid_json(self):
self._assert_validation_error( self._assert_validation_error(
"not valid json", "not valid json",
["custom_field_lookup"], ["custom_field_query"],
"must be valid JSON", "must be valid JSON",
) )
def test_invalid_expression(self): def test_invalid_expression(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps("valid json but not valid expr"), json.dumps("valid json but not valid expr"),
["custom_field_lookup"], ["custom_field_query"],
"Invalid custom field lookup expression", "Invalid custom field query expression",
) )
def test_invalid_custom_field_name(self): def test_invalid_custom_field_name(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps(["invalid name", "iexact", "foo"]), json.dumps(["invalid name", "iexact", "foo"]),
["custom_field_lookup", "0"], ["custom_field_query", "0"],
"is not a valid custom field", "is not a valid custom field",
) )
def test_invalid_operator(self): def test_invalid_operator(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps(["integer_field", "iexact", "foo"]), json.dumps(["integer_field", "iexact", "foo"]),
["custom_field_lookup", "1"], ["custom_field_query", "1"],
"does not support lookup expr", "does not support query expr",
) )
def test_invalid_value(self): def test_invalid_value(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps(["select_field", "exact", "not an option"]), json.dumps(["select_field", "exact", "not an option"]),
["custom_field_lookup", "2"], ["custom_field_query", "2"],
"integer", "integer",
) )
def test_invalid_logical_operator(self): def test_invalid_logical_operator(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps(["invalid op", ["integer_field", "gt", 0]]), json.dumps(["invalid op", ["integer_field", "gt", 0]]),
["custom_field_lookup", "0"], ["custom_field_query", "0"],
"Invalid logical operator", "Invalid logical operator",
) )
def test_invalid_expr_list(self): def test_invalid_expr_list(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps(["AND", "not a list"]), json.dumps(["AND", "not a list"]),
["custom_field_lookup", "1"], ["custom_field_query", "1"],
"Invalid expression list", "Invalid expression list",
) )
def test_invalid_operator_prefix(self): def test_invalid_operator_prefix(self):
self._assert_validation_error( self._assert_validation_error(
json.dumps(["integer_field", "foo__gt", 0]), json.dumps(["integer_field", "foo__gt", 0]),
["custom_field_lookup", "1"], ["custom_field_query", "1"],
"does not support lookup expr", "does not support query expr",
) )
def test_query_too_deep(self): def test_query_too_deep(self):
@ -536,7 +536,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
query = ["NOT", query] query = ["NOT", query]
self._assert_validation_error( self._assert_validation_error(
json.dumps(query), json.dumps(query),
["custom_field_lookup", *(["1"] * 10)], ["custom_field_query", *(["1"] * 10)],
"Maximum nesting depth exceeded", "Maximum nesting depth exceeded",
) )
@ -545,6 +545,6 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
query = ["AND", [atom for _ in range(21)]] query = ["AND", [atom for _ in range(21)]]
self._assert_validation_error( self._assert_validation_error(
json.dumps(query), json.dumps(query),
["custom_field_lookup", "1", "20"], ["custom_field_query", "1", "20"],
"Maximum number of query conditions exceeded", "Maximum number of query conditions exceeded",
) )