diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts index e499a36f5..30bee4d92 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts @@ -46,6 +46,7 @@ import { FILTER_OWNER_ANY, FILTER_OWNER_DOES_NOT_INCLUDE, FILTER_OWNER_ISNULL, + FILTER_CUSTOM_FIELDS, } from 'src/app/data/filter-rule-type' import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent' import { PaperlessDocumentType } from 'src/app/data/paperless-document-type' @@ -240,6 +241,18 @@ describe('FilterEditorComponent', () => { expect(component.textFilterTarget).toEqual('asn') // TEXT_FILTER_TARGET_ASN })) + it('should ingest text filter rules for custom fields', fakeAsync(() => { + expect(component.textFilter).toEqual(null) + component.filterRules = [ + { + rule_type: FILTER_CUSTOM_FIELDS, + value: 'foo', + }, + ] + expect(component.textFilter).toEqual('foo') + expect(component.textFilterTarget).toEqual('custom-fields') // TEXT_FILTER_TARGET_CUSTOM_FIELDS + })) + it('should ingest text filter rules for doc asn is null', fakeAsync(() => { expect(component.textFilterTarget).toEqual('title-content') expect(component.textFilterModifier).toEqual('equals') // TEXT_FILTER_MODIFIER_EQUALS @@ -956,12 +969,30 @@ describe('FilterEditorComponent', () => { ]) })) - it('should convert user input to correct filter rules on full text query', fakeAsync(() => { + it('should convert user input to correct filter rules on custom fields query', fakeAsync(() => { component.textFilterInput.nativeElement.value = 'foo' component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) const textFieldTargetDropdown = fixture.debugElement.queryAll( By.directive(NgbDropdownItem) )[3] + textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_CUSTOM_FIELDS + fixture.detectChanges() + tick(400) + expect(component.textFilterTarget).toEqual('custom-fields') + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_CUSTOM_FIELDS, + value: 'foo', + }, + ]) + })) + + it('should convert user input to correct filter rules on full text query', fakeAsync(() => { + component.textFilterInput.nativeElement.value = 'foo' + component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) + const textFieldTargetDropdown = fixture.debugElement.queryAll( + By.directive(NgbDropdownItem) + )[4] textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_ASN fixture.detectChanges() tick(400) diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index 78b1bdc83..030f4ec07 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -48,6 +48,7 @@ import { FILTER_OWNER_DOES_NOT_INCLUDE, FILTER_OWNER_ISNULL, FILTER_OWNER_ANY, + FILTER_CUSTOM_FIELDS, } from 'src/app/data/filter-rule-type' import { FilterableDropdownSelectionModel, @@ -74,6 +75,7 @@ const TEXT_FILTER_TARGET_TITLE_CONTENT = 'title-content' const TEXT_FILTER_TARGET_ASN = 'asn' const TEXT_FILTER_TARGET_FULLTEXT_QUERY = 'fulltext-query' const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike' +const TEXT_FILTER_TARGET_CUSTOM_FIELDS = 'custom-fields' const TEXT_FILTER_MODIFIER_EQUALS = 'equals' const TEXT_FILTER_MODIFIER_NULL = 'is null' @@ -204,6 +206,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy { name: $localize`Title & content`, }, { id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN` }, + { + id: TEXT_FILTER_TARGET_CUSTOM_FIELDS, + name: $localize`Custom fields`, + }, { id: TEXT_FILTER_TARGET_FULLTEXT_QUERY, name: $localize`Advanced search`, @@ -321,6 +327,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy { this._textFilter = rule.value this.textFilterTarget = TEXT_FILTER_TARGET_ASN break + case FILTER_CUSTOM_FIELDS: + this._textFilter = rule.value + this.textFilterTarget = TEXT_FILTER_TARGET_CUSTOM_FIELDS + break case FILTER_FULLTEXT_QUERY: let allQueryArgs = rule.value.split(',') let textQueryArgs = [] @@ -552,6 +562,15 @@ export class FilterEditorComponent implements OnInit, OnDestroy { }) } } + if ( + this._textFilter && + this.textFilterTarget == TEXT_FILTER_TARGET_CUSTOM_FIELDS + ) { + filterRules.push({ + rule_type: FILTER_CUSTOM_FIELDS, + value: this._textFilter, + }) + } if ( this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_QUERY diff --git a/src-ui/src/app/data/filter-rule-type.ts b/src-ui/src/app/data/filter-rule-type.ts index f65f52fd2..a6c73fe29 100644 --- a/src-ui/src/app/data/filter-rule-type.ts +++ b/src-ui/src/app/data/filter-rule-type.ts @@ -46,6 +46,8 @@ export const FILTER_OWNER_ANY = 33 export const FILTER_OWNER_ISNULL = 34 export const FILTER_OWNER_DOES_NOT_INCLUDE = 35 +export const FILTER_CUSTOM_FIELDS = 36 + export const FILTER_RULE_TYPES: FilterRuleType[] = [ { id: FILTER_TITLE, @@ -271,6 +273,12 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [ datatype: 'number', multi: true, }, + { + id: FILTER_CUSTOM_FIELDS, + filtervar: 'custom_fields__icontains', + datatype: 'string', + multi: false, + }, ] export interface FilterRuleType { diff --git a/src/documents/filters.py b/src/documents/filters.py index 2d4cdf825..21d120047 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -82,6 +82,21 @@ class TitleContentFilter(Filter): return qs +class CustomFieldsFilter(Filter): + def filter(self, qs, value): + if value: + return ( + qs.filter(custom_fields__field__name__icontains=value) + | qs.filter(custom_fields__value_text__icontains=value) + | qs.filter(custom_fields__value_bool__icontains=value) + | qs.filter(custom_fields__value_int__icontains=value) + | qs.filter(custom_fields__value_date__icontains=value) + | qs.filter(custom_fields__value_url__icontains=value) + ) + else: + return qs + + class DocumentFilterSet(FilterSet): is_tagged = BooleanFilter( label="Is tagged", @@ -108,6 +123,8 @@ class DocumentFilterSet(FilterSet): owner__id__none = ObjectFilter(field_name="owner", exclude=True) + custom_fields__icontains = CustomFieldsFilter() + class Meta: model = Document fields = { @@ -132,6 +149,7 @@ class DocumentFilterSet(FilterSet): "storage_path__name": CHAR_KWARGS, "owner": ["isnull"], "owner__id": ID_KWARGS, + "custom_fields": ["icontains"], } diff --git a/src/documents/index.py b/src/documents/index.py index b9970b6c6..2e2585071 100644 --- a/src/documents/index.py +++ b/src/documents/index.py @@ -31,6 +31,7 @@ from whoosh.searching import Searcher from whoosh.writing import AsyncWriter # from documents.models import CustomMetadata +from documents.models import CustomFieldInstance from documents.models import Document from documents.models import Note from documents.models import User @@ -61,8 +62,8 @@ def get_schema(): has_path=BOOLEAN(), notes=TEXT(), num_notes=NUMERIC(sortable=True, signed=False), - # custom_metadata=TEXT(), - # custom_field_count=NUMERIC(sortable=True, signed=False), + custom_fields=TEXT(), + custom_field_count=NUMERIC(sortable=True, signed=False), owner=TEXT(), owner_id=NUMERIC(), has_owner=BOOLEAN(), @@ -111,9 +112,9 @@ def update_document(writer: AsyncWriter, doc: Document): tags = ",".join([t.name for t in doc.tags.all()]) tags_ids = ",".join([str(t.id) for t in doc.tags.all()]) notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)]) - # custom_fields = ",".join( - # [str(c) for c in CustomMetadata.objects.filter(document=doc)], - # ) + custom_fields = ",".join( + [str(c) for c in CustomFieldInstance.objects.filter(document=doc)], + ) asn = doc.archive_serial_number if asn is not None and ( asn < Document.ARCHIVE_SERIAL_NUMBER_MIN @@ -153,8 +154,8 @@ def update_document(writer: AsyncWriter, doc: Document): has_path=doc.storage_path is not None, notes=notes, num_notes=len(notes), - # custom_metadata=custom_fields, - # custom_field_count=len(custom_fields), + custom_fields=custom_fields, + custom_field_count=len(doc.custom_fields.all()), owner=doc.owner.username if doc.owner else None, owner_id=doc.owner.id if doc.owner else None, has_owner=doc.owner is not None, @@ -193,6 +194,7 @@ class DelayedQuery: "created": ("created", ["date__lt", "date__gt"]), "checksum": ("checksum", ["icontains", "istartswith"]), "original_filename": ("original_filename", ["icontains", "istartswith"]), + "custom_fields": ("custom_fields", ["icontains", "istartswith"]), } def _get_query(self): @@ -358,7 +360,15 @@ class DelayedFullTextQuery(DelayedQuery): def _get_query(self): q_str = self.query_params["query"] qp = MultifieldParser( - ["content", "title", "correspondent", "tag", "type", "notes"], + [ + "content", + "title", + "correspondent", + "tag", + "type", + "notes", + "custom_fields", + ], self.searcher.ixreader.schema, ) qp.add_plugin(DateParserPlugin(basedate=timezone.now())) diff --git a/src/documents/models.py b/src/documents/models.py index 233faac2e..743f2ec1c 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -974,7 +974,7 @@ class CustomFieldInstance(models.Model): ] def __str__(self) -> str: - return str(self.field) + f" : {self.value}" + return str(self.field.name) + f" : {self.value}" @property def value(self): diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py index 8464e1c63..d9359ef3c 100644 --- a/src/documents/tests/test_api.py +++ b/src/documents/tests/test_api.py @@ -35,6 +35,8 @@ from documents import index from documents.data_models import DocumentSource from documents.models import ConsumptionTemplate from documents.models import Correspondent +from documents.models import CustomField +from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType from documents.models import MatchingModel @@ -347,11 +349,36 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): tag_2 = Tag.objects.create(name="t2") tag_3 = Tag.objects.create(name="t3") + cf1 = CustomField.objects.create( + name="stringfield", + data_type=CustomField.FieldDataType.STRING, + ) + cf2 = CustomField.objects.create( + name="numberfield", + data_type=CustomField.FieldDataType.INT, + ) + doc1.tags.add(tag_inbox) doc2.tags.add(tag_2) doc3.tags.add(tag_2) doc3.tags.add(tag_3) + cf1_d1 = CustomFieldInstance.objects.create( + document=doc1, + field=cf1, + value_text="foobard1", + ) + CustomFieldInstance.objects.create( + document=doc1, + field=cf2, + value_int=999, + ) + cf1_d3 = CustomFieldInstance.objects.create( + document=doc3, + field=cf1, + value_text="foobard3", + ) + response = self.client.get("/api/documents/?is_in_inbox=true") self.assertEqual(response.status_code, status.HTTP_200_OK) results = response.data["results"] @@ -423,6 +450,31 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): results = response.data["results"] self.assertEqual(len(results), 0) + # custom field name + response = self.client.get( + f"/api/documents/?custom_fields__icontains={cf1.name}", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + results = response.data["results"] + self.assertEqual(len(results), 2) + + # custom field value + response = self.client.get( + f"/api/documents/?custom_fields__icontains={cf1_d1.value}", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], doc1.id) + + response = self.client.get( + f"/api/documents/?custom_fields__icontains={cf1_d3.value}", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + results = response.data["results"] + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["id"], doc3.id) + def test_document_checksum_filter(self): Document.objects.create( title="none1", @@ -1146,6 +1198,14 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): dt2 = DocumentType.objects.create(name="type2") sp = StoragePath.objects.create(name="path") sp2 = StoragePath.objects.create(name="path2") + cf1 = CustomField.objects.create( + name="string field", + data_type=CustomField.FieldDataType.STRING, + ) + cf2 = CustomField.objects.create( + name="number field", + data_type=CustomField.FieldDataType.INT, + ) d1 = Document.objects.create(checksum="1", correspondent=c, content="test") d2 = Document.objects.create(checksum="2", document_type=dt, content="test") @@ -1176,6 +1236,22 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): content="test", ) + cf1_d1 = CustomFieldInstance.objects.create( + document=d1, + field=cf1, + value_text="foobard1", + ) + cf2_d1 = CustomFieldInstance.objects.create( + document=d1, + field=cf2, + value_int=999, + ) + cf1_d4 = CustomFieldInstance.objects.create( + document=d4, + field=cf1, + value_text="foobard4", + ) + with AsyncWriter(index.open_index()) as writer: for doc in Document.objects.all(): index.update_document(writer, doc) @@ -1304,6 +1380,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"), ), ) + self.assertIn( d5.id, search_query( @@ -1322,6 +1399,27 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): [d4.id, d5.id], ) + self.assertIn( + d1.id, + search_query( + "&custom_fields__icontains=" + cf1_d1.value, + ), + ) + + self.assertIn( + d1.id, + search_query( + "&custom_fields__icontains=" + str(cf2_d1.value), + ), + ) + + self.assertIn( + d4.id, + search_query( + "&custom_fields__icontains=" + cf1_d4.value, + ), + ) + def test_search_filtering_respect_owner(self): """ GIVEN: