Basic custom field searching

src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts

@@ -46,6 +46,7 @@ import {
   FILTER_OWNER_ANY,
   FILTER_OWNER_DOES_NOT_INCLUDE,
   FILTER_OWNER_ISNULL,
+  FILTER_CUSTOM_FIELDS,
 } from 'src/app/data/filter-rule-type'
 import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
 import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
@@ -240,6 +241,18 @@ describe('FilterEditorComponent', () => {
     expect(component.textFilterTarget).toEqual('asn') // TEXT_FILTER_TARGET_ASN
   }))
 
+  it('should ingest text filter rules for custom fields', fakeAsync(() => {
+    expect(component.textFilter).toEqual(null)
+    component.filterRules = [
+      {
+        rule_type: FILTER_CUSTOM_FIELDS,
+        value: 'foo',
+      },
+    ]
+    expect(component.textFilter).toEqual('foo')
+    expect(component.textFilterTarget).toEqual('custom-fields') // TEXT_FILTER_TARGET_CUSTOM_FIELDS
+  }))
+
   it('should ingest text filter rules for doc asn is null', fakeAsync(() => {
     expect(component.textFilterTarget).toEqual('title-content')
     expect(component.textFilterModifier).toEqual('equals') // TEXT_FILTER_MODIFIER_EQUALS
@@ -956,12 +969,30 @@ describe('FilterEditorComponent', () => {
     ])
   }))
 
-  it('should convert user input to correct filter rules on full text query', fakeAsync(() => {
+  it('should convert user input to correct filter rules on custom fields query', fakeAsync(() => {
     component.textFilterInput.nativeElement.value = 'foo'
     component.textFilterInput.nativeElement.dispatchEvent(new Event('input'))
     const textFieldTargetDropdown = fixture.debugElement.queryAll(
       By.directive(NgbDropdownItem)
     )[3]
+    textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_CUSTOM_FIELDS
+    fixture.detectChanges()
+    tick(400)
+    expect(component.textFilterTarget).toEqual('custom-fields')
+    expect(component.filterRules).toEqual([
+      {
+        rule_type: FILTER_CUSTOM_FIELDS,
+        value: 'foo',
+      },
+    ])
+  }))
+
+  it('should convert user input to correct filter rules on full text query', fakeAsync(() => {
+    component.textFilterInput.nativeElement.value = 'foo'
+    component.textFilterInput.nativeElement.dispatchEvent(new Event('input'))
+    const textFieldTargetDropdown = fixture.debugElement.queryAll(
+      By.directive(NgbDropdownItem)
+    )[4]
     textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_ASN
     fixture.detectChanges()
     tick(400)

src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts

@@ -48,6 +48,7 @@ import {
   FILTER_OWNER_DOES_NOT_INCLUDE,
   FILTER_OWNER_ISNULL,
   FILTER_OWNER_ANY,
+  FILTER_CUSTOM_FIELDS,
 } from 'src/app/data/filter-rule-type'
 import {
   FilterableDropdownSelectionModel,
@@ -74,6 +75,7 @@ const TEXT_FILTER_TARGET_TITLE_CONTENT = 'title-content'
 const TEXT_FILTER_TARGET_ASN = 'asn'
 const TEXT_FILTER_TARGET_FULLTEXT_QUERY = 'fulltext-query'
 const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike'
+const TEXT_FILTER_TARGET_CUSTOM_FIELDS = 'custom-fields'
 
 const TEXT_FILTER_MODIFIER_EQUALS = 'equals'
 const TEXT_FILTER_MODIFIER_NULL = 'is null'
@@ -204,6 +206,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
         name: $localize`Title & content`,
       },
       { id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN` },
+      {
+        id: TEXT_FILTER_TARGET_CUSTOM_FIELDS,
+        name: $localize`Custom fields`,
+      },
       {
         id: TEXT_FILTER_TARGET_FULLTEXT_QUERY,
         name: $localize`Advanced search`,
@@ -321,6 +327,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
           this._textFilter = rule.value
           this.textFilterTarget = TEXT_FILTER_TARGET_ASN
           break
+        case FILTER_CUSTOM_FIELDS:
+          this._textFilter = rule.value
+          this.textFilterTarget = TEXT_FILTER_TARGET_CUSTOM_FIELDS
+          break
         case FILTER_FULLTEXT_QUERY:
           let allQueryArgs = rule.value.split(',')
           let textQueryArgs = []
@@ -552,6 +562,15 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
         })
       }
     }
+    if (
+      this._textFilter &&
+      this.textFilterTarget == TEXT_FILTER_TARGET_CUSTOM_FIELDS
+    ) {
+      filterRules.push({
+        rule_type: FILTER_CUSTOM_FIELDS,
+        value: this._textFilter,
+      })
+    }
     if (
       this._textFilter &&
       this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_QUERY

src-ui/src/app/data/filter-rule-type.ts

@@ -46,6 +46,8 @@ export const FILTER_OWNER_ANY = 33
 export const FILTER_OWNER_ISNULL = 34
 export const FILTER_OWNER_DOES_NOT_INCLUDE = 35
 
+export const FILTER_CUSTOM_FIELDS = 36
+
 export const FILTER_RULE_TYPES: FilterRuleType[] = [
   {
     id: FILTER_TITLE,
@@ -271,6 +273,12 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [
     datatype: 'number',
     multi: true,
   },
+  {
+    id: FILTER_CUSTOM_FIELDS,
+    filtervar: 'custom_fields__icontains',
+    datatype: 'string',
+    multi: false,
+  },
 ]
 
 export interface FilterRuleType {

src/documents/filters.py

@@ -82,6 +82,21 @@ class TitleContentFilter(Filter):
             return qs
 
 
+class CustomFieldsFilter(Filter):
+    def filter(self, qs, value):
+        if value:
+            return (
+                qs.filter(custom_fields__field__name__icontains=value)
+                | qs.filter(custom_fields__value_text__icontains=value)
+                | qs.filter(custom_fields__value_bool__icontains=value)
+                | qs.filter(custom_fields__value_int__icontains=value)
+                | qs.filter(custom_fields__value_date__icontains=value)
+                | qs.filter(custom_fields__value_url__icontains=value)
+            )
+        else:
+            return qs
+
+
 class DocumentFilterSet(FilterSet):
     is_tagged = BooleanFilter(
         label="Is tagged",
@@ -108,6 +123,8 @@ class DocumentFilterSet(FilterSet):
 
     owner__id__none = ObjectFilter(field_name="owner", exclude=True)
 
+    custom_fields__icontains = CustomFieldsFilter()
+
     class Meta:
         model = Document
         fields = {
@@ -132,6 +149,7 @@ class DocumentFilterSet(FilterSet):
             "storage_path__name": CHAR_KWARGS,
             "owner": ["isnull"],
             "owner__id": ID_KWARGS,
+            "custom_fields": ["icontains"],
         }
 
 

src/documents/index.py

@@ -31,6 +31,7 @@ from whoosh.searching import Searcher
 from whoosh.writing import AsyncWriter
 
 # from documents.models import CustomMetadata
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import Note
 from documents.models import User
@@ -61,8 +62,8 @@ def get_schema():
         has_path=BOOLEAN(),
         notes=TEXT(),
         num_notes=NUMERIC(sortable=True, signed=False),
-        # custom_metadata=TEXT(),
+        custom_fields=TEXT(),
-        # custom_field_count=NUMERIC(sortable=True, signed=False),
+        custom_field_count=NUMERIC(sortable=True, signed=False),
         owner=TEXT(),
         owner_id=NUMERIC(),
         has_owner=BOOLEAN(),
@@ -111,9 +112,9 @@ def update_document(writer: AsyncWriter, doc: Document):
     tags = ",".join([t.name for t in doc.tags.all()])
     tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
     notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
-    # custom_fields = ",".join(
+    custom_fields = ",".join(
-    #    [str(c) for c in CustomMetadata.objects.filter(document=doc)],
+        [str(c) for c in CustomFieldInstance.objects.filter(document=doc)],
-    # )
+    )
     asn = doc.archive_serial_number
     if asn is not None and (
         asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@@ -153,8 +154,8 @@ def update_document(writer: AsyncWriter, doc: Document):
         has_path=doc.storage_path is not None,
         notes=notes,
         num_notes=len(notes),
-        # custom_metadata=custom_fields,
+        custom_fields=custom_fields,
-        # custom_field_count=len(custom_fields),
+        custom_field_count=len(doc.custom_fields.all()),
         owner=doc.owner.username if doc.owner else None,
         owner_id=doc.owner.id if doc.owner else None,
         has_owner=doc.owner is not None,
@@ -193,6 +194,7 @@ class DelayedQuery:
         "created": ("created", ["date__lt", "date__gt"]),
         "checksum": ("checksum", ["icontains", "istartswith"]),
         "original_filename": ("original_filename", ["icontains", "istartswith"]),
+        "custom_fields": ("custom_fields", ["icontains", "istartswith"]),
     }
 
     def _get_query(self):
@@ -358,7 +360,15 @@ class DelayedFullTextQuery(DelayedQuery):
     def _get_query(self):
         q_str = self.query_params["query"]
         qp = MultifieldParser(
-            ["content", "title", "correspondent", "tag", "type", "notes"],
+            [
+                "content",
+                "title",
+                "correspondent",
+                "tag",
+                "type",
+                "notes",
+                "custom_fields",
+            ],
             self.searcher.ixreader.schema,
         )
         qp.add_plugin(DateParserPlugin(basedate=timezone.now()))

src/documents/models.py

@@ -974,7 +974,7 @@ class CustomFieldInstance(models.Model):
         ]
 
     def __str__(self) -> str:
-        return str(self.field) + f" : {self.value}"
+        return str(self.field.name) + f" : {self.value}"
 
     @property
     def value(self):

src/documents/tests/test_api.py

@@ -35,6 +35,8 @@ from documents import index
 from documents.data_models import DocumentSource
 from documents.models import ConsumptionTemplate
 from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import MatchingModel
@@ -347,11 +349,36 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         tag_2 = Tag.objects.create(name="t2")
         tag_3 = Tag.objects.create(name="t3")
 
+        cf1 = CustomField.objects.create(
+            name="stringfield",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        cf2 = CustomField.objects.create(
+            name="numberfield",
+            data_type=CustomField.FieldDataType.INT,
+        )
+
         doc1.tags.add(tag_inbox)
         doc2.tags.add(tag_2)
         doc3.tags.add(tag_2)
         doc3.tags.add(tag_3)
 
+        cf1_d1 = CustomFieldInstance.objects.create(
+            document=doc1,
+            field=cf1,
+            value_text="foobard1",
+        )
+        CustomFieldInstance.objects.create(
+            document=doc1,
+            field=cf2,
+            value_int=999,
+        )
+        cf1_d3 = CustomFieldInstance.objects.create(
+            document=doc3,
+            field=cf1,
+            value_text="foobard3",
+        )
+
         response = self.client.get("/api/documents/?is_in_inbox=true")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         results = response.data["results"]
@@ -423,6 +450,31 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         results = response.data["results"]
         self.assertEqual(len(results), 0)
 
+        # custom field name
+        response = self.client.get(
+            f"/api/documents/?custom_fields__icontains={cf1.name}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        results = response.data["results"]
+        self.assertEqual(len(results), 2)
+
+        # custom field value
+        response = self.client.get(
+            f"/api/documents/?custom_fields__icontains={cf1_d1.value}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        results = response.data["results"]
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["id"], doc1.id)
+
+        response = self.client.get(
+            f"/api/documents/?custom_fields__icontains={cf1_d3.value}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        results = response.data["results"]
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0]["id"], doc3.id)
+
     def test_document_checksum_filter(self):
         Document.objects.create(
             title="none1",
@@ -1146,6 +1198,14 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
         dt2 = DocumentType.objects.create(name="type2")
         sp = StoragePath.objects.create(name="path")
         sp2 = StoragePath.objects.create(name="path2")
+        cf1 = CustomField.objects.create(
+            name="string field",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        cf2 = CustomField.objects.create(
+            name="number field",
+            data_type=CustomField.FieldDataType.INT,
+        )
 
         d1 = Document.objects.create(checksum="1", correspondent=c, content="test")
         d2 = Document.objects.create(checksum="2", document_type=dt, content="test")
@@ -1176,6 +1236,22 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             content="test",
         )
 
+        cf1_d1 = CustomFieldInstance.objects.create(
+            document=d1,
+            field=cf1,
+            value_text="foobard1",
+        )
+        cf2_d1 = CustomFieldInstance.objects.create(
+            document=d1,
+            field=cf2,
+            value_int=999,
+        )
+        cf1_d4 = CustomFieldInstance.objects.create(
+            document=d4,
+            field=cf1,
+            value_text="foobard4",
+        )
+
         with AsyncWriter(index.open_index()) as writer:
             for doc in Document.objects.all():
                 index.update_document(writer, doc)
@@ -1304,6 +1380,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
                 + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"),
             ),
         )
+
         self.assertIn(
             d5.id,
             search_query(
@@ -1322,6 +1399,27 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
             [d4.id, d5.id],
         )
 
+        self.assertIn(
+            d1.id,
+            search_query(
+                "&custom_fields__icontains=" + cf1_d1.value,
+            ),
+        )
+
+        self.assertIn(
+            d1.id,
+            search_query(
+                "&custom_fields__icontains=" + str(cf2_d1.value),
+            ),
+        )
+
+        self.assertIn(
+            d4.id,
+            search_query(
+                "&custom_fields__icontains=" + cf1_d4.value,
+            ),
+        )
+
     def test_search_filtering_respect_owner(self):
         """
         GIVEN: