lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Basic custom field searching

Browse Source
This commit is contained in:
shamoon 2023-11-03 16:58:29 -07:00
parent ba7be2c57e
commit 4152166693
7 changed files with 194 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts
View File

@ -46,6 +46,7 @@ import {
FILTER_OWNER_ANY,
FILTER_OWNER_DOES_NOT_INCLUDE,
FILTER_OWNER_ISNULL,
FILTER_CUSTOM_FIELDS,
} from 'src/app/data/filter-rule-type'
import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
@ -240,6 +241,18 @@ describe('FilterEditorComponent', () => {
expect(component.textFilterTarget).toEqual('asn') // TEXT_FILTER_TARGET_ASN
}))
it('should ingest text filter rules for custom fields', fakeAsync(() => {
expect(component.textFilter).toEqual(null)
component.filterRules = [
{
rule_type: FILTER_CUSTOM_FIELDS,
value: 'foo',
},
]
expect(component.textFilter).toEqual('foo')
expect(component.textFilterTarget).toEqual('custom-fields') // TEXT_FILTER_TARGET_CUSTOM_FIELDS
}))
it('should ingest text filter rules for doc asn is null', fakeAsync(() => {
expect(component.textFilterTarget).toEqual('title-content')
expect(component.textFilterModifier).toEqual('equals') // TEXT_FILTER_MODIFIER_EQUALS
@ -956,12 +969,30 @@ describe('FilterEditorComponent', () => {
])
}))
it('should convert user input to correct filter rules on full text query', fakeAsync(() => {
it('should convert user input to correct filter rules on custom fields query', fakeAsync(() => {
component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent(new Event('input'))
const textFieldTargetDropdown = fixture.debugElement.queryAll(
By.directive(NgbDropdownItem)
)[3]
textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_CUSTOM_FIELDS
fixture.detectChanges()
tick(400)
expect(component.textFilterTarget).toEqual('custom-fields')
expect(component.filterRules).toEqual([
{
rule_type: FILTER_CUSTOM_FIELDS,
value: 'foo',
},
])
}))
it('should convert user input to correct filter rules on full text query', fakeAsync(() => {
component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent(new Event('input'))
const textFieldTargetDropdown = fixture.debugElement.queryAll(
By.directive(NgbDropdownItem)
)[4]
textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_ASN
fixture.detectChanges()
tick(400)

19
src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts
View File

@ -48,6 +48,7 @@ import {
FILTER_OWNER_DOES_NOT_INCLUDE,
FILTER_OWNER_ISNULL,
FILTER_OWNER_ANY,
FILTER_CUSTOM_FIELDS,
} from 'src/app/data/filter-rule-type'
import {
FilterableDropdownSelectionModel,
@ -74,6 +75,7 @@ const TEXT_FILTER_TARGET_TITLE_CONTENT = 'title-content'
const TEXT_FILTER_TARGET_ASN = 'asn'
const TEXT_FILTER_TARGET_FULLTEXT_QUERY = 'fulltext-query'
const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike'
const TEXT_FILTER_TARGET_CUSTOM_FIELDS = 'custom-fields'
const TEXT_FILTER_MODIFIER_EQUALS = 'equals'
const TEXT_FILTER_MODIFIER_NULL = 'is null'
@ -204,6 +206,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
name: $localize`Title & content`,
},
{ id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN` },
{
id: TEXT_FILTER_TARGET_CUSTOM_FIELDS,
name: $localize`Custom fields`,
},
{
id: TEXT_FILTER_TARGET_FULLTEXT_QUERY,
name: $localize`Advanced search`,
@ -321,6 +327,10 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
this._textFilter = rule.value
this.textFilterTarget = TEXT_FILTER_TARGET_ASN
break
case FILTER_CUSTOM_FIELDS:
this._textFilter = rule.value
this.textFilterTarget = TEXT_FILTER_TARGET_CUSTOM_FIELDS
break
case FILTER_FULLTEXT_QUERY:
let allQueryArgs = rule.value.split(',')
let textQueryArgs = []
@ -552,6 +562,15 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
})
}
}
if (
this._textFilter &&
this.textFilterTarget == TEXT_FILTER_TARGET_CUSTOM_FIELDS
) {
filterRules.push({
rule_type: FILTER_CUSTOM_FIELDS,
value: this._textFilter,
})
}
if (
this._textFilter &&
this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_QUERY

8
src-ui/src/app/data/filter-rule-type.ts
View File

@ -46,6 +46,8 @@ export const FILTER_OWNER_ANY = 33
export const FILTER_OWNER_ISNULL = 34
export const FILTER_OWNER_DOES_NOT_INCLUDE = 35
export const FILTER_CUSTOM_FIELDS = 36
export const FILTER_RULE_TYPES: FilterRuleType[] = [
{
id: FILTER_TITLE,
@ -271,6 +273,12 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [
datatype: 'number',
multi: true,
},
{
id: FILTER_CUSTOM_FIELDS,
filtervar: 'custom_fields__icontains',
datatype: 'string',
multi: false,
},
]
export interface FilterRuleType {

18
src/documents/filters.py
View File

@ -82,6 +82,21 @@ class TitleContentFilter(Filter):
return qs
class CustomFieldsFilter(Filter):
def filter(self, qs, value):
if value:
return (
qs.filter(custom_fields__field__name__icontains=value)
| qs.filter(custom_fields__value_text__icontains=value)
| qs.filter(custom_fields__value_bool__icontains=value)
| qs.filter(custom_fields__value_int__icontains=value)
| qs.filter(custom_fields__value_date__icontains=value)
| qs.filter(custom_fields__value_url__icontains=value)
)
else:
return qs
class DocumentFilterSet(FilterSet):
is_tagged = BooleanFilter(
label="Is tagged",
@ -108,6 +123,8 @@ class DocumentFilterSet(FilterSet):
owner__id__none = ObjectFilter(field_name="owner", exclude=True)
custom_fields__icontains = CustomFieldsFilter()
class Meta:
model = Document
fields = {
@ -132,6 +149,7 @@ class DocumentFilterSet(FilterSet):
"storage_path__name": CHAR_KWARGS,
"owner": ["isnull"],
"owner__id": ID_KWARGS,
"custom_fields": ["icontains"],
}

26
src/documents/index.py
View File

@ -31,6 +31,7 @@ from whoosh.searching import Searcher
from whoosh.writing import AsyncWriter
# from documents.models import CustomMetadata
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.models import User
@ -61,8 +62,8 @@ def get_schema():
has_path=BOOLEAN(),
notes=TEXT(),
num_notes=NUMERIC(sortable=True, signed=False),
# custom_metadata=TEXT(),
# custom_field_count=NUMERIC(sortable=True, signed=False),
custom_fields=TEXT(),
custom_field_count=NUMERIC(sortable=True, signed=False),
owner=TEXT(),
owner_id=NUMERIC(),
has_owner=BOOLEAN(),
@ -111,9 +112,9 @@ def update_document(writer: AsyncWriter, doc: Document):
tags = ",".join([t.name for t in doc.tags.all()])
tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
notes = ",".join([str(c.note) for c in Note.objects.filter(document=doc)])
# custom_fields = ",".join(
# [str(c) for c in CustomMetadata.objects.filter(document=doc)],
# )
custom_fields = ",".join(
[str(c) for c in CustomFieldInstance.objects.filter(document=doc)],
)
asn = doc.archive_serial_number
if asn is not None and (
asn < Document.ARCHIVE_SERIAL_NUMBER_MIN
@ -153,8 +154,8 @@ def update_document(writer: AsyncWriter, doc: Document):
has_path=doc.storage_path is not None,
notes=notes,
num_notes=len(notes),
# custom_metadata=custom_fields,
# custom_field_count=len(custom_fields),
custom_fields=custom_fields,
custom_field_count=len(doc.custom_fields.all()),
owner=doc.owner.username if doc.owner else None,
owner_id=doc.owner.id if doc.owner else None,
has_owner=doc.owner is not None,
@ -193,6 +194,7 @@ class DelayedQuery:
"created": ("created", ["date__lt", "date__gt"]),
"checksum": ("checksum", ["icontains", "istartswith"]),
"original_filename": ("original_filename", ["icontains", "istartswith"]),
"custom_fields": ("custom_fields", ["icontains", "istartswith"]),
}
def _get_query(self):
@ -358,7 +360,15 @@ class DelayedFullTextQuery(DelayedQuery):
def _get_query(self):
q_str = self.query_params["query"]
qp = MultifieldParser(
["content", "title", "correspondent", "tag", "type", "notes"],
[
"content",
"title",
"correspondent",
"tag",
"type",
"notes",
"custom_fields",
],
self.searcher.ixreader.schema,
)
qp.add_plugin(DateParserPlugin(basedate=timezone.now()))

2
src/documents/models.py
View File

@ -974,7 +974,7 @@ class CustomFieldInstance(models.Model):
]
def __str__(self) -> str:
return str(self.field) + f" : {self.value}"
return str(self.field.name) + f" : {self.value}"
@property
def value(self):

98
src/documents/tests/test_api.py
View File

@ -35,6 +35,8 @@ from documents import index
from documents.data_models import DocumentSource
from documents.models import ConsumptionTemplate
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
@ -347,11 +349,36 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
tag_2 = Tag.objects.create(name="t2")
tag_3 = Tag.objects.create(name="t3")
cf1 = CustomField.objects.create(
name="stringfield",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="numberfield",
data_type=CustomField.FieldDataType.INT,
)
doc1.tags.add(tag_inbox)
doc2.tags.add(tag_2)
doc3.tags.add(tag_2)
doc3.tags.add(tag_3)
cf1_d1 = CustomFieldInstance.objects.create(
document=doc1,
field=cf1,
value_text="foobard1",
)
CustomFieldInstance.objects.create(
document=doc1,
field=cf2,
value_int=999,
)
cf1_d3 = CustomFieldInstance.objects.create(
document=doc3,
field=cf1,
value_text="foobard3",
)
response = self.client.get("/api/documents/?is_in_inbox=true")
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
@ -423,6 +450,31 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
results = response.data["results"]
self.assertEqual(len(results), 0)
# custom field name
response = self.client.get(
f"/api/documents/?custom_fields__icontains={cf1.name}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
self.assertEqual(len(results), 2)
# custom field value
response = self.client.get(
f"/api/documents/?custom_fields__icontains={cf1_d1.value}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], doc1.id)
response = self.client.get(
f"/api/documents/?custom_fields__icontains={cf1_d3.value}",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
results = response.data["results"]
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], doc3.id)
def test_document_checksum_filter(self):
Document.objects.create(
title="none1",
@ -1146,6 +1198,14 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
dt2 = DocumentType.objects.create(name="type2")
sp = StoragePath.objects.create(name="path")
sp2 = StoragePath.objects.create(name="path2")
cf1 = CustomField.objects.create(
name="string field",
data_type=CustomField.FieldDataType.STRING,
)
cf2 = CustomField.objects.create(
name="number field",
data_type=CustomField.FieldDataType.INT,
)
d1 = Document.objects.create(checksum="1", correspondent=c, content="test")
d2 = Document.objects.create(checksum="2", document_type=dt, content="test")
@ -1176,6 +1236,22 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
content="test",
)
cf1_d1 = CustomFieldInstance.objects.create(
document=d1,
field=cf1,
value_text="foobard1",
)
cf2_d1 = CustomFieldInstance.objects.create(
document=d1,
field=cf2,
value_int=999,
)
cf1_d4 = CustomFieldInstance.objects.create(
document=d4,
field=cf1,
value_text="foobard4",
)
with AsyncWriter(index.open_index()) as writer:
for doc in Document.objects.all():
index.update_document(writer, doc)
@ -1304,6 +1380,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
+ datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"),
),
)
self.assertIn(
d5.id,
search_query(
@ -1322,6 +1399,27 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase):
[d4.id, d5.id],
)
self.assertIn(
d1.id,
search_query(
"&custom_fields__icontains=" + cf1_d1.value,
),
)
self.assertIn(
d1.id,
search_query(
"&custom_fields__icontains=" + str(cf2_d1.value),
),
)
self.assertIn(
d4.id,
search_query(
"&custom_fields__icontains=" + cf1_d4.value,
),
)
def test_search_filtering_respect_owner(self):
"""
GIVEN: