Support consumption template source

Pipfile

@@ -3,7 +3,6 @@ url = "https://pypi.python.org/simple"
 verify_ssl = true
 name = "pypi"
 
-
 [packages]
 dateparser = "~=1.1"
 # WARNING: django does not use semver.
@@ -51,6 +50,7 @@ pdf2image = "*"
 flower = "*"
 bleach = "*"
 zxing-cpp = {version = "*", platform_machine = "== 'x86_64'"}
+django-multiselectfield = "*"
 
 [dev-packages]
 # Linting

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.html

@@ -10,6 +10,7 @@
           <pngx-input-text i18n-title title="Name" formControlName="name" [error]="error?.name"></pngx-input-text>
           <pngx-input-number i18n-title title="Rule order" formControlName="order" [showAdd]="false" [error]="error?.order"></pngx-input-number>
           <p class="small" i18n>Paperless-ngx will process mails that match <em>any</em> of the filters specified below.</p>
+          <pngx-input-select i18n-title title="Filter sources" [items]="sourceOptions" [multiple]="true" formControlName="sources" [error]="error?.filter_filename"></pngx-input-select>
           <pngx-input-text i18n-title title="Filter filename" formControlName="filter_filename" i18n-hint hint="Apply template to documents that match this filename. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." [error]="error?.filter_filename"></pngx-input-text>
           <pngx-input-text i18n-title title="Filter path" formControlName="filter_path" i18n-hint hint="Apply template to documents that match this path. Wildcards specified as * are allowed. Case insensitive." [error]="error?.filter_path"></pngx-input-text>
         </div>

src-ui/src/app/components/common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component.ts

@@ -2,16 +2,12 @@ import { Component } from '@angular/core'
 import { FormGroup, FormControl } from '@angular/forms'
 import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'
 import { first } from 'rxjs'
-import { PaperlessConsumptionTemplate } from 'src/app/data/paperless-consumption-template'
+import {
+  DocumentSource,
+  PaperlessConsumptionTemplate,
+} from 'src/app/data/paperless-consumption-template'
 import { PaperlessCorrespondent } from 'src/app/data/paperless-correspondent'
 import { PaperlessDocumentType } from 'src/app/data/paperless-document-type'
-import {
-  MailFilterAttachmentType,
-  MailRuleConsumptionScope,
-  MailAction,
-  MailMetadataTitleOption,
-  MailMetadataCorrespondentOption,
-} from 'src/app/data/paperless-mail-rule'
 import { PaperlessStoragePath } from 'src/app/data/paperless-storage-path'
 import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service'
 import { CorrespondentService } from 'src/app/services/rest/correspondent.service'
@@ -21,6 +17,21 @@ import { UserService } from 'src/app/services/rest/user.service'
 import { SettingsService } from 'src/app/services/settings.service'
 import { EditDialogComponent } from '../edit-dialog.component'
 
+const SOURCE_OPTIONS = [
+  {
+    id: DocumentSource.ConsumeFolder,
+    name: $localize`Documents uploaded via consume folder`,
+  },
+  {
+    id: DocumentSource.ApiUpload,
+    name: $localize`Documents uploaded via api upload`,
+  },
+  {
+    id: DocumentSource.MailFetch,
+    name: $localize`Documents uploaded via mail fetch`,
+  },
+]
+
 @Component({
   selector: 'pngx-consumption-template-edit-dialog',
   templateUrl: './consumption-template-edit-dialog.component.html',
@@ -74,6 +85,7 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
       filter_filename: new FormControl(null),
       filter_path: new FormControl(null),
       order: new FormControl(null),
+      sources: new FormControl([]),
       assign_tags: new FormControl([]),
       assign_owner: new FormControl(null),
       assign_document_type: new FormControl(null),
@@ -85,4 +97,8 @@ export class ConsumptionTemplateEditDialogComponent extends EditDialogComponent<
       assign_change_groups: new FormControl(null),
     })
   }
+
+  get sourceOptions() {
+    return SOURCE_OPTIONS
+  }
 }

src-ui/src/app/components/manage/consmption-templates-list/consmption-templates-list.component.spec.ts

@@ -9,7 +9,10 @@ import {
   NgbModalModule,
 } from '@ng-bootstrap/ng-bootstrap'
 import { of, throwError } from 'rxjs'
-import { PaperlessConsumptionTemplate } from 'src/app/data/paperless-consumption-template'
+import {
+  DocumentSource,
+  PaperlessConsumptionTemplate,
+} from 'src/app/data/paperless-consumption-template'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
 import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service'
 import { ToastService } from 'src/app/services/toast.service'
@@ -24,6 +27,11 @@ const templates: PaperlessConsumptionTemplate[] = [
     id: 0,
     name: 'Template 1',
     order: 0,
+    sources: [
+      DocumentSource.ConsumeFolder,
+      DocumentSource.ApiUpload,
+      DocumentSource.MailFetch,
+    ],
     filter_filename: 'foo',
     filter_path: 'bar',
     assign_tags: [1, 2, 3],
@@ -32,6 +40,7 @@ const templates: PaperlessConsumptionTemplate[] = [
     id: 1,
     name: 'Template 2',
     order: 1,
+    sources: [DocumentSource.MailFetch],
     filter_filename: null,
     filter_path: 'foo/bar',
     assign_owner: 1,

src-ui/src/app/data/paperless-consumption-template.ts

@@ -1,10 +1,18 @@
 import { ObjectWithPermissions } from './object-with-permissions'
 
+export enum DocumentSource {
+  ConsumeFolder = 1,
+  ApiUpload = 2,
+  MailFetch = 3,
+}
+
 export interface PaperlessConsumptionTemplate extends ObjectWithPermissions {
   name: string
 
   order: number
 
+  sources: DocumentSource[]
+
   filter_filename: string
 
   filter_path: string

src/documents/consumer.py

@@ -21,6 +21,7 @@ from django.utils import timezone
 from filelock import FileLock
 from rest_framework.reverse import reverse
 
+from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentMetadataOverrides
 from documents.permissions import set_permissions_for_object
 from documents.utils import copy_basic_file_stats
@@ -595,16 +596,25 @@ class Consumer(LoggingMixin):
 
     def get_template_overrides(
         self,
-        input_doc: Path,
+        input_doc: ConsumableDocument,
     ) -> DocumentMetadataOverrides:
         overrides = DocumentMetadataOverrides()
         for template in ConsumptionTemplate.objects.all():
             template_overrides = DocumentMetadataOverrides()
 
-            if fnmatch(
+            if int(input_doc.source) in list(template.sources) and (
-                input_doc.name.lower(),
+                (
-                template.filter_filename.lower(),
+                    template.filter_filename is not None
-            ) or input_doc.match(template.filter_path):
+                    and fnmatch(
+                        input_doc.original_file.name.lower(),
+                        template.filter_filename.lower(),
+                    )
+                )
+                or (
+                    template.filter_path is not None
+                    and input_doc.original_file.match(template.filter_path)
+                )
+            ):
                 self.log.info(f"Document matched consumption template {template.name}")
                 if template.assign_tags is not None:
                     template_overrides.tag_ids = [

src/documents/migrations/1039_consumptiontemplate.py

@@ -1,10 +1,13 @@
-# Generated by Django 4.1.11 on 2023-09-16 05:01
+# Generated by Django 4.1.11 on 2023-09-16 18:04
 
 import django.db.models.deletion
+import multiselectfield.db.fields
 from django.conf import settings
 from django.db import migrations
 from django.db import models
 
+# TODO: migrate permissions, upgrade mail rules?
+
 
 class Migration(migrations.Migration):
     dependencies = [
@@ -31,6 +34,18 @@ class Migration(migrations.Migration):
                     models.CharField(max_length=256, unique=True, verbose_name="name"),
                 ),
                 ("order", models.IntegerField(default=0, verbose_name="order")),
+                (
+                    "sources",
+                    multiselectfield.db.fields.MultiSelectField(
+                        choices=[
+                            (1, "Consume Folder"),
+                            (2, "Api Upload"),
+                            (3, "Mail Fetch"),
+                        ],
+                        default="1,2,3",
+                        max_length=3,
+                    ),
+                ),
                 (
                     "filter_path",
                     models.CharField(

src/documents/models.py

@@ -18,7 +18,9 @@ from django.core.validators import MinValueValidator
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from multiselectfield import MultiSelectField
 
+from documents.data_models import DocumentSource
 from documents.parsers import get_default_file_extension
 
 ALL_STATES = sorted(states.ALL_STATES)
@@ -738,6 +740,13 @@ class ShareLink(models.Model):
         return f"Share Link for {self.document.title}"
 
 
+DOCUMENT_SOURCE = (
+    (int(DocumentSource.ConsumeFolder), _("Consume Folder")),
+    (int(DocumentSource.ApiUpload), _("Api Upload")),
+    (int(DocumentSource.MailFetch), _("Mail Fetch")),
+)
+
+
 class ConsumptionTemplate(ModelWithOwner):
     class Meta:
         verbose_name = _("consumption template")
@@ -747,6 +756,12 @@ class ConsumptionTemplate(ModelWithOwner):
 
     order = models.IntegerField(_("order"), default=0)
 
+    sources = MultiSelectField(
+        max_length=3,
+        choices=DOCUMENT_SOURCE,
+        default=f"{DocumentSource.ConsumeFolder},{DocumentSource.ApiUpload},{DocumentSource.MailFetch}",
+    )
+
     filter_path = models.CharField(
         _("filter path"),
         max_length=256,

src/documents/serialisers.py

@@ -13,13 +13,16 @@ from django.utils.text import slugify
 from django.utils.translation import gettext as _
 from guardian.core import ObjectPermissionChecker
 from guardian.shortcuts import get_users_with_perms
+from rest_framework import fields
 from rest_framework import serializers
 from rest_framework.fields import SerializerMethodField
 
+from documents.data_models import DocumentSource
 from documents.permissions import get_groups_with_only_permission
 from documents.permissions import set_permissions_for_object
 
 from . import bulk_edit
+from .models import DOCUMENT_SOURCE
 from .models import ConsumptionTemplate
 from .models import Correspondent
 from .models import Document
@@ -1039,17 +1042,28 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions
 
 
 class ConsumptionTemplateSerializer(OwnedObjectSerializer):
+    order = serializers.IntegerField(required=False)
+    sources = fields.MultipleChoiceField(
+        choices=DOCUMENT_SOURCE,
+        allow_empty=False,
+        default={
+            DocumentSource.ConsumeFolder,
+            DocumentSource.ApiUpload,
+            DocumentSource.MailFetch,
+        },
+    )
     assign_correspondent = CorrespondentField(allow_null=True, required=False)
     assign_tags = TagsField(many=True, allow_null=True, required=False)
     assign_document_type = DocumentTypeField(allow_null=True, required=False)
     assign_storage_path = StoragePathField(allow_null=True, required=False)
-    order = serializers.IntegerField(required=False)
 
     class Meta:
         model = ConsumptionTemplate
         fields = [
             "id",
             "name",
+            "order",
+            "sources",
             "filter_path",
             "filter_filename",
             "assign_tags",
@@ -1061,7 +1075,6 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
             "assign_view_groups",
             "assign_change_users",
             "assign_change_groups",
-            "order",
             "owner",
             "user_can_change",
             "permissions",
@@ -1081,6 +1094,7 @@ class ConsumptionTemplateSerializer(OwnedObjectSerializer):
     #     return mail_rule
 
     # def validate(self, attrs):
+    # TODO: require path or filename filter
     #     if (
     #         attrs["action"] == ConsumptionTemplate.MailAction.TAG
     #         or attrs["action"] == ConsumptionTemplate.MailAction.MOVE

src/documents/tasks.py

@@ -155,7 +155,7 @@ def consume_file(
                 logger.info(f"Found ASN in barcode: {overrides.asn}")
 
     template_overrides = Consumer().get_template_overrides(
-        input_doc=input_doc.original_file,
+        input_doc=input_doc,
     )
 
     overrides = merge_overrides(overridesA=overrides, overridesB=template_overrides)