lgcosta/paperless-ngx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use discreet model for email / webhook

Browse Source
This commit is contained in:
shamoon
2024-11-25 21:09:13 -08:00
parent 69a4331d99
commit 6b36ee7819
11 changed files with 501 additions and 323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
src/documents/migrations/1059_workflowaction_email_body_and_more.py
View File

@@ -1,119 +0,0 @@
# Generated by Django 5.1.3 on 2024-11-24 18:30
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1058_workflowtrigger_schedule_date_custom_field_and_more"),
]
operations = [
migrations.AddField(
model_name="workflowaction",
name="email_body",
field=models.TextField(
blank=True,
help_text="The body (message) of the email, can include some placeholders, see documentation.",
null=True,
verbose_name="email body",
),
),
migrations.AddField(
model_name="workflowaction",
name="email_include_document",
field=models.BooleanField(
default=False,
verbose_name="include document in email",
),
),
migrations.AddField(
model_name="workflowaction",
name="email_subject",
field=models.CharField(
blank=True,
help_text="The subject of the email, can include some placeholders, see documentation.",
max_length=256,
null=True,
verbose_name="email subject",
),
),
migrations.AddField(
model_name="workflowaction",
name="email_to",
field=models.TextField(
blank=True,
help_text="The destination email addresses, comma separated.",
null=True,
verbose_name="emails to",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook_body",
field=models.TextField(
blank=True,
help_text="The body to send with the webhook URL if parameters not used.",
null=True,
verbose_name="webhook body",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook_headers",
field=models.JSONField(
blank=True,
help_text="The headers to send with the webhook URL.",
null=True,
verbose_name="webhook headers",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook_include_document",
field=models.BooleanField(
default=False,
verbose_name="include document in webhook",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook_params",
field=models.JSONField(
blank=True,
help_text="The parameters to send with the webhook URL if body not used.",
null=True,
verbose_name="webhook parameters",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook_url",
field=models.URLField(
blank=True,
help_text="The destination URL for the notification.",
null=True,
verbose_name="webhook url",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook_use_params",
field=models.BooleanField(default=True, verbose_name="use parameters"),
),
migrations.AlterField(
model_name="workflowaction",
name="type",
field=models.PositiveIntegerField(
choices=[
(1, "Assignment"),
(2, "Removal"),
(3, "Email"),
(4, "Webhook"),
],
default=1,
verbose_name="Workflow Action Type",
),
),
]

154
src/documents/migrations/1059_workflowactionemail_workflowactionwebhook_and_more.py Normal file
View File

@@ -0,0 +1,154 @@
# Generated by Django 5.1.3 on 2024-11-26 04:07
import django.db.models.deletion
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "1058_workflowtrigger_schedule_date_custom_field_and_more"),
]
operations = [
migrations.CreateModel(
name="WorkflowActionEmail",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"subject",
models.CharField(
help_text="The subject of the email, can include some placeholders, see documentation.",
max_length=256,
verbose_name="email subject",
),
),
(
"body",
models.TextField(
help_text="The body (message) of the email, can include some placeholders, see documentation.",
verbose_name="email body",
),
),
(
"to",
models.TextField(
help_text="The destination email addresses, comma separated.",
verbose_name="emails to",
),
),
(
"include_document",
models.BooleanField(
default=False,
verbose_name="include document in email",
),
),
],
),
migrations.CreateModel(
name="WorkflowActionWebhook",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"url",
models.URLField(
help_text="The destination URL for the notification.",
verbose_name="webhook url",
),
),
(
"use_params",
models.BooleanField(default=True, verbose_name="use parameters"),
),
(
"params",
models.JSONField(
blank=True,
help_text="The parameters to send with the webhook URL if body not used.",
null=True,
verbose_name="webhook parameters",
),
),
(
"body",
models.TextField(
blank=True,
help_text="The body to send with the webhook URL if parameters not used.",
null=True,
verbose_name="webhook body",
),
),
(
"headers",
models.JSONField(
blank=True,
help_text="The headers to send with the webhook URL.",
null=True,
verbose_name="webhook headers",
),
),
(
"include_document",
models.BooleanField(
default=False,
verbose_name="include document in webhook",
),
),
],
),
migrations.AlterField(
model_name="workflowaction",
name="type",
field=models.PositiveIntegerField(
choices=[
(1, "Assignment"),
(2, "Removal"),
(3, "Email"),
(4, "Webhook"),
],
default=1,
verbose_name="Workflow Action Type",
),
),
migrations.AddField(
model_name="workflowaction",
name="email",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="action",
to="documents.workflowactionemail",
verbose_name="email",
),
),
migrations.AddField(
model_name="workflowaction",
name="webhook",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="action",
to="documents.workflowactionwebhook",
verbose_name="webhook",
),
),
]

154
src/documents/models.py
View File

@@ -1156,6 +1156,85 @@ class WorkflowTrigger(models.Model):
return f"WorkflowTrigger {self.pk}"
class WorkflowActionEmail(models.Model):
subject = models.CharField(
_("email subject"),
max_length=256,
null=False,
help_text=_(
"The subject of the email, can include some placeholders, "
"see documentation.",
),
)
body = models.TextField(
_("email body"),
null=False,
help_text=_(
"The body (message) of the email, can include some placeholders, "
"see documentation.",
),
)
to = models.TextField(
_("emails to"),
null=False,
help_text=_(
"The destination email addresses, comma separated.",
),
)
include_document = models.BooleanField(
default=False,
verbose_name=_("include document in email"),
)
def __str__(self):
return f"Workflow Email Action {self.pk}"
class WorkflowActionWebhook(models.Model):
url = models.URLField(
_("webhook url"),
null=False,
help_text=_("The destination URL for the notification."),
)
use_params = models.BooleanField(
default=True,
verbose_name=_("use parameters"),
)
params = models.JSONField(
_("webhook parameters"),
null=True,
blank=True,
help_text=_("The parameters to send with the webhook URL if body not used."),
)
body = models.TextField(
_("webhook body"),
null=True,
blank=True,
help_text=_("The body to send with the webhook URL if parameters not used."),
)
headers = models.JSONField(
_("webhook headers"),
null=True,
blank=True,
help_text=_("The headers to send with the webhook URL."),
)
include_document = models.BooleanField(
default=False,
verbose_name=_("include document in webhook"),
)
def __str__(self):
return f"Workflow Webhook Action {self.pk}"
class WorkflowAction(models.Model):
class WorkflowActionType(models.IntegerChoices):
ASSIGNMENT = (
@@ -1375,77 +1454,22 @@ class WorkflowAction(models.Model):
verbose_name=_("remove all custom fields"),
)
email_subject = models.CharField(
_("email subject"),
max_length=256,
email = models.ForeignKey(
WorkflowActionEmail,
null=True,
blank=True,
help_text=_(
"The subject of the email, can include some placeholders, "
"see documentation.",
),
on_delete=models.SET_NULL,
related_name="action",
verbose_name=_("email"),
)
email_body = models.TextField(
_("email body"),
webhook = models.ForeignKey(
WorkflowActionWebhook,
null=True,
blank=True,
help_text=_(
"The body (message) of the email, can include some placeholders, "
"see documentation.",
),
)
email_to = models.TextField(
_("emails to"),
null=True,
blank=True,
help_text=_(
"The destination email addresses, comma separated.",
),
)
email_include_document = models.BooleanField(
default=False,
verbose_name=_("include document in email"),
)
webhook_url = models.URLField(
_("webhook url"),
null=True,
blank=True,
help_text=_("The destination URL for the notification."),
)
webhook_use_params = models.BooleanField(
default=True,
verbose_name=_("use parameters"),
)
webhook_params = models.JSONField(
_("webhook parameters"),
null=True,
blank=True,
help_text=_("The parameters to send with the webhook URL if body not used."),
)
webhook_body = models.TextField(
_("webhook body"),
null=True,
blank=True,
help_text=_("The body to send with the webhook URL if parameters not used."),
)
webhook_headers = models.JSONField(
_("webhook headers"),
null=True,
blank=True,
help_text=_("The headers to send with the webhook URL."),
)
webhook_include_document = models.BooleanField(
default=False,
verbose_name=_("include document in webhook"),
on_delete=models.SET_NULL,
related_name="action",
verbose_name=_("webhook"),
)
class Meta:

106
src/documents/serialisers.py
View File

@@ -49,6 +49,8 @@ from documents.models import Tag
from documents.models import UiSettings
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowActionEmail
from documents.models import WorkflowActionWebhook
from documents.models import WorkflowTrigger
from documents.parsers import is_mime_type_supported
from documents.permissions import get_groups_with_only_permission
@@ -1807,12 +1809,44 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer):
return attrs
class WorkflowActionEmailSerializer(serializers.ModelSerializer):
id = serializers.IntegerField(allow_null=True, required=False)
class Meta:
model = WorkflowActionEmail
fields = [
"id",
"subject",
"body",
"to",
"include_document",
]
class WorkflowActionWebhookSerializer(serializers.ModelSerializer):
id = serializers.IntegerField(allow_null=True, required=False)
class Meta:
model = WorkflowActionWebhook
fields = [
"id",
"url",
"use_params",
"params",
"body",
"headers",
"include_document",
]
class WorkflowActionSerializer(serializers.ModelSerializer):
id = serializers.IntegerField(required=False, allow_null=True)
assign_correspondent = CorrespondentField(allow_null=True, required=False)
assign_tags = TagsField(many=True, allow_null=True, required=False)
assign_document_type = DocumentTypeField(allow_null=True, required=False)
assign_storage_path = StoragePathField(allow_null=True, required=False)
email = WorkflowActionEmailSerializer(allow_null=True, required=False)
webhook = WorkflowActionWebhookSerializer(allow_null=True, required=False)
class Meta:
model = WorkflowAction
@@ -1847,15 +1881,8 @@ class WorkflowActionSerializer(serializers.ModelSerializer):
"remove_view_groups",
"remove_change_users",
"remove_change_groups",
"email_to",
"email_subject",
"email_body",
"email_include_document",
"webhook_url",
"webhook_use_params",
"webhook_params",
"webhook_body",
"webhook_headers",
"email",
"webhook",
]
def validate(self, attrs):
@@ -1893,37 +1920,22 @@ class WorkflowActionSerializer(serializers.ModelSerializer):
{"assign_title": f'Invalid f-string detected: "{e.args[0]}"'},
)
if "type" in attrs and attrs["type"] == WorkflowAction.WorkflowActionType.EMAIL:
if (
"email_subject" not in attrs
or attrs["email_subject"] is None
or len(attrs["email_subject"]) == 0
or "email_body" not in attrs
or attrs["email_body"] is None
or len(attrs["email_body"]) == 0
):
raise serializers.ValidationError(
"Email subject and body required",
)
elif (
"email_to" not in attrs
or attrs["email_to"] is None
or len(attrs["email_to"]) == 0
):
raise serializers.ValidationError(
"Email recipient required",
)
if (
"type" in attrs
and attrs["type"] == WorkflowAction.WorkflowActionType.EMAIL
and "email" not in attrs
):
raise serializers.ValidationError(
"Email data is required for email actions",
)
if (
"type" in attrs
and attrs["type"] == WorkflowAction.WorkflowActionType.WEBHOOK
) and (
"webhook_url" not in attrs
or attrs["webhook_url"] is None
or len(attrs["webhook_url"]) == 0
and "webhook" not in attrs
):
raise serializers.ValidationError(
"Webhook URL required",
"Webhook data is required for webhook actions",
)
return attrs
@@ -1980,11 +1992,34 @@ class WorkflowSerializer(serializers.ModelSerializer):
remove_change_users = action.pop("remove_change_users", None)
remove_change_groups = action.pop("remove_change_groups", None)
email_data = action.pop("email", None)
webhook_data = action.pop("webhook", None)
action_instance, _ = WorkflowAction.objects.update_or_create(
id=action.get("id"),
defaults=action,
)
if email_data is not None:
serializer = WorkflowActionEmailSerializer(data=email_data)
serializer.is_valid(raise_exception=True)
email, _ = WorkflowActionEmail.objects.update_or_create(
id=email_data.get("id"),
defaults=serializer.validated_data,
)
action_instance.email = email
action_instance.save()
if webhook_data is not None:
serializer = WorkflowActionWebhookSerializer(data=webhook_data)
serializer.is_valid(raise_exception=True)
webhook, _ = WorkflowActionWebhook.objects.update_or_create(
id=webhook_data.get("id"),
defaults=serializer.validated_data,
)
action_instance.webhook = webhook
action_instance.save()
if assign_tags is not None:
action_instance.assign_tags.set(assign_tags)
if assign_view_users is not None:
@@ -2037,6 +2072,9 @@ class WorkflowSerializer(serializers.ModelSerializer):
if action.workflows.all().count() == 0:
action.delete()
WorkflowActionEmail.objects.filter(action=None).delete()
WorkflowActionWebhook.objects.filter(action=None).delete()
def create(self, validated_data) -> Workflow:
if "triggers" in validated_data:
triggers = validated_data.pop("triggers")

26
src/documents/signals/handlers.py
View File

@@ -891,7 +891,7 @@ def run_workflows(
added = timezone.localtime(document.added)
created = timezone.localtime(document.created)
subject = parse_w_workflow_placeholders(
action.email_subject,
action.email.subject,
correspondent,
document_type,
owner_username,
@@ -902,7 +902,7 @@ def run_workflows(
doc_url,
)
body = parse_w_workflow_placeholders(
action.email_body,
action.email.body,
correspondent,
document_type,
owner_username,
@@ -916,13 +916,13 @@ def run_workflows(
email = EmailMessage(
subject=subject,
body=body,
to=action.email_to.split(","),
to=action.email.to.split(","),
)
if action.email_include_document:
if action.email.include_document:
email.attach_file(document.source_path)
n_messages = email.send()
logger.debug(
f"Sent {n_messages} notification email(s) to {action.email_to}",
f"Sent {n_messages} notification email(s) to {action.email.to}",
extra={"group": logging_group},
)
except Exception as e:
@@ -949,9 +949,9 @@ def run_workflows(
try:
data = {}
if action.webhook_use_params:
if action.webhook.use_params:
try:
for key, value in action.webhook_params.items():
for key, value in action.webhook.params.items():
data[key] = parse_w_workflow_placeholders(
value,
correspondent,
@@ -970,7 +970,7 @@ def run_workflows(
)
else:
data = parse_w_workflow_placeholders(
action.webhook_body,
action.webhook.body,
correspondent,
document_type,
owner_username,
@@ -981,30 +981,30 @@ def run_workflows(
doc_url,
)
headers = {}
if action.webhook_headers:
if action.webhook.headers:
try:
headers = {
str(k): str(v) for k, v in action.webhook_headers.items()
str(k): str(v) for k, v in action.webhook.headers.items()
}
except Exception as e:
logger.error(
f"Error occurred parsing webhook headers: {e}",
extra={"group": logging_group},
)
if action.webhook_include_document:
if action.webhook.include_document:
with open(document.source_path, "rb") as f:
files = {
"file": (document.original_filename, f, document.mime_type),
}
httpx.post(
action.webhook_url,
action.webhook.url,
data=data,
files=files,
headers=headers,
)
else:
httpx.post(
action.webhook_url,
action.webhook.url,
data=data,
headers=headers,
)

20
src/documents/tests/test_api_workflows.py
View File

@@ -484,8 +484,10 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"actions": [
{
"type": WorkflowAction.WorkflowActionType.EMAIL,
"email_subject": "Subject",
"email_body": "Body",
"email": {
"subject": "Subject",
"body": "Body",
},
},
],
},
@@ -511,9 +513,12 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"actions": [
{
"type": WorkflowAction.WorkflowActionType.EMAIL,
"email_subject": "Subject",
"email_body": "Body",
"email_to": "me@example.com",
"email": {
"subject": "Subject",
"body": "Body",
"to": "me@example.com",
"include_document": False,
},
},
],
},
@@ -572,7 +577,10 @@ class TestApiWorkflows(DirectoriesMixin, APITestCase):
"actions": [
{
"type": WorkflowAction.WorkflowActionType.WEBHOOK,
"webhook_url": "https://example.com",
"webhook": {
"url": "https://example.com",
"include_document": False,
},
},
],
},

93
src/documents/tests/test_workflows.py
View File

@@ -31,6 +31,8 @@ from documents.models import StoragePath
from documents.models import Tag
from documents.models import Workflow
from documents.models import WorkflowAction
from documents.models import WorkflowActionEmail
from documents.models import WorkflowActionWebhook
from documents.models import WorkflowRun
from documents.models import WorkflowTrigger
from documents.signals import document_consumption_finished
@@ -2109,12 +2111,15 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
email_action = WorkflowActionEmail.objects.create(
subject="Test Notification: {doc_title}",
body="Test message: {doc_url}",
to="user@example.com",
include_document=False,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.EMAIL,
email_subject="Test Notification: {doc_title}",
email_body="Test message: {doc_url}",
email_to="user@example.com",
email_include_document=False,
email=email_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2161,12 +2166,15 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
email_action = WorkflowActionEmail.objects.create(
subject="Test Notification: {doc_title}",
body="Test message: {doc_url}",
to="me@example.com",
include_document=True,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.EMAIL,
email_subject="Test Notification: {doc_title}",
email_body="Test message: {doc_url}",
email_to="me@example.com",
email_include_document=True,
email=email_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2202,11 +2210,14 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
email_action = WorkflowActionEmail.objects.create(
subject="Test Notification: {doc_title}",
body="Test message: {doc_url}",
to="me@example.com",
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.EMAIL,
email_subject="Test Notification: {doc_title}",
email_body="Test message: {doc_url}",
email_to="me@example.com",
email=email_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2247,11 +2258,14 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
email_action = WorkflowActionEmail.objects.create(
subject="Test Notification: {doc_title}",
body="Test message: {doc_url}",
to="me@example.com",
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.EMAIL,
email_subject="Test Notification: {doc_title}",
email_body="Test message: {doc_url}",
email_to="me@example.com",
email=email_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2296,12 +2310,15 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
webhook_action = WorkflowActionWebhook.objects.create(
use_params=False,
body="Test message: {doc_url}",
url="http://paperless-ngx.com",
include_document=False,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.WEBHOOK,
webhook_use_params=False,
webhook_body="Test message: {doc_url}",
webhook_url="http://paperless-ngx.com",
webhook_include_document=False,
webhook=webhook_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2348,12 +2365,15 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
webhook_action = WorkflowActionWebhook.objects.create(
use_params=False,
body="Test message: {doc_url}",
url="http://paperless-ngx.com",
include_document=True,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.WEBHOOK,
webhook_use_params=False,
webhook_body="Test message: {doc_url}",
webhook_url="http://paperless-ngx.com",
webhook_include_document=True,
webhook=webhook_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2373,6 +2393,7 @@ class TestWorkflows(
correspondent=self.c,
original_filename="simple.pdf",
filename=test_file,
mime_type="application/pdf",
)
run_workflows(WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED, doc)
@@ -2380,7 +2401,7 @@ class TestWorkflows(
mock_post.assert_called_once_with(
"http://paperless-ngx.com",
data=f"Test message: http://localhost:8000/documents/{doc.id}/",
files={"file": ("simple.pdf", mock.ANY)},
files={"file": ("simple.pdf", mock.ANY, "application/pdf")},
headers={},
)
@@ -2402,15 +2423,18 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.WEBHOOK,
webhook_use_params=True,
webhook_params={
webhook_action = WorkflowActionWebhook.objects.create(
use_params=True,
params={
"title": "Test webhook: {doc_title}",
"body": "Test message: {doc_url}",
},
webhook_url="http://paperless-ngx.com",
webhook_include_document=True,
url="http://paperless-ngx.com",
include_document=True,
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.WEBHOOK,
webhook=webhook_action,
)
w = Workflow.objects.create(
name="Workflow 1",
@@ -2447,12 +2471,15 @@ class TestWorkflows(
trigger = WorkflowTrigger.objects.create(
type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED,
)
webhook_action = WorkflowActionWebhook.objects.create(
url="http://paperless-ngx.com",
use_params=True,
params="invalid",
headers="invalid",
)
action = WorkflowAction.objects.create(
type=WorkflowAction.WorkflowActionType.WEBHOOK,
webhook_url="http://paperless-ngx.com",
webhook_use_params=True,
webhook_params="invalid",
webhook_headers="invalid",
webhook=webhook_action,
)
w = Workflow.objects.create(
name="Workflow 1",