diff --git a/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts b/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts
index cd804bc15..7cf47a91b 100644
--- a/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts
+++ b/src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.ts
@@ -69,7 +69,7 @@ export class CustomFieldsDropdownComponent implements OnDestroy {
 
   private updateUnusedFields() {
     this.unusedFields = this.customFields.filter(
-      (f) => !this.existingFields.find((e) => e.field.id === f.id)
+      (f) => !this.existingFields?.find((e) => e.field.id === f.id)
     )
   }
 
diff --git a/src-ui/src/app/components/common/input/abstract-input.ts b/src-ui/src/app/components/common/input/abstract-input.ts
index 9bd35cdad..5e95de0bf 100644
--- a/src-ui/src/app/components/common/input/abstract-input.ts
+++ b/src-ui/src/app/components/common/input/abstract-input.ts
@@ -1,4 +1,12 @@
-import { Directive, ElementRef, Input, OnInit, ViewChild } from '@angular/core'
+import {
+  Directive,
+  ElementRef,
+  EventEmitter,
+  Input,
+  OnInit,
+  Output,
+  ViewChild,
+} from '@angular/core'
 import { ControlValueAccessor } from '@angular/forms'
 import { v4 as uuidv4 } from 'uuid'
 
@@ -41,9 +49,18 @@ export class AbstractInputComponent<T> implements OnInit, ControlValueAccessor {
   @Input()
   error: string
 
+  @Input()
+  hint: string
+
   @Input()
   horizontal: boolean = false
 
+  @Input()
+  removable: boolean = false
+
+  @Output()
+  removed: EventEmitter<AbstractInputComponent<any>> = new EventEmitter()
+
   value: T
 
   ngOnInit(): void {
@@ -51,7 +68,4 @@ export class AbstractInputComponent<T> implements OnInit, ControlValueAccessor {
   }
 
   inputId: string
-
-  @Input()
-  hint: string
 }
diff --git a/src-ui/src/app/components/common/input/date/date.component.html b/src-ui/src/app/components/common/input/date/date.component.html
index 14e802bf6..d894536cf 100644
--- a/src-ui/src/app/components/common/input/date/date.component.html
+++ b/src-ui/src/app/components/common/input/date/date.component.html
@@ -1,7 +1,12 @@
 <div class="mb-3">
   <div class="row">
-    <div class="d-flex align-items-center" [class.col-md-3]="horizontal">
+    <div class="d-flex align-items-center position-relative hidden-button-container" [class.col-md-3]="horizontal">
       <label class="form-label mb-md-0" [for]="inputId">{{title}}</label>
+      <button type="button" *ngIf="removable" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
+        <svg class="sidebaricon" fill="currentColor">
+          <use xlink:href="assets/bootstrap-icons.svg#x"/>
+        </svg>&nbsp;<ng-container i18n>Remove</ng-container>
+      </button>
     </div>
     <div [class.col-md-9]="horizontal">
       <div class="input-group" [class.is-invalid]="error">
diff --git a/src-ui/src/app/components/common/input/number/number.component.html b/src-ui/src/app/components/common/input/number/number.component.html
index 55980f6c7..0628187f1 100644
--- a/src-ui/src/app/components/common/input/number/number.component.html
+++ b/src-ui/src/app/components/common/input/number/number.component.html
@@ -1,7 +1,12 @@
 <div class="mb-3">
   <div class="row">
-    <div class="d-flex align-items-center" [class.col-md-3]="horizontal">
+    <div class="d-flex align-items-center position-relative hidden-button-container" [class.col-md-3]="horizontal">
       <label class="form-label mb-md-0" [for]="inputId">{{title}}</label>
+      <button type="button" *ngIf="removable" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
+        <svg class="sidebaricon" fill="currentColor">
+          <use xlink:href="assets/bootstrap-icons.svg#x"/>
+        </svg>&nbsp;<ng-container i18n>Remove</ng-container>
+      </button>
     </div>
     <div [class.col-md-9]="horizontal">
       <div class="input-group" [class.is-invalid]="error">
diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html
index 6808ec9c3..08cbfc9d7 100644
--- a/src-ui/src/app/components/common/input/select/select.component.html
+++ b/src-ui/src/app/components/common/input/select/select.component.html
@@ -1,7 +1,12 @@
 <div class="mb-3 paperless-input-select" [class.disabled]="disabled">
   <div class="row">
-    <div class="d-flex align-items-center" [class.col-md-3]="horizontal">
+    <div class="d-flex align-items-center position-relative hidden-button-container" [class.col-md-3]="horizontal">
       <label *ngIf="title" class="form-label mb-md-0" [for]="inputId">{{title}}</label>
+      <button type="button" *ngIf="removable" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
+        <svg class="sidebaricon" fill="currentColor">
+          <use xlink:href="assets/bootstrap-icons.svg#x"/>
+        </svg>&nbsp;<ng-container i18n>Remove</ng-container>
+      </button>
     </div>
     <div [class.col-md-9]="horizontal">
       <div [class.input-group]="allowCreateNew || showFilter">
diff --git a/src-ui/src/app/components/common/input/text/text.component.html b/src-ui/src/app/components/common/input/text/text.component.html
index a863e0103..83b2b7b9e 100644
--- a/src-ui/src/app/components/common/input/text/text.component.html
+++ b/src-ui/src/app/components/common/input/text/text.component.html
@@ -1,7 +1,12 @@
 <div class="mb-3">
   <div class="row">
-    <div class="d-flex align-items-center" [class.col-md-3]="horizontal">
+    <div class="d-flex align-items-center position-relative hidden-button-container" [class.col-md-3]="horizontal">
       <label class="form-label mb-md-0" [for]="inputId">{{title}}</label>
+      <button type="button" *ngIf="removable" class="btn btn-sm btn-danger position-absolute left-0" (click)="removed.emit(this)">
+        <svg class="sidebaricon" fill="currentColor">
+          <use xlink:href="assets/bootstrap-icons.svg#x"/>
+        </svg>&nbsp;<ng-container i18n>Remove</ng-container>
+      </button>
     </div>
     <div [class.col-md-9]="horizontal">
       <input #inputField type="text" class="form-control" [class.is-invalid]="error" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" [disabled]="disabled">
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index 2e2c932cf..f81f96f26 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -51,7 +51,7 @@
     <pngx-custom-fields-dropdown
         [documentId]="documentId"
         [disabled]="!userIsOwner"
-        [existingFields]="customFields"
+        [existingFields]="document?.custom_fields"
         (added)="addField($event)">
     </pngx-custom-fields-dropdown>
 
@@ -100,11 +100,11 @@
                             <pngx-input-select [items]="storagePaths" i18n-title title="Storage path" formControlName="storage_path" [allowNull]="true" [showFilter]="true" [horizontal]="true" (filterDocuments)="filterDocuments($event)"
                                 (createNew)="createStoragePath($event)" [suggestions]="suggestions?.storage_paths" i18n-placeholder placeholder="Default" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.StoragePath }"></pngx-input-select>
                             <pngx-input-tags formControlName="tags" [suggestions]="suggestions?.tags" [showFilter]="true" [horizontal]="true" (filterDocuments)="filterDocuments($event)" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Tag }"></pngx-input-tags>
-                            <ng-container *ngFor="let fieldInstance of customFields; let i = index">
+                            <ng-container *ngFor="let fieldInstance of document?.custom_fields; let i = index">
                                 <div [formGroup]="customFieldFormFields.controls[i]">
-                                    <pngx-input-text formControlName="value" *ngIf="fieldInstance.field.data_type === PaperlessCustomFieldDataType.String" [title]="fieldInstance.field.name" [horizontal]="true"></pngx-input-text>
-                                    <pngx-input-date formControlName="value" *ngIf="fieldInstance.field.data_type === PaperlessCustomFieldDataType.Date" [title]="fieldInstance.field.name" [horizontal]="true"></pngx-input-date>
-                                    <pngx-input-number formControlName="value" *ngIf="fieldInstance.field.data_type === PaperlessCustomFieldDataType.Integer" [title]="fieldInstance.field.name" [horizontal]="true" [showAdd]="false"></pngx-input-number>
+                                    <pngx-input-text formControlName="value" *ngIf="fieldInstance.field.data_type === PaperlessCustomFieldDataType.String" [title]="fieldInstance.field.name" [removable]="true" (removed)="removeField($event)" [horizontal]="true"></pngx-input-text>
+                                    <pngx-input-date formControlName="value" *ngIf="fieldInstance.field.data_type === PaperlessCustomFieldDataType.Date" [title]="fieldInstance.field.name" [removable]="true" (removed)="removeField($event)" [horizontal]="true"></pngx-input-date>
+                                    <pngx-input-number formControlName="value" *ngIf="fieldInstance.field.data_type === PaperlessCustomFieldDataType.Integer" [title]="fieldInstance.field.name" [removable]="true" (removed)="removeField($event)" [horizontal]="true" [showAdd]="false"></pngx-input-number>
                                 </div>
                             </ng-container>
                         </div>
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index f06e256d7..b204a82fd 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -68,6 +68,7 @@ import {
   PaperlessCustomFieldDataType,
 } from 'src/app/data/paperless-custom-field'
 import { PaperlessCustomFieldInstance } from 'src/app/data/paperless-custom-field-instance'
+import { AbstractInputComponent } from '../common/input/abstract-input'
 
 enum DocumentDetailNavIDs {
   Details = 1,
@@ -141,7 +142,6 @@ export class DocumentDetailComponent
   ogDate: Date
 
   public readonly PaperlessCustomFieldDataType = PaperlessCustomFieldDataType
-  customFields: PaperlessCustomFieldInstance[] = []
 
   @ViewChild('nav') nav: NgbNav
   @ViewChild('pdfPreview') set pdfPreview(element) {
@@ -394,7 +394,7 @@ export class DocumentDetailComponent
   updateComponent(doc: PaperlessDocument) {
     this.document = doc
     this.requiresPassword = false
-    this.customFields = doc.custom_fields
+    // this.customFields = doc.custom_fields.concat([])
     this.updateFormForCustomFields()
     this.documentsService
       .getMetadata(doc.id)
@@ -448,23 +448,6 @@ export class DocumentDetailComponent
     return this.documentForm.get('custom_fields') as FormArray
   }
 
-  updateFormForCustomFields() {
-    this.customFieldFormFields.clear()
-    this.customFields.forEach((fieldInstance) => {
-      this.customFieldFormFields.push(
-        new FormGroup({
-          field: new FormGroup({
-            id: new FormControl(fieldInstance.field.id),
-            name: new FormControl(fieldInstance.field.name),
-            data_type: new FormControl(fieldInstance.field.data_type),
-          }),
-          value: new FormControl(fieldInstance.value),
-        }),
-        { emitEvent: false }
-      )
-    })
-  }
-
   createDocumentType(newName: string) {
     var modal = this.modalService.open(DocumentTypeEditDialogComponent, {
       backdrop: 'static',
@@ -542,9 +525,8 @@ export class DocumentDetailComponent
             set_permissions: doc.permissions,
           }
           this.title = doc.title
-          this.documentForm.patchValue(doc)
-          this.customFields = doc.custom_fields
           this.updateFormForCustomFields()
+          this.documentForm.patchValue(doc)
           this.openDocumentService.setDirty(doc, false)
         },
         error: () => {
@@ -855,13 +837,41 @@ export class DocumentDetailComponent
     this.documentListViewService.quickFilter(filterRules)
   }
 
+  updateFormForCustomFields(emitEvent: boolean = false) {
+    this.customFieldFormFields.clear({ emitEvent: false })
+    this.document.custom_fields.forEach((fieldInstance) => {
+      this.customFieldFormFields.push(
+        new FormGroup({
+          field: new FormGroup({
+            id: new FormControl(fieldInstance.field.id),
+            name: new FormControl(fieldInstance.field.name),
+            data_type: new FormControl(fieldInstance.field.data_type),
+          }),
+          value: new FormControl(fieldInstance.value),
+        }),
+        { emitEvent }
+      )
+    })
+  }
+
   addField(field: PaperlessCustomField) {
-    this.customFields.push({
+    this.document.custom_fields.push({
       field,
       value: null,
       document: this.documentId,
       created: new Date(),
     })
-    this.updateFormForCustomFields()
+    this.updateFormForCustomFields(true)
+  }
+
+  removeField(input: AbstractInputComponent<any>) {
+    // ok for now as custom field name unique is a constraint
+    const customFieldIndex = this.document.custom_fields.findIndex(
+      (f) => f.field.name === input.title
+    )
+    if (customFieldIndex) {
+      this.document.custom_fields.splice(customFieldIndex, 1)
+      this.updateFormForCustomFields(true)
+    }
   }
 }
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index fde841ffb..dd779bda6 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -661,3 +661,17 @@ code {
 .cdk-drag-animating {
   transition: transform 300ms cubic-bezier(0, 0, 0.2, 1);
 }
+
+.hidden-button-container {
+  button {
+    opacity: 0;
+    pointer-events: none;
+    transition: opacity .2s ease;
+  }
+  &:hover {
+    button {
+      opacity: 1;
+      pointer-events: initial;
+    }
+  }
+}
diff --git a/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py b/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py
index 2719f7878..90856740d 100644
--- a/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py
+++ b/src/documents/migrations/1040_customfield_customfieldboolean_customfielddate_and_more.py
@@ -219,6 +219,13 @@ class Migration(migrations.Migration):
                 ),
             ],
         ),
+        migrations.AddConstraint(
+            model_name="customfield",
+            constraint=models.UniqueConstraint(
+                fields=("name",),
+                name="documents_customfield_unique_name",
+            ),
+        ),
         migrations.AddConstraint(
             model_name="customfieldinstance",
             constraint=models.UniqueConstraint(
diff --git a/src/documents/models.py b/src/documents/models.py
index 8483ed6e3..1cd8c959c 100644
--- a/src/documents/models.py
+++ b/src/documents/models.py
@@ -916,6 +916,12 @@ class CustomField(models.Model):
         ordering = ("created",)
         verbose_name = _("custom field")
         verbose_name_plural = _("custom fields")
+        constraints = [
+            models.UniqueConstraint(
+                fields=["name"],
+                name="%(app_label)s_%(class)s_unique_name",
+            ),
+        ]
 
     def __str__(self) -> str:
         return f"{self.name} : {self.data_type}"
diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index 35e6b7de4..cf20e86bd 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -464,7 +464,7 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
                 field_instance["value"] = data_custom_field["value"]
         return values
 
-    def update(self, instance, validated_data):
+    def update(self, instance: Document, validated_data):
         if "custom_fields" in validated_data:
             custom_fields = validated_data.pop("custom_fields")
             for field_data in custom_fields:
@@ -473,6 +473,19 @@ class DocumentSerializer(OwnedObjectSerializer, DynamicFieldsModelSerializer):
                     field=field_data["field"],
                     value=field_data["value"],
                 )
+            existing_fields = CustomFieldInstance.objects.filter(document=instance)
+            for existing_field in existing_fields:
+                if (
+                    not len(
+                        [
+                            f
+                            for f in custom_fields
+                            if f["field"]["id"] == existing_field.field.id
+                        ],
+                    )
+                    > 0
+                ):
+                    existing_field.delete()
         if "created_date" in validated_data and "created" not in validated_data:
             new_datetime = datetime.datetime.combine(
                 validated_data.get("created_date"),