From 91da6eff800df764ff7f9d41b4cff8835e58e3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Schr=C3=B6ter?= Date: Mon, 13 Jan 2025 15:33:42 +0100 Subject: [PATCH] feat: add correspondent management and update related components --- app_llm.go | 11 +++- paperless.go | 50 +++++++++++++++---- paperless_test.go | 3 +- types.go | 13 ++--- web-app/src/DocumentProcessor.tsx | 17 +++++++ web-app/src/components/DocumentCard.tsx | 6 +++ web-app/src/components/DocumentsToProcess.tsx | 13 +++++ web-app/src/components/SuggestionCard.tsx | 20 ++++++++ web-app/src/components/SuggestionsReview.tsx | 3 ++ web-app/tsconfig.app.tsbuildinfo | 2 +- web-app/tsconfig.node.tsbuildinfo | 2 +- 11 files changed, 120 insertions(+), 20 deletions(-) diff --git a/app_llm.go b/app_llm.go index 7547c06..9c1c284 100644 --- a/app_llm.go +++ b/app_llm.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "fmt" "image" + "slices" "strings" "sync" @@ -61,6 +62,7 @@ func (app *App) getSuggestedTags( content string, suggestedTitle string, availableTags []string, + originalTags []string, logger *logrus.Entry) ([]string, error) { likelyLanguage := getLikelyLanguage() @@ -71,6 +73,7 @@ func (app *App) getSuggestedTags( err := tagTemplate.Execute(&promptBuffer, map[string]interface{}{ "Language": likelyLanguage, "AvailableTags": availableTags, + "OriginalTags": originalTags, "Title": suggestedTitle, "Content": content, }) @@ -103,6 +106,12 @@ func (app *App) getSuggestedTags( suggestedTags[i] = strings.TrimSpace(tag) } + // append the original tags to the suggested tags + suggestedTags = append(suggestedTags, originalTags...) + // Remove duplicates + slices.Sort(suggestedTags) + suggestedTags = slices.Compact(suggestedTags) + // Filter out tags that are not in the available tags list filteredTags := []string{} for _, tag := range suggestedTags { @@ -278,7 +287,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque } if suggestionRequest.GenerateTags { - suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, docLogger) + suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, doc.Tags, docLogger) if err != nil { mu.Lock() errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) diff --git a/paperless.go b/paperless.go index 1aa5158..1ef2ca9 100644 --- a/paperless.go +++ b/paperless.go @@ -163,6 +163,11 @@ func (client *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []st return nil, err } + allCorrespondents, err := client.GetAllCorrespondents(ctx) + if err != nil { + return nil, err + } + documents := make([]Document, 0, len(documentsResponse.Results)) for _, result := range documentsResponse.Results { tagNames := make([]string, len(result.Tags)) @@ -175,11 +180,22 @@ func (client *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []st } } + correspondentName := "" + if result.Correspondent != 0 { + for name, id := range allCorrespondents { + if result.Correspondent == id { + correspondentName = name + break + } + } + } + documents = append(documents, Document{ - ID: result.ID, - Title: result.Title, - Content: result.Content, - Tags: tagNames, + ID: result.ID, + Title: result.Title, + Content: result.Content, + Correspondent: correspondentName, + Tags: tagNames, }) } @@ -227,6 +243,12 @@ func (client *PaperlessClient) GetDocument(ctx context.Context, documentID int) return Document{}, err } + allCorrespondents, err := client.GetAllCorrespondents(ctx) + if err != nil { + return Document{}, err + } + + // Match tag IDs to tag names tagNames := make([]string, len(documentResponse.Tags)) for i, resultTagID := range documentResponse.Tags { for tagName, tagID := range allTags { @@ -237,11 +259,21 @@ func (client *PaperlessClient) GetDocument(ctx context.Context, documentID int) } } + // Match correspondent ID to correspondent name + correspondentName := "" + for name, id := range allCorrespondents { + if documentResponse.Correspondent == id { + correspondentName = name + break + } + } + return Document{ - ID: documentResponse.ID, - Title: documentResponse.Title, - Content: documentResponse.Content, - Tags: tagNames, + ID: documentResponse.ID, + Title: documentResponse.Title, + Content: documentResponse.Content, + Correspondent: correspondentName, + Tags: tagNames, }, nil } @@ -302,8 +334,6 @@ func (client *PaperlessClient) UpdateDocuments(ctx context.Context, documents [] // remove autoTag to prevent infinite loop - this is required in case of undo tags = removeTagFromList(tags, autoTag) - // keep previous tags - tags = append(tags, originalTags...) // remove duplicates slices.Sort(tags) tags = slices.Compact(tags) diff --git a/paperless_test.go b/paperless_test.go index c9be3a0..e6a215a 100644 --- a/paperless_test.go +++ b/paperless_test.go @@ -348,7 +348,8 @@ func TestUpdateDocuments(t *testing.T) { // Expected updated fields expectedFields := map[string]interface{}{ "title": "New Title", - "tags": []interface{}{float64(idTag1), float64(idTag2), float64(idTag3)}, // keep also previous tags + // do not keep previous tags since the tag generation will already take care to include old ones: + "tags": []interface{}{float64(idTag2), float64(idTag3)}, } assert.Equal(t, expectedFields, updatedFields) diff --git a/types.go b/types.go index 6bb16a7..3f33433 100644 --- a/types.go +++ b/types.go @@ -11,7 +11,7 @@ type GetDocumentsApiResponse struct { All []int `json:"all"` Results []struct { ID int `json:"id"` - Correspondent interface{} `json:"correspondent"` + Correspondent int `json:"correspondent"` DocumentType interface{} `json:"document_type"` StoragePath interface{} `json:"storage_path"` Title string `json:"title"` @@ -38,7 +38,7 @@ type GetDocumentsApiResponse struct { type GetDocumentApiResponse struct { ID int `json:"id"` - Correspondent interface{} `json:"correspondent"` + Correspondent int `json:"correspondent"` DocumentType interface{} `json:"document_type"` StoragePath interface{} `json:"storage_path"` Title string `json:"title"` @@ -59,10 +59,11 @@ type GetDocumentApiResponse struct { // Document is a stripped down version of the document object from paperless-ngx. // Response payload for /documents endpoint and part of request payload for /generate-suggestions endpoint type Document struct { - ID int `json:"id"` - Title string `json:"title"` - Content string `json:"content"` - Tags []string `json:"tags"` + ID int `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + Tags []string `json:"tags"` + Correspondent string `json:"correspondent"` } // GenerateSuggestionsRequest is the request payload for generating suggestions for /generate-suggestions endpoint diff --git a/web-app/src/DocumentProcessor.tsx b/web-app/src/DocumentProcessor.tsx index 093a5e1..f8efd13 100644 --- a/web-app/src/DocumentProcessor.tsx +++ b/web-app/src/DocumentProcessor.tsx @@ -11,12 +11,14 @@ export interface Document { title: string; content: string; tags: string[]; + correspondent: string; } export interface GenerateSuggestionsRequest { documents: Document[]; generate_titles?: boolean; generate_tags?: boolean; + generate_correspondents?: boolean; } export interface DocumentSuggestion { @@ -25,6 +27,7 @@ export interface DocumentSuggestion { suggested_title?: string; suggested_tags?: string[]; suggested_content?: string; + suggested_correspondent?: string; } export interface TagOption { @@ -43,6 +46,7 @@ const DocumentProcessor: React.FC = () => { const [filterTag, setFilterTag] = useState(null); const [generateTitles, setGenerateTitles] = useState(true); const [generateTags, setGenerateTags] = useState(true); + const [generateCorrespondents, setGenerateCorrespondents] = useState(true); const [error, setError] = useState(null); // Custom hook to fetch initial data @@ -81,6 +85,7 @@ const DocumentProcessor: React.FC = () => { documents, generate_titles: generateTitles, generate_tags: generateTags, + generate_correspondents: generateCorrespondents, }; const { data } = await axios.post( @@ -137,6 +142,7 @@ const DocumentProcessor: React.FC = () => { ); }; + const handleTitleChange = (docId: number, title: string) => { setSuggestions((prevSuggestions) => prevSuggestions.map((doc) => @@ -145,6 +151,14 @@ const DocumentProcessor: React.FC = () => { ); }; + const handleCorrespondentChange = (docId: number, correspondent: string) => { + setSuggestions((prevSuggestions) => + prevSuggestions.map((doc) => + doc.id === docId ? { ...doc, suggested_correspondent: correspondent } : doc + ) + ); + } + const resetSuggestions = () => { setSuggestions([]); }; @@ -214,6 +228,8 @@ const DocumentProcessor: React.FC = () => { setGenerateTitles={setGenerateTitles} generateTags={generateTags} setGenerateTags={setGenerateTags} + generateCorrespondents={generateCorrespondents} + setGenerateCorrespondents={setGenerateCorrespondents} onProcess={handleProcessDocuments} processing={processing} onReload={reloadDocuments} @@ -225,6 +241,7 @@ const DocumentProcessor: React.FC = () => { onTitleChange={handleTitleChange} onTagAddition={handleTagAddition} onTagDeletion={handleTagDeletion} + onCorrespondentChange={handleCorrespondentChange} onBack={resetSuggestions} onUpdate={handleUpdateDocuments} updating={updating} diff --git a/web-app/src/components/DocumentCard.tsx b/web-app/src/components/DocumentCard.tsx index bc6ec2a..1412fbe 100644 --- a/web-app/src/components/DocumentCard.tsx +++ b/web-app/src/components/DocumentCard.tsx @@ -13,6 +13,9 @@ const DocumentCard: React.FC = ({ document }) => ( ? `${document.content.substring(0, 100)}...` : document.content}

+

+ Correspondent: {document.correspondent} +

{document.tags.map((tag) => ( = ({ document }) => (

{document.title}

{document.content}

+

+ Correspondent: {document.correspondent} +

{document.tags.map((tag) => ( >; generateTags: boolean; setGenerateTags: React.Dispatch>; + generateCorrespondents: boolean; + setGenerateCorrespondents: React.Dispatch>; onProcess: () => void; processing: boolean; onReload: () => void; @@ -20,6 +22,8 @@ const DocumentsToProcess: React.FC = ({ setGenerateTitles, generateTags, setGenerateTags, + generateCorrespondents, + setGenerateCorrespondents, onProcess, processing, onReload, @@ -64,6 +68,15 @@ const DocumentsToProcess: React.FC = ({ /> Generate Tags +
diff --git a/web-app/src/components/SuggestionCard.tsx b/web-app/src/components/SuggestionCard.tsx index 361ce25..6b7be1f 100644 --- a/web-app/src/components/SuggestionCard.tsx +++ b/web-app/src/components/SuggestionCard.tsx @@ -8,6 +8,7 @@ interface SuggestionCardProps { onTitleChange: (docId: number, title: string) => void; onTagAddition: (docId: number, tag: TagOption) => void; onTagDeletion: (docId: number, index: number) => void; + onCorrespondentChange: (docId: number, correspondent: string) => void; } const SuggestionCard: React.FC = ({ @@ -16,6 +17,7 @@ const SuggestionCard: React.FC = ({ onTitleChange, onTagAddition, onTagDeletion, + onCorrespondentChange, }) => { const sortedAvailableTags = availableTags.sort((a, b) => a.name.localeCompare(b.name)); const document = suggestion.original_document; @@ -49,6 +51,9 @@ const SuggestionCard: React.FC = ({
+ = ({ className="w-full border border-gray-300 dark:border-gray-600 rounded px-2 py-1 mt-2 focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-gray-200" />
+ ({ @@ -99,6 +107,18 @@ const SuggestionCard: React.FC = ({ }} />
+
+ + onCorrespondentChange(suggestion.id, e.target.value)} + className="w-full border border-gray-300 dark:border-gray-600 rounded px-2 py-1 mt-2 focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-gray-200" + placeholder="Correspondent" + /> +
); diff --git a/web-app/src/components/SuggestionsReview.tsx b/web-app/src/components/SuggestionsReview.tsx index 3a1f648..a0cc5af 100644 --- a/web-app/src/components/SuggestionsReview.tsx +++ b/web-app/src/components/SuggestionsReview.tsx @@ -8,6 +8,7 @@ interface SuggestionsReviewProps { onTitleChange: (docId: number, title: string) => void; onTagAddition: (docId: number, tag: TagOption) => void; onTagDeletion: (docId: number, index: number) => void; + onCorrespondentChange: (docId: number, correspondent: string) => void; onBack: () => void; onUpdate: () => void; updating: boolean; @@ -19,6 +20,7 @@ const SuggestionsReview: React.FC = ({ onTitleChange, onTagAddition, onTagDeletion, + onCorrespondentChange, onBack, onUpdate, updating, @@ -36,6 +38,7 @@ const SuggestionsReview: React.FC = ({ onTitleChange={onTitleChange} onTagAddition={onTagAddition} onTagDeletion={onTagDeletion} + onCorrespondentChange={onCorrespondentChange} /> ))} diff --git a/web-app/tsconfig.app.tsbuildinfo b/web-app/tsconfig.app.tsbuildinfo index 40eb039..577769a 100644 --- a/web-app/tsconfig.app.tsbuildinfo +++ b/web-app/tsconfig.app.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/app.tsx","./src/documentprocessor.tsx","./src/experimentalocr.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/documentcard.tsx","./src/components/documentstoprocess.tsx","./src/components/nodocuments.tsx","./src/components/successmodal.tsx","./src/components/suggestioncard.tsx","./src/components/suggestionsreview.tsx"],"version":"5.6.2"} \ No newline at end of file +{"root":["./src/app.tsx","./src/documentprocessor.tsx","./src/experimentalocr.tsx","./src/history.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/documentcard.tsx","./src/components/documentstoprocess.tsx","./src/components/nodocuments.tsx","./src/components/sidebar.tsx","./src/components/successmodal.tsx","./src/components/suggestioncard.tsx","./src/components/suggestionsreview.tsx","./src/components/undocard.tsx"],"version":"5.7.2"} \ No newline at end of file diff --git a/web-app/tsconfig.node.tsbuildinfo b/web-app/tsconfig.node.tsbuildinfo index 98ef2f9..1e7ed27 100644 --- a/web-app/tsconfig.node.tsbuildinfo +++ b/web-app/tsconfig.node.tsbuildinfo @@ -1 +1 @@ -{"root":["./vite.config.ts"],"version":"5.6.2"} \ No newline at end of file +{"root":["./vite.config.ts"],"version":"5.7.2"} \ No newline at end of file