diff --git a/README.md b/README.md index fd0b0a7..97464ea 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ https://github.com/user-attachments/assets/bd5d38b9-9309-40b9-93ca-918dfa4f3fd4 Here’s an example `docker-compose.yml` to spin up **paperless-gpt** alongside paperless-ngx: ```yaml -version: '3.7' +version: "3.7" services: paperless-ngx: image: ghcr.io/paperless-ngx/paperless-ngx:latest @@ -102,7 +102,7 @@ services: volumes: - ./prompts:/app/prompts # Mount the prompts directory ports: - - '8080:8080' + - "8080:8080" depends_on: - paperless-ngx ``` @@ -146,6 +146,8 @@ services: ### Environment Variables +**Note:** When using Ollama, ensure that the Ollama server is running and accessible from the paperless-gpt container. +======= | Variable | Description | Required | |------------------------|------------------------------------------------------------------------------------------------------------------|----------| | `PAPERLESS_BASE_URL` | URL of your paperless-ngx instance (e.g. `http://paperless-ngx:8000`). | Yes | @@ -167,7 +169,9 @@ services: | `WEBUI_PATH` | Path for static content. Default: `./web-app/dist`. | No | | `AUTO_GENERATE_TITLE` | Generate titles automatically if `paperless-gpt-auto` is used. Default: `true`. | No | | `AUTO_GENERATE_TAGS` | Generate tags automatically if `paperless-gpt-auto` is used. Default: `true`. | No | +| `AUTO_GENERATE_CORRESPONDENTS` | Generate correspondents automatically if `paperless-gpt-auto` is used. Default: `true`. | No | | `OCR_LIMIT_PAGES` | Limit the number of pages for OCR. Set to `0` for no limit. Default: `5`. | No | +| `CORRESPONDENT_BLACK_LIST` | A comma-separated list of names to exclude from the correspondents suggestions. Example: `John Doe, Jane Smith`. ### Custom Prompt Templates diff --git a/app_http_handlers.go b/app_http_handlers.go index 909edbd..58809ec 100644 --- a/app_http_handlers.go +++ b/app_http_handlers.go @@ -98,7 +98,7 @@ func (app *App) getAllTagsHandler(c *gin.Context) { func (app *App) documentsHandler(c *gin.Context) { ctx := c.Request.Context() - documents, err := app.Client.GetDocumentsByTags(ctx, []string{manualTag}) + documents, err := app.Client.GetDocumentsByTags(ctx, []string{manualTag}, 25) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error fetching documents: %v", err)}) log.Errorf("Error fetching documents: %v", err) diff --git a/app_llm.go b/app_llm.go index f070c13..9c1c284 100644 --- a/app_llm.go +++ b/app_llm.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "fmt" "image" + "slices" "strings" "sync" @@ -15,12 +16,53 @@ import ( "github.com/tmc/langchaingo/llms" ) +// getSuggestedCorrespondent generates a suggested correspondent for a document using the LLM +func (app *App) getSuggestedCorrespondent(ctx context.Context, content string, suggestedTitle string, availableCorrespondents []string, correspondentBlackList []string) (string, error) { + likelyLanguage := getLikelyLanguage() + + templateMutex.RLock() + defer templateMutex.RUnlock() + + var promptBuffer bytes.Buffer + err := correspondentTemplate.Execute(&promptBuffer, map[string]interface{}{ + "Language": likelyLanguage, + "AvailableCorrespondents": availableCorrespondents, + "BlackList": correspondentBlackList, + "Title": suggestedTitle, + "Content": content, + }) + if err != nil { + return "", fmt.Errorf("error executing correspondent template: %v", err) + } + + prompt := promptBuffer.String() + log.Debugf("Correspondent suggestion prompt: %s", prompt) + + completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{ + { + Parts: []llms.ContentPart{ + llms.TextContent{ + Text: prompt, + }, + }, + Role: llms.ChatMessageTypeHuman, + }, + }) + if err != nil { + return "", fmt.Errorf("error getting response from LLM: %v", err) + } + + response := strings.TrimSpace(completion.Choices[0].Content) + return response, nil +} + // getSuggestedTags generates suggested tags for a document using the LLM func (app *App) getSuggestedTags( ctx context.Context, content string, suggestedTitle string, availableTags []string, + originalTags []string, logger *logrus.Entry) ([]string, error) { likelyLanguage := getLikelyLanguage() @@ -31,6 +73,7 @@ func (app *App) getSuggestedTags( err := tagTemplate.Execute(&promptBuffer, map[string]interface{}{ "Language": likelyLanguage, "AvailableTags": availableTags, + "OriginalTags": originalTags, "Title": suggestedTitle, "Content": content, }) @@ -63,6 +106,12 @@ func (app *App) getSuggestedTags( suggestedTags[i] = strings.TrimSpace(tag) } + // append the original tags to the suggested tags + suggestedTags = append(suggestedTags, originalTags...) + // Remove duplicates + slices.Sort(suggestedTags) + suggestedTags = slices.Compact(suggestedTags) + // Filter out tags that are not in the available tags list filteredTags := []string{} for _, tag := range suggestedTags { @@ -190,6 +239,18 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque availableTagNames = append(availableTagNames, tagName) } + // Prepare a list of document correspodents + availableCorrespondentsMap, err := app.Client.GetAllCorrespondents(ctx) + if err != nil { + return nil, fmt.Errorf("failed to fetch available correspondents: %v", err) + } + + // Prepare a list of correspondent names + availableCorrespondentNames := make([]string, 0, len(availableCorrespondentsMap)) + for correspondentName := range availableCorrespondentsMap { + availableCorrespondentNames = append(availableCorrespondentNames, correspondentName) + } + documents := suggestionRequest.Documents documentSuggestions := []DocumentSuggestion{} @@ -212,6 +273,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque var suggestedTitle string var suggestedTags []string + var suggestedCorrespondent string if suggestionRequest.GenerateTitles { suggestedTitle, err = app.getSuggestedTitle(ctx, content, docLogger) @@ -225,7 +287,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque } if suggestionRequest.GenerateTags { - suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, docLogger) + suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, doc.Tags, docLogger) if err != nil { mu.Lock() errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) @@ -235,6 +297,18 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque } } + if suggestionRequest.GenerateCorrespondents { + suggestedCorrespondent, err = app.getSuggestedCorrespondent(ctx, content, suggestedTitle, availableCorrespondentNames, correspondentBlackList) + if err != nil { + mu.Lock() + errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) + mu.Unlock() + log.Errorf("Error generating correspondents for document %d: %v", documentID, err) + return + } + + } + mu.Lock() suggestion := DocumentSuggestion{ ID: documentID, @@ -256,6 +330,13 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque suggestion.SuggestedTags = doc.Tags } + // Correspondents + if suggestionRequest.GenerateCorrespondents { + log.Printf("Suggested correspondent for document %d: %s", documentID, suggestedCorrespondent) + suggestion.SuggestedCorrespondent = suggestedCorrespondent + } else { + suggestion.SuggestedCorrespondent = "" + } // Remove manual tag from the list of suggested tags suggestion.RemoveTags = []string{manualTag, autoTag} diff --git a/main.go b/main.go index 496ca0a..ebfacbc 100644 --- a/main.go +++ b/main.go @@ -30,29 +30,33 @@ var ( log = logrus.New() // Environment Variables - paperlessBaseURL = os.Getenv("PAPERLESS_BASE_URL") - paperlessAPIToken = os.Getenv("PAPERLESS_API_TOKEN") - openaiAPIKey = os.Getenv("OPENAI_API_KEY") - manualTag = os.Getenv("MANUAL_TAG") - autoTag = os.Getenv("AUTO_TAG") - manualOcrTag = os.Getenv("MANUAL_OCR_TAG") // Not used yet - autoOcrTag = os.Getenv("AUTO_OCR_TAG") - llmProvider = os.Getenv("LLM_PROVIDER") - llmModel = os.Getenv("LLM_MODEL") - visionLlmProvider = os.Getenv("VISION_LLM_PROVIDER") - visionLlmModel = os.Getenv("VISION_LLM_MODEL") - logLevel = strings.ToLower(os.Getenv("LOG_LEVEL")) - listenInterface = os.Getenv("LISTEN_INTERFACE") - webuiPath = os.Getenv("WEBUI_PATH") - autoGenerateTitle = os.Getenv("AUTO_GENERATE_TITLE") - autoGenerateTags = os.Getenv("AUTO_GENERATE_TAGS") - limitOcrPages int // Will be read from OCR_LIMIT_PAGES + correspondentBlackList = strings.Split(os.Getenv("CORRESPONDENT_BLACK_LIST"), ",") + + paperlessBaseURL = os.Getenv("PAPERLESS_BASE_URL") + paperlessAPIToken = os.Getenv("PAPERLESS_API_TOKEN") + openaiAPIKey = os.Getenv("OPENAI_API_KEY") + manualTag = os.Getenv("MANUAL_TAG") + autoTag = os.Getenv("AUTO_TAG") + manualOcrTag = os.Getenv("MANUAL_OCR_TAG") // Not used yet + autoOcrTag = os.Getenv("AUTO_OCR_TAG") + llmProvider = os.Getenv("LLM_PROVIDER") + llmModel = os.Getenv("LLM_MODEL") + visionLlmProvider = os.Getenv("VISION_LLM_PROVIDER") + visionLlmModel = os.Getenv("VISION_LLM_MODEL") + logLevel = strings.ToLower(os.Getenv("LOG_LEVEL")) + listenInterface = os.Getenv("LISTEN_INTERFACE") + webuiPath = os.Getenv("WEBUI_PATH") + autoGenerateTitle = os.Getenv("AUTO_GENERATE_TITLE") + autoGenerateTags = os.Getenv("AUTO_GENERATE_TAGS") + autoGenerateCorrespondents = os.Getenv("AUTO_GENERATE_CORRESPONDENTS") + limitOcrPages int // Will be read from OCR_LIMIT_PAGES // Templates - titleTemplate *template.Template - tagTemplate *template.Template - ocrTemplate *template.Template - templateMutex sync.RWMutex + titleTemplate *template.Template + tagTemplate *template.Template + correspondentTemplate *template.Template + ocrTemplate *template.Template + templateMutex sync.RWMutex // Default templates defaultTitleTemplate = `I will provide you with the content of a document that has been partially read by OCR (so it may contain errors). @@ -77,7 +81,33 @@ Content: Please concisely select the {{.Language}} tags from the list above that best describe the document. Be very selective and only choose the most relevant tags since too many tags will make the document less discoverable. ` + defaultCorrespondentTemplate = `I will provide you with the content of a document. Your task is to suggest a correspondent that is most relevant to the document. +Correspondents are the senders of documents that reach you. In the other direction, correspondents are the recipients of documents that you send. +In Paperless-ngx we can imagine correspondents as virtual drawers in which all documents of a person or company are stored. With just one click, we can find all the documents assigned to a specific correspondent. +Try to suggest a correspondent, either from the example list or come up with a new correspondent. + +Respond only with a correspondent, without any additional information! + +Be sure to choose a correspondent that is most relevant to the document. +Try to avoid any legal or financial suffixes like "GmbH" or "AG" in the correspondent name. For example use "Microsoft" instead of "Microsoft Ireland Operations Limited" or "Amazon" instead of "Amazon EU S.a.r.l.". + +If you can't find a suitable correspondent, you can respond with "Unknown". + +Example Correspondents: +{{.AvailableCorrespondents | join ", "}} + +List of Correspondents with Blacklisted Names. Please avoid these correspondents or variations of their names: +{{.BlackList | join ", "}} + +Title of the document: +{{.Title}} + +The content is likely in {{.Language}}. + +Document Content: +{{.Content}} +` defaultOcrPrompt = `Just transcribe the text in this image and preserve the formatting and layout (high quality OCR). Do that for ALL the text in the image. Be thorough and pay attention. This is very important. The image is from a text document so be sure to continue until the bottom of the page. Thanks a lot! You tend to forget about some text in the image so please focus! Use markdown format but without a code block.` ) @@ -363,7 +393,7 @@ func documentLogger(documentID int) *logrus.Entry { func (app *App) processAutoTagDocuments() (int, error) { ctx := context.Background() - documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoTag}) + documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoTag}, 25) if err != nil { return 0, fmt.Errorf("error fetching documents with autoTag: %w", err) } @@ -380,9 +410,10 @@ func (app *App) processAutoTagDocuments() (int, error) { docLogger.Info("Processing document for auto-tagging") suggestionRequest := GenerateSuggestionsRequest{ - Documents: []Document{document}, - GenerateTitles: strings.ToLower(autoGenerateTitle) != "false", - GenerateTags: strings.ToLower(autoGenerateTags) != "false", + Documents: []Document{document}, + GenerateTitles: strings.ToLower(autoGenerateTitle) != "false", + GenerateTags: strings.ToLower(autoGenerateTags) != "false", + GenerateCorrespondents: strings.ToLower(autoGenerateCorrespondents) != "false", } suggestions, err := app.generateDocumentSuggestions(ctx, suggestionRequest, docLogger) @@ -404,7 +435,7 @@ func (app *App) processAutoTagDocuments() (int, error) { func (app *App) processAutoOcrTagDocuments() (int, error) { ctx := context.Background() - documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoOcrTag}) + documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoOcrTag}, 25) if err != nil { return 0, fmt.Errorf("error fetching documents with autoOcrTag: %w", err) } @@ -504,6 +535,21 @@ func loadTemplates() { log.Fatalf("Failed to parse tag template: %v", err) } + // Load correspondent template + correspondentTemplatePath := filepath.Join(promptsDir, "correspondent_prompt.tmpl") + correspondentTemplateContent, err := os.ReadFile(correspondentTemplatePath) + if err != nil { + log.Errorf("Could not read %s, using default template: %v", correspondentTemplatePath, err) + correspondentTemplateContent = []byte(defaultCorrespondentTemplate) + if err := os.WriteFile(correspondentTemplatePath, correspondentTemplateContent, os.ModePerm); err != nil { + log.Fatalf("Failed to write default correspondent template to disk: %v", err) + } + } + correspondentTemplate, err = template.New("correspondent").Funcs(sprig.FuncMap()).Parse(string(correspondentTemplateContent)) + if err != nil { + log.Fatalf("Failed to parse correspondent template: %v", err) + } + // Load OCR template ocrTemplatePath := filepath.Join(promptsDir, "ocr_prompt.tmpl") ocrTemplateContent, err := os.ReadFile(ocrTemplatePath) diff --git a/paperless.go b/paperless.go index 8c33f21..1ef2ca9 100644 --- a/paperless.go +++ b/paperless.go @@ -67,29 +67,29 @@ func NewPaperlessClient(baseURL, apiToken string) *PaperlessClient { } // Do method to make requests to the Paperless-NGX API -func (c *PaperlessClient) Do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { - url := fmt.Sprintf("%s/%s", c.BaseURL, strings.TrimLeft(path, "/")) +func (client *PaperlessClient) Do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + url := fmt.Sprintf("%s/%s", client.BaseURL, strings.TrimLeft(path, "/")) req, err := http.NewRequestWithContext(ctx, method, url, body) if err != nil { return nil, err } - req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.APIToken)) + req.Header.Set("Authorization", fmt.Sprintf("Token %s", client.APIToken)) // Set Content-Type if body is present if body != nil { req.Header.Set("Content-Type", "application/json") } - return c.HTTPClient.Do(req) + return client.HTTPClient.Do(req) } // GetAllTags retrieves all tags from the Paperless-NGX API -func (c *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error) { +func (client *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error) { tagIDMapping := make(map[string]int) path := "api/tags/" for path != "" { - resp, err := c.Do(ctx, "GET", path, nil) + resp, err := client.Do(ctx, "GET", path, nil) if err != nil { return nil, err } @@ -120,8 +120,8 @@ func (c *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error // Extract relative path from the Next URL if tagsResponse.Next != "" { nextURL := tagsResponse.Next - if strings.HasPrefix(nextURL, c.BaseURL) { - nextURL = strings.TrimPrefix(nextURL, c.BaseURL+"/") + if strings.HasPrefix(nextURL, client.BaseURL) { + nextURL = strings.TrimPrefix(nextURL, client.BaseURL+"/") } path = nextURL } else { @@ -133,15 +133,15 @@ func (c *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error } // GetDocumentsByTags retrieves documents that match the specified tags -func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string) ([]Document, error) { +func (client *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string, pageSize int) ([]Document, error) { tagQueries := make([]string, len(tags)) for i, tag := range tags { tagQueries[i] = fmt.Sprintf("tags__name__iexact=%s", tag) } searchQuery := strings.Join(tagQueries, "&") - path := fmt.Sprintf("api/documents/?%s", urlEncode(searchQuery)) + path := fmt.Sprintf("api/documents/?%s&page_size=%d", urlEncode(searchQuery), pageSize) - resp, err := c.Do(ctx, "GET", path, nil) + resp, err := client.Do(ctx, "GET", path, nil) if err != nil { return nil, err } @@ -158,7 +158,12 @@ func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string) return nil, err } - allTags, err := c.GetAllTags(ctx) + allTags, err := client.GetAllTags(ctx) + if err != nil { + return nil, err + } + + allCorrespondents, err := client.GetAllCorrespondents(ctx) if err != nil { return nil, err } @@ -175,11 +180,22 @@ func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string) } } + correspondentName := "" + if result.Correspondent != 0 { + for name, id := range allCorrespondents { + if result.Correspondent == id { + correspondentName = name + break + } + } + } + documents = append(documents, Document{ - ID: result.ID, - Title: result.Title, - Content: result.Content, - Tags: tagNames, + ID: result.ID, + Title: result.Title, + Content: result.Content, + Correspondent: correspondentName, + Tags: tagNames, }) } @@ -187,9 +203,9 @@ func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string) } // DownloadPDF downloads the PDF file of the specified document -func (c *PaperlessClient) DownloadPDF(ctx context.Context, document Document) ([]byte, error) { +func (client *PaperlessClient) DownloadPDF(ctx context.Context, document Document) ([]byte, error) { path := fmt.Sprintf("api/documents/%d/download/", document.ID) - resp, err := c.Do(ctx, "GET", path, nil) + resp, err := client.Do(ctx, "GET", path, nil) if err != nil { return nil, err } @@ -203,9 +219,9 @@ func (c *PaperlessClient) DownloadPDF(ctx context.Context, document Document) ([ return io.ReadAll(resp.Body) } -func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Document, error) { +func (client *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Document, error) { path := fmt.Sprintf("api/documents/%d/", documentID) - resp, err := c.Do(ctx, "GET", path, nil) + resp, err := client.Do(ctx, "GET", path, nil) if err != nil { return Document{}, err } @@ -222,11 +238,17 @@ func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Docu return Document{}, err } - allTags, err := c.GetAllTags(ctx) + allTags, err := client.GetAllTags(ctx) if err != nil { return Document{}, err } + allCorrespondents, err := client.GetAllCorrespondents(ctx) + if err != nil { + return Document{}, err + } + + // Match tag IDs to tag names tagNames := make([]string, len(documentResponse.Tags)) for i, resultTagID := range documentResponse.Tags { for tagName, tagID := range allTags { @@ -237,23 +259,51 @@ func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Docu } } + // Match correspondent ID to correspondent name + correspondentName := "" + for name, id := range allCorrespondents { + if documentResponse.Correspondent == id { + correspondentName = name + break + } + } + return Document{ - ID: documentResponse.ID, - Title: documentResponse.Title, - Content: documentResponse.Content, - Tags: tagNames, + ID: documentResponse.ID, + Title: documentResponse.Title, + Content: documentResponse.Content, + Correspondent: correspondentName, + Tags: tagNames, }, nil } // UpdateDocuments updates the specified documents with suggested changes -func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []DocumentSuggestion, db *gorm.DB, isUndo bool) error { +func (client *PaperlessClient) UpdateDocuments(ctx context.Context, documents []DocumentSuggestion, db *gorm.DB, isUndo bool) error { // Fetch all available tags - availableTags, err := c.GetAllTags(ctx) + availableTags, err := client.GetAllTags(ctx) if err != nil { log.Errorf("Error fetching available tags: %v", err) return err } + documentsContainSuggestedCorrespondent := false + for _, document := range documents { + if document.SuggestedCorrespondent != "" { + documentsContainSuggestedCorrespondent = true + break + } + } + + availableCorrespondents := make(map[string]int) + if documentsContainSuggestedCorrespondent { + availableCorrespondents, err = client.GetAllCorrespondents(ctx) + if err != nil { + log.Errorf("Error fetching available correspondents: %v", + err) + return err + } + } + for _, document := range documents { documentID := document.ID @@ -284,8 +334,6 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum // remove autoTag to prevent infinite loop - this is required in case of undo tags = removeTagFromList(tags, autoTag) - // keep previous tags - tags = append(tags, originalTags...) // remove duplicates slices.Sort(tags) tags = slices.Compact(tags) @@ -306,12 +354,27 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum } newTags = append(newTags, tagID) } else { - log.Warnf("Tag '%s' does not exist in paperless-ngx, skipping.", tagName) + log.Errorf("Suggested tag '%s' does not exist in paperless-ngx, skipping.", tagName) } } - updatedFields["tags"] = newTags + // Map suggested correspondent names to IDs + if document.SuggestedCorrespondent != "" { + if correspondentID, exists := availableCorrespondents[document.SuggestedCorrespondent]; exists { + updatedFields["correspondent"] = correspondentID + } else { + newCorrespondent := instantiateCorrespondent(document.SuggestedCorrespondent) + newCorrespondentID, err := client.CreateCorrespondent(context.Background(), newCorrespondent) + if err != nil { + log.Errorf("Error creating correspondent with name %s: %v\n", document.SuggestedCorrespondent, err) + return err + } + log.Infof("Created correspondent with name %s and ID %d\n", document.SuggestedCorrespondent, newCorrespondentID) + updatedFields["correspondent"] = newCorrespondentID + } + } + suggestedTitle := document.SuggestedTitle if len(suggestedTitle) > 128 { suggestedTitle = suggestedTitle[:128] @@ -341,7 +404,7 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum // Send the update request using the generic Do method path := fmt.Sprintf("api/documents/%d/", documentID) - resp, err := c.Do(ctx, "PATCH", path, bytes.NewBuffer(jsonData)) + resp, err := client.Do(ctx, "PATCH", path, bytes.NewBuffer(jsonData)) if err != nil { log.Errorf("Error updating document %d: %v", documentID, err) return err @@ -399,9 +462,9 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum // DownloadDocumentAsImages downloads the PDF file of the specified document and converts it to images // If limitPages > 0, only the first N pages will be processed -func (c *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, documentId int, limitPages int) ([]string, error) { +func (client *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, documentId int, limitPages int) ([]string, error) { // Create a directory named after the document ID - docDir := filepath.Join(c.GetCacheFolder(), fmt.Sprintf("/document-%d", documentId)) + docDir := filepath.Join(client.GetCacheFolder(), fmt.Sprintf("document-%d", documentId)) if _, err := os.Stat(docDir); os.IsNotExist(err) { err = os.MkdirAll(docDir, 0755) if err != nil { @@ -429,7 +492,7 @@ func (c *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, document // Proceed with downloading and converting the document to images path := fmt.Sprintf("api/documents/%d/download/", documentId) - resp, err := c.Do(ctx, "GET", path, nil) + resp, err := client.Do(ctx, "GET", path, nil) if err != nil { return nil, err } @@ -526,14 +589,97 @@ func (c *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, document } // GetCacheFolder returns the cache folder for the PaperlessClient -func (c *PaperlessClient) GetCacheFolder() string { - if c.CacheFolder == "" { - c.CacheFolder = filepath.Join(os.TempDir(), "paperless-gpt") +func (client *PaperlessClient) GetCacheFolder() string { + if client.CacheFolder == "" { + client.CacheFolder = filepath.Join(os.TempDir(), "paperless-gpt") } - return c.CacheFolder + return client.CacheFolder } // urlEncode encodes a string for safe URL usage func urlEncode(s string) string { return strings.ReplaceAll(s, " ", "+") } + +// instantiateCorrespondent creates a new Correspondent object with default values +func instantiateCorrespondent(name string) Correspondent { + return Correspondent{ + Name: name, + MatchingAlgorithm: 0, + Match: "", + IsInsensitive: true, + Owner: nil, + } +} + +// CreateCorrespondent creates a new correspondent in Paperless-NGX +func (client *PaperlessClient) CreateCorrespondent(ctx context.Context, correspondent Correspondent) (int, error) { + url := "api/correspondents/" + + // Marshal the correspondent data to JSON + jsonData, err := json.Marshal(correspondent) + if err != nil { + return 0, err + } + + // Send the POST request + resp, err := client.Do(ctx, "POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + bodyBytes, _ := io.ReadAll(resp.Body) + return 0, fmt.Errorf("error creating correspondent: %d, %s", resp.StatusCode, string(bodyBytes)) + } + + // Decode the response body to get the ID of the created correspondent + var createdCorrespondent struct { + ID int `json:"id"` + } + err = json.NewDecoder(resp.Body).Decode(&createdCorrespondent) + if err != nil { + return 0, err + } + + return createdCorrespondent.ID, nil +} + +// CorrespondentResponse represents the response structure for correspondents +type CorrespondentResponse struct { + Results []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"results"` +} + +// GetAllCorrespondents retrieves all correspondents from the Paperless-NGX API +func (client *PaperlessClient) GetAllCorrespondents(ctx context.Context) (map[string]int, error) { + correspondentIDMapping := make(map[string]int) + path := "api/correspondents/?page_size=9999" + + resp, err := client.Do(ctx, "GET", path, nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("error fetching correspondents: %d, %s", resp.StatusCode, string(bodyBytes)) + } + + var correspondentsResponse CorrespondentResponse + + err = json.NewDecoder(resp.Body).Decode(&correspondentsResponse) + if err != nil { + return nil, err + } + + for _, correspondent := range correspondentsResponse.Results { + correspondentIDMapping[correspondent.Name] = correspondent.ID + } + + return correspondentIDMapping, nil +} diff --git a/paperless_test.go b/paperless_test.go index 9693b24..30b10a0 100644 --- a/paperless_test.go +++ b/paperless_test.go @@ -55,6 +55,12 @@ func newTestEnv(t *testing.T) *testEnv { env.client = NewPaperlessClient(env.server.URL, "test-token") env.client.HTTPClient = env.server.Client() + // Add mock response for /api/correspondents/ + env.setMockResponse("/api/correspondents/", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"results": [{"id": 1, "name": "Alpha"}, {"id": 2, "name": "Beta"}]}`)) + }) + return env } @@ -176,7 +182,7 @@ func TestGetDocumentsByTags(t *testing.T) { documentsResponse := GetDocumentsApiResponse{ Results: []struct { ID int `json:"id"` - Correspondent interface{} `json:"correspondent"` + Correspondent int `json:"correspondent"` DocumentType interface{} `json:"document_type"` StoragePath interface{} `json:"storage_path"` Title string `json:"title"` @@ -200,16 +206,18 @@ func TestGetDocumentsByTags(t *testing.T) { } `json:"__search_hit__"` }{ { - ID: 1, - Title: "Document 1", - Content: "Content 1", - Tags: []int{1, 2}, + ID: 1, + Title: "Document 1", + Content: "Content 1", + Tags: []int{1, 2}, + Correspondent: 1, }, { - ID: 2, - Title: "Document 2", - Content: "Content 2", - Tags: []int{2, 3}, + ID: 2, + Title: "Document 2", + Content: "Content 2", + Tags: []int{2, 3}, + Correspondent: 2, }, }, } @@ -227,7 +235,7 @@ func TestGetDocumentsByTags(t *testing.T) { // Set mock responses env.setMockResponse("/api/documents/", func(w http.ResponseWriter, r *http.Request) { // Verify query parameters - expectedQuery := "tags__name__iexact=tag1&tags__name__iexact=tag2" + expectedQuery := "tags__name__iexact=tag1&tags__name__iexact=tag2&page_size=25" assert.Equal(t, expectedQuery, r.URL.RawQuery) w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(documentsResponse) @@ -240,21 +248,23 @@ func TestGetDocumentsByTags(t *testing.T) { ctx := context.Background() tags := []string{"tag1", "tag2"} - documents, err := env.client.GetDocumentsByTags(ctx, tags) + documents, err := env.client.GetDocumentsByTags(ctx, tags, 25) require.NoError(t, err) expectedDocuments := []Document{ { - ID: 1, - Title: "Document 1", - Content: "Content 1", - Tags: []string{"tag1", "tag2"}, + ID: 1, + Title: "Document 1", + Content: "Content 1", + Tags: []string{"tag1", "tag2"}, + Correspondent: "Alpha", }, { - ID: 2, - Title: "Document 2", - Content: "Content 2", - Tags: []string{"tag2", "tag3"}, + ID: 2, + Title: "Document 2", + Content: "Content 2", + Tags: []string{"tag2", "tag3"}, + Correspondent: "Beta", }, } @@ -348,7 +358,8 @@ func TestUpdateDocuments(t *testing.T) { // Expected updated fields expectedFields := map[string]interface{}{ "title": "New Title", - "tags": []interface{}{float64(idTag1), float64(idTag2), float64(idTag3)}, // keep also previous tags + // do not keep previous tags since the tag generation will already take care to include old ones: + "tags": []interface{}{float64(idTag2), float64(idTag3)}, } assert.Equal(t, expectedFields, updatedFields) @@ -412,7 +423,7 @@ func TestDownloadDocumentAsImages_ManyPages(t *testing.T) { ID: 321, } - // Get sample PDF from tests/pdf/sample.pdf + // Get sample PDF from tests/pdf/many-pages.pdf pdfFile := "tests/pdf/many-pages.pdf" pdfContent, err := os.ReadFile(pdfFile) require.NoError(t, err) diff --git a/types.go b/types.go index 11fbc53..3f33433 100644 --- a/types.go +++ b/types.go @@ -11,7 +11,7 @@ type GetDocumentsApiResponse struct { All []int `json:"all"` Results []struct { ID int `json:"id"` - Correspondent interface{} `json:"correspondent"` + Correspondent int `json:"correspondent"` DocumentType interface{} `json:"document_type"` StoragePath interface{} `json:"storage_path"` Title string `json:"title"` @@ -38,7 +38,7 @@ type GetDocumentsApiResponse struct { type GetDocumentApiResponse struct { ID int `json:"id"` - Correspondent interface{} `json:"correspondent"` + Correspondent int `json:"correspondent"` DocumentType interface{} `json:"document_type"` StoragePath interface{} `json:"storage_path"` Title string `json:"title"` @@ -59,25 +59,46 @@ type GetDocumentApiResponse struct { // Document is a stripped down version of the document object from paperless-ngx. // Response payload for /documents endpoint and part of request payload for /generate-suggestions endpoint type Document struct { - ID int `json:"id"` - Title string `json:"title"` - Content string `json:"content"` - Tags []string `json:"tags"` + ID int `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + Tags []string `json:"tags"` + Correspondent string `json:"correspondent"` } // GenerateSuggestionsRequest is the request payload for generating suggestions for /generate-suggestions endpoint type GenerateSuggestionsRequest struct { - Documents []Document `json:"documents"` - GenerateTitles bool `json:"generate_titles,omitempty"` - GenerateTags bool `json:"generate_tags,omitempty"` + Documents []Document `json:"documents"` + GenerateTitles bool `json:"generate_titles,omitempty"` + GenerateTags bool `json:"generate_tags,omitempty"` + GenerateCorrespondents bool `json:"generate_correspondents,omitempty"` } // DocumentSuggestion is the response payload for /generate-suggestions endpoint and the request payload for /update-documents endpoint (as an array) type DocumentSuggestion struct { - ID int `json:"id"` - OriginalDocument Document `json:"original_document"` - SuggestedTitle string `json:"suggested_title,omitempty"` - SuggestedTags []string `json:"suggested_tags,omitempty"` - SuggestedContent string `json:"suggested_content,omitempty"` - RemoveTags []string `json:"remove_tags,omitempty"` + ID int `json:"id"` + OriginalDocument Document `json:"original_document"` + SuggestedTitle string `json:"suggested_title,omitempty"` + SuggestedTags []string `json:"suggested_tags,omitempty"` + SuggestedContent string `json:"suggested_content,omitempty"` + SuggestedCorrespondent string `json:"suggested_correspondent,omitempty"` + RemoveTags []string `json:"remove_tags,omitempty"` +} + +type Correspondent struct { + Name string `json:"name"` + MatchingAlgorithm int `json:"matching_algorithm"` + Match string `json:"match"` + IsInsensitive bool `json:"is_insensitive"` + Owner *int `json:"owner"` + SetPermissions struct { + View struct { + Users []int `json:"users"` + Groups []int `json:"groups"` + } `json:"view"` + Change struct { + Users []int `json:"users"` + Groups []int `json:"groups"` + } `json:"change"` + } `json:"set_permissions"` } diff --git a/web-app/src/DocumentProcessor.tsx b/web-app/src/DocumentProcessor.tsx index 093a5e1..f8efd13 100644 --- a/web-app/src/DocumentProcessor.tsx +++ b/web-app/src/DocumentProcessor.tsx @@ -11,12 +11,14 @@ export interface Document { title: string; content: string; tags: string[]; + correspondent: string; } export interface GenerateSuggestionsRequest { documents: Document[]; generate_titles?: boolean; generate_tags?: boolean; + generate_correspondents?: boolean; } export interface DocumentSuggestion { @@ -25,6 +27,7 @@ export interface DocumentSuggestion { suggested_title?: string; suggested_tags?: string[]; suggested_content?: string; + suggested_correspondent?: string; } export interface TagOption { @@ -43,6 +46,7 @@ const DocumentProcessor: React.FC = () => { const [filterTag, setFilterTag] = useState(null); const [generateTitles, setGenerateTitles] = useState(true); const [generateTags, setGenerateTags] = useState(true); + const [generateCorrespondents, setGenerateCorrespondents] = useState(true); const [error, setError] = useState(null); // Custom hook to fetch initial data @@ -81,6 +85,7 @@ const DocumentProcessor: React.FC = () => { documents, generate_titles: generateTitles, generate_tags: generateTags, + generate_correspondents: generateCorrespondents, }; const { data } = await axios.post( @@ -137,6 +142,7 @@ const DocumentProcessor: React.FC = () => { ); }; + const handleTitleChange = (docId: number, title: string) => { setSuggestions((prevSuggestions) => prevSuggestions.map((doc) => @@ -145,6 +151,14 @@ const DocumentProcessor: React.FC = () => { ); }; + const handleCorrespondentChange = (docId: number, correspondent: string) => { + setSuggestions((prevSuggestions) => + prevSuggestions.map((doc) => + doc.id === docId ? { ...doc, suggested_correspondent: correspondent } : doc + ) + ); + } + const resetSuggestions = () => { setSuggestions([]); }; @@ -214,6 +228,8 @@ const DocumentProcessor: React.FC = () => { setGenerateTitles={setGenerateTitles} generateTags={generateTags} setGenerateTags={setGenerateTags} + generateCorrespondents={generateCorrespondents} + setGenerateCorrespondents={setGenerateCorrespondents} onProcess={handleProcessDocuments} processing={processing} onReload={reloadDocuments} @@ -225,6 +241,7 @@ const DocumentProcessor: React.FC = () => { onTitleChange={handleTitleChange} onTagAddition={handleTagAddition} onTagDeletion={handleTagDeletion} + onCorrespondentChange={handleCorrespondentChange} onBack={resetSuggestions} onUpdate={handleUpdateDocuments} updating={updating} diff --git a/web-app/src/components/DocumentCard.tsx b/web-app/src/components/DocumentCard.tsx index bc6ec2a..1412fbe 100644 --- a/web-app/src/components/DocumentCard.tsx +++ b/web-app/src/components/DocumentCard.tsx @@ -13,6 +13,9 @@ const DocumentCard: React.FC = ({ document }) => ( ? `${document.content.substring(0, 100)}...` : document.content}

+

+ Correspondent: {document.correspondent} +

{document.tags.map((tag) => ( = ({ document }) => (

{document.title}

{document.content}

+

+ Correspondent: {document.correspondent} +

{document.tags.map((tag) => ( >; generateTags: boolean; setGenerateTags: React.Dispatch>; + generateCorrespondents: boolean; + setGenerateCorrespondents: React.Dispatch>; onProcess: () => void; processing: boolean; onReload: () => void; @@ -20,6 +22,8 @@ const DocumentsToProcess: React.FC = ({ setGenerateTitles, generateTags, setGenerateTags, + generateCorrespondents, + setGenerateCorrespondents, onProcess, processing, onReload, @@ -64,6 +68,15 @@ const DocumentsToProcess: React.FC = ({ /> Generate Tags +
diff --git a/web-app/src/components/SuggestionCard.tsx b/web-app/src/components/SuggestionCard.tsx index 361ce25..6b7be1f 100644 --- a/web-app/src/components/SuggestionCard.tsx +++ b/web-app/src/components/SuggestionCard.tsx @@ -8,6 +8,7 @@ interface SuggestionCardProps { onTitleChange: (docId: number, title: string) => void; onTagAddition: (docId: number, tag: TagOption) => void; onTagDeletion: (docId: number, index: number) => void; + onCorrespondentChange: (docId: number, correspondent: string) => void; } const SuggestionCard: React.FC = ({ @@ -16,6 +17,7 @@ const SuggestionCard: React.FC = ({ onTitleChange, onTagAddition, onTagDeletion, + onCorrespondentChange, }) => { const sortedAvailableTags = availableTags.sort((a, b) => a.name.localeCompare(b.name)); const document = suggestion.original_document; @@ -49,6 +51,9 @@ const SuggestionCard: React.FC = ({
+ = ({ className="w-full border border-gray-300 dark:border-gray-600 rounded px-2 py-1 mt-2 focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-gray-200" />
+ ({ @@ -99,6 +107,18 @@ const SuggestionCard: React.FC = ({ }} />
+
+ + onCorrespondentChange(suggestion.id, e.target.value)} + className="w-full border border-gray-300 dark:border-gray-600 rounded px-2 py-1 mt-2 focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-gray-200" + placeholder="Correspondent" + /> +
); diff --git a/web-app/src/components/SuggestionsReview.tsx b/web-app/src/components/SuggestionsReview.tsx index 3a1f648..a0cc5af 100644 --- a/web-app/src/components/SuggestionsReview.tsx +++ b/web-app/src/components/SuggestionsReview.tsx @@ -8,6 +8,7 @@ interface SuggestionsReviewProps { onTitleChange: (docId: number, title: string) => void; onTagAddition: (docId: number, tag: TagOption) => void; onTagDeletion: (docId: number, index: number) => void; + onCorrespondentChange: (docId: number, correspondent: string) => void; onBack: () => void; onUpdate: () => void; updating: boolean; @@ -19,6 +20,7 @@ const SuggestionsReview: React.FC = ({ onTitleChange, onTagAddition, onTagDeletion, + onCorrespondentChange, onBack, onUpdate, updating, @@ -36,6 +38,7 @@ const SuggestionsReview: React.FC = ({ onTitleChange={onTitleChange} onTagAddition={onTagAddition} onTagDeletion={onTagDeletion} + onCorrespondentChange={onCorrespondentChange} /> ))} diff --git a/web-app/tsconfig.app.tsbuildinfo b/web-app/tsconfig.app.tsbuildinfo index 40eb039..577769a 100644 --- a/web-app/tsconfig.app.tsbuildinfo +++ b/web-app/tsconfig.app.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/app.tsx","./src/documentprocessor.tsx","./src/experimentalocr.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/documentcard.tsx","./src/components/documentstoprocess.tsx","./src/components/nodocuments.tsx","./src/components/successmodal.tsx","./src/components/suggestioncard.tsx","./src/components/suggestionsreview.tsx"],"version":"5.6.2"} \ No newline at end of file +{"root":["./src/app.tsx","./src/documentprocessor.tsx","./src/experimentalocr.tsx","./src/history.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/documentcard.tsx","./src/components/documentstoprocess.tsx","./src/components/nodocuments.tsx","./src/components/sidebar.tsx","./src/components/successmodal.tsx","./src/components/suggestioncard.tsx","./src/components/suggestionsreview.tsx","./src/components/undocard.tsx"],"version":"5.7.2"} \ No newline at end of file diff --git a/web-app/tsconfig.node.tsbuildinfo b/web-app/tsconfig.node.tsbuildinfo index 98ef2f9..1e7ed27 100644 --- a/web-app/tsconfig.node.tsbuildinfo +++ b/web-app/tsconfig.node.tsbuildinfo @@ -1 +1 @@ -{"root":["./vite.config.ts"],"version":"5.6.2"} \ No newline at end of file +{"root":["./vite.config.ts"],"version":"5.7.2"} \ No newline at end of file