Mirrored-Projects/paperless-gpt
1
0
Fork 0
mirror of https://github.com/icereed/paperless-gpt.git synced 2025-03-12 12:58:02 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

LLM powered Correspondents handling (#89)

Browse source 
---------

Co-authored-by: Jonas Hess <Jonas@Hess.pm>
This commit is contained in:
Icereed 2025-01-13 15:59:29 +01:00 committed by GitHub
parent c1104449dd
commit e661989829
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 475 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
README.md
View file

@ -74,7 +74,7 @@ https://github.com/user-attachments/assets/bd5d38b9-9309-40b9-93ca-918dfa4f3fd4
Here’s an example `docker-compose.yml` to spin up **paperless-gpt** alongside paperless-ngx:
```yaml
version: '3.7'
version: "3.7"
services:
paperless-ngx:
image: ghcr.io/paperless-ngx/paperless-ngx:latest
@ -102,7 +102,7 @@ services:
volumes:
- ./prompts:/app/prompts # Mount the prompts directory
ports:
- '8080:8080'
- "8080:8080"
depends_on:
- paperless-ngx
```
@ -146,6 +146,8 @@ services:
### Environment Variables
**Note:** When using Ollama, ensure that the Ollama server is running and accessible from the paperless-gpt container.
=======
| Variable | Description | Required |
|------------------------|------------------------------------------------------------------------------------------------------------------|----------|
| `PAPERLESS_BASE_URL` | URL of your paperless-ngx instance (e.g. `http://paperless-ngx:8000`). | Yes |
@ -167,7 +169,9 @@ services:
| `WEBUI_PATH` | Path for static content. Default: `./web-app/dist`. | No |
| `AUTO_GENERATE_TITLE` | Generate titles automatically if `paperless-gpt-auto` is used. Default: `true`. | No |
| `AUTO_GENERATE_TAGS` | Generate tags automatically if `paperless-gpt-auto` is used. Default: `true`. | No |
| `AUTO_GENERATE_CORRESPONDENTS` | Generate correspondents automatically if `paperless-gpt-auto` is used. Default: `true`. | No |
| `OCR_LIMIT_PAGES` | Limit the number of pages for OCR. Set to `0` for no limit. Default: `5`. | No |
| `CORRESPONDENT_BLACK_LIST` | A comma-separated list of names to exclude from the correspondents suggestions. Example: `John Doe, Jane Smith`.
### Custom Prompt Templates

2
app_http_handlers.go
View file

@ -98,7 +98,7 @@ func (app *App) getAllTagsHandler(c *gin.Context) {
func (app *App) documentsHandler(c *gin.Context) {
ctx := c.Request.Context()
documents, err := app.Client.GetDocumentsByTags(ctx, []string{manualTag})
documents, err := app.Client.GetDocumentsByTags(ctx, []string{manualTag}, 25)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error fetching documents: %v", err)})
log.Errorf("Error fetching documents: %v", err)

83
app_llm.go
View file

@ -6,6 +6,7 @@ import (
"encoding/base64"
"fmt"
"image"
"slices"
"strings"
"sync"
@ -15,12 +16,53 @@ import (
"github.com/tmc/langchaingo/llms"
)
// getSuggestedCorrespondent generates a suggested correspondent for a document using the LLM
func (app *App) getSuggestedCorrespondent(ctx context.Context, content string, suggestedTitle string, availableCorrespondents []string, correspondentBlackList []string) (string, error) {
likelyLanguage := getLikelyLanguage()
templateMutex.RLock()
defer templateMutex.RUnlock()
var promptBuffer bytes.Buffer
err := correspondentTemplate.Execute(&promptBuffer, map[string]interface{}{
"Language": likelyLanguage,
"AvailableCorrespondents": availableCorrespondents,
"BlackList": correspondentBlackList,
"Title": suggestedTitle,
"Content": content,
})
if err != nil {
return "", fmt.Errorf("error executing correspondent template: %v", err)
}
prompt := promptBuffer.String()
log.Debugf("Correspondent suggestion prompt: %s", prompt)
completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{
{
Parts: []llms.ContentPart{
llms.TextContent{
Text: prompt,
},
},
Role: llms.ChatMessageTypeHuman,
},
})
if err != nil {
return "", fmt.Errorf("error getting response from LLM: %v", err)
}
response := strings.TrimSpace(completion.Choices[0].Content)
return response, nil
}
// getSuggestedTags generates suggested tags for a document using the LLM
func (app *App) getSuggestedTags(
ctx context.Context,
content string,
suggestedTitle string,
availableTags []string,
originalTags []string,
logger *logrus.Entry) ([]string, error) {
likelyLanguage := getLikelyLanguage()
@ -31,6 +73,7 @@ func (app *App) getSuggestedTags(
err := tagTemplate.Execute(&promptBuffer, map[string]interface{}{
"Language": likelyLanguage,
"AvailableTags": availableTags,
"OriginalTags": originalTags,
"Title": suggestedTitle,
"Content": content,
})
@ -63,6 +106,12 @@ func (app *App) getSuggestedTags(
suggestedTags[i] = strings.TrimSpace(tag)
}
// append the original tags to the suggested tags
suggestedTags = append(suggestedTags, originalTags...)
// Remove duplicates
slices.Sort(suggestedTags)
suggestedTags = slices.Compact(suggestedTags)
// Filter out tags that are not in the available tags list
filteredTags := []string{}
for _, tag := range suggestedTags {
@ -190,6 +239,18 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
availableTagNames = append(availableTagNames, tagName)
}
// Prepare a list of document correspodents
availableCorrespondentsMap, err := app.Client.GetAllCorrespondents(ctx)
if err != nil {
return nil, fmt.Errorf("failed to fetch available correspondents: %v", err)
}
// Prepare a list of correspondent names
availableCorrespondentNames := make([]string, 0, len(availableCorrespondentsMap))
for correspondentName := range availableCorrespondentsMap {
availableCorrespondentNames = append(availableCorrespondentNames, correspondentName)
}
documents := suggestionRequest.Documents
documentSuggestions := []DocumentSuggestion{}
@ -212,6 +273,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
var suggestedTitle string
var suggestedTags []string
var suggestedCorrespondent string
if suggestionRequest.GenerateTitles {
suggestedTitle, err = app.getSuggestedTitle(ctx, content, docLogger)
@ -225,7 +287,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
}
if suggestionRequest.GenerateTags {
suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, docLogger)
suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, doc.Tags, docLogger)
if err != nil {
mu.Lock()
errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err))
@ -235,6 +297,18 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
}
}
if suggestionRequest.GenerateCorrespondents {
suggestedCorrespondent, err = app.getSuggestedCorrespondent(ctx, content, suggestedTitle, availableCorrespondentNames, correspondentBlackList)
if err != nil {
mu.Lock()
errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err))
mu.Unlock()
log.Errorf("Error generating correspondents for document %d: %v", documentID, err)
return
}
}
mu.Lock()
suggestion := DocumentSuggestion{
ID: documentID,
@ -256,6 +330,13 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
suggestion.SuggestedTags = doc.Tags
}
// Correspondents
if suggestionRequest.GenerateCorrespondents {
log.Printf("Suggested correspondent for document %d: %s", documentID, suggestedCorrespondent)
suggestion.SuggestedCorrespondent = suggestedCorrespondent
} else {
suggestion.SuggestedCorrespondent = ""
}
// Remove manual tag from the list of suggested tags
suggestion.RemoveTags = []string{manualTag, autoTag}

98
main.go
View file

@ -30,29 +30,33 @@ var (
log = logrus.New()
// Environment Variables
paperlessBaseURL = os.Getenv("PAPERLESS_BASE_URL")
paperlessAPIToken = os.Getenv("PAPERLESS_API_TOKEN")
openaiAPIKey = os.Getenv("OPENAI_API_KEY")
manualTag = os.Getenv("MANUAL_TAG")
autoTag = os.Getenv("AUTO_TAG")
manualOcrTag = os.Getenv("MANUAL_OCR_TAG") // Not used yet
autoOcrTag = os.Getenv("AUTO_OCR_TAG")
llmProvider = os.Getenv("LLM_PROVIDER")
llmModel = os.Getenv("LLM_MODEL")
visionLlmProvider = os.Getenv("VISION_LLM_PROVIDER")
visionLlmModel = os.Getenv("VISION_LLM_MODEL")
logLevel = strings.ToLower(os.Getenv("LOG_LEVEL"))
listenInterface = os.Getenv("LISTEN_INTERFACE")
webuiPath = os.Getenv("WEBUI_PATH")
autoGenerateTitle = os.Getenv("AUTO_GENERATE_TITLE")
autoGenerateTags = os.Getenv("AUTO_GENERATE_TAGS")
limitOcrPages int // Will be read from OCR_LIMIT_PAGES
correspondentBlackList = strings.Split(os.Getenv("CORRESPONDENT_BLACK_LIST"), ",")
paperlessBaseURL = os.Getenv("PAPERLESS_BASE_URL")
paperlessAPIToken = os.Getenv("PAPERLESS_API_TOKEN")
openaiAPIKey = os.Getenv("OPENAI_API_KEY")
manualTag = os.Getenv("MANUAL_TAG")
autoTag = os.Getenv("AUTO_TAG")
manualOcrTag = os.Getenv("MANUAL_OCR_TAG") // Not used yet
autoOcrTag = os.Getenv("AUTO_OCR_TAG")
llmProvider = os.Getenv("LLM_PROVIDER")
llmModel = os.Getenv("LLM_MODEL")
visionLlmProvider = os.Getenv("VISION_LLM_PROVIDER")
visionLlmModel = os.Getenv("VISION_LLM_MODEL")
logLevel = strings.ToLower(os.Getenv("LOG_LEVEL"))
listenInterface = os.Getenv("LISTEN_INTERFACE")
webuiPath = os.Getenv("WEBUI_PATH")
autoGenerateTitle = os.Getenv("AUTO_GENERATE_TITLE")
autoGenerateTags = os.Getenv("AUTO_GENERATE_TAGS")
autoGenerateCorrespondents = os.Getenv("AUTO_GENERATE_CORRESPONDENTS")
limitOcrPages int // Will be read from OCR_LIMIT_PAGES
// Templates
titleTemplate *template.Template
tagTemplate *template.Template
ocrTemplate *template.Template
templateMutex sync.RWMutex
titleTemplate *template.Template
tagTemplate *template.Template
correspondentTemplate *template.Template
ocrTemplate *template.Template
templateMutex sync.RWMutex
// Default templates
defaultTitleTemplate = `I will provide you with the content of a document that has been partially read by OCR (so it may contain errors).
@ -77,7 +81,33 @@ Content:
Please concisely select the {{.Language}} tags from the list above that best describe the document.
Be very selective and only choose the most relevant tags since too many tags will make the document less discoverable.
`
defaultCorrespondentTemplate = `I will provide you with the content of a document. Your task is to suggest a correspondent that is most relevant to the document.
Correspondents are the senders of documents that reach you. In the other direction, correspondents are the recipients of documents that you send.
In Paperless-ngx we can imagine correspondents as virtual drawers in which all documents of a person or company are stored. With just one click, we can find all the documents assigned to a specific correspondent.
Try to suggest a correspondent, either from the example list or come up with a new correspondent.
Respond only with a correspondent, without any additional information!
Be sure to choose a correspondent that is most relevant to the document.
Try to avoid any legal or financial suffixes like "GmbH" or "AG" in the correspondent name. For example use "Microsoft" instead of "Microsoft Ireland Operations Limited" or "Amazon" instead of "Amazon EU S.a.r.l.".
If you can't find a suitable correspondent, you can respond with "Unknown".
Example Correspondents:
{{.AvailableCorrespondents | join ", "}}
List of Correspondents with Blacklisted Names. Please avoid these correspondents or variations of their names:
{{.BlackList | join ", "}}
Title of the document:
{{.Title}}
The content is likely in {{.Language}}.
Document Content:
{{.Content}}
`
defaultOcrPrompt = `Just transcribe the text in this image and preserve the formatting and layout (high quality OCR). Do that for ALL the text in the image. Be thorough and pay attention. This is very important. The image is from a text document so be sure to continue until the bottom of the page. Thanks a lot! You tend to forget about some text in the image so please focus! Use markdown format but without a code block.`
)
@ -363,7 +393,7 @@ func documentLogger(documentID int) *logrus.Entry {
func (app *App) processAutoTagDocuments() (int, error) {
ctx := context.Background()
documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoTag})
documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoTag}, 25)
if err != nil {
return 0, fmt.Errorf("error fetching documents with autoTag: %w", err)
}
@ -380,9 +410,10 @@ func (app *App) processAutoTagDocuments() (int, error) {
docLogger.Info("Processing document for auto-tagging")
suggestionRequest := GenerateSuggestionsRequest{
Documents: []Document{document},
GenerateTitles: strings.ToLower(autoGenerateTitle) != "false",
GenerateTags: strings.ToLower(autoGenerateTags) != "false",
Documents: []Document{document},
GenerateTitles: strings.ToLower(autoGenerateTitle) != "false",
GenerateTags: strings.ToLower(autoGenerateTags) != "false",
GenerateCorrespondents: strings.ToLower(autoGenerateCorrespondents) != "false",
}
suggestions, err := app.generateDocumentSuggestions(ctx, suggestionRequest, docLogger)
@ -404,7 +435,7 @@ func (app *App) processAutoTagDocuments() (int, error) {
func (app *App) processAutoOcrTagDocuments() (int, error) {
ctx := context.Background()
documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoOcrTag})
documents, err := app.Client.GetDocumentsByTags(ctx, []string{autoOcrTag}, 25)
if err != nil {
return 0, fmt.Errorf("error fetching documents with autoOcrTag: %w", err)
}
@ -504,6 +535,21 @@ func loadTemplates() {
log.Fatalf("Failed to parse tag template: %v", err)
}
// Load correspondent template
correspondentTemplatePath := filepath.Join(promptsDir, "correspondent_prompt.tmpl")
correspondentTemplateContent, err := os.ReadFile(correspondentTemplatePath)
if err != nil {
log.Errorf("Could not read %s, using default template: %v", correspondentTemplatePath, err)
correspondentTemplateContent = []byte(defaultCorrespondentTemplate)
if err := os.WriteFile(correspondentTemplatePath, correspondentTemplateContent, os.ModePerm); err != nil {
log.Fatalf("Failed to write default correspondent template to disk: %v", err)
}
}
correspondentTemplate, err = template.New("correspondent").Funcs(sprig.FuncMap()).Parse(string(correspondentTemplateContent))
if err != nil {
log.Fatalf("Failed to parse correspondent template: %v", err)
}
// Load OCR template
ocrTemplatePath := filepath.Join(promptsDir, "ocr_prompt.tmpl")
ocrTemplateContent, err := os.ReadFile(ocrTemplatePath)

224
paperless.go
View file

@ -67,29 +67,29 @@ func NewPaperlessClient(baseURL, apiToken string) *PaperlessClient {
}
// Do method to make requests to the Paperless-NGX API
func (c *PaperlessClient) Do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
url := fmt.Sprintf("%s/%s", c.BaseURL, strings.TrimLeft(path, "/"))
func (client *PaperlessClient) Do(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) {
url := fmt.Sprintf("%s/%s", client.BaseURL, strings.TrimLeft(path, "/"))
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", fmt.Sprintf("Token %s", c.APIToken))
req.Header.Set("Authorization", fmt.Sprintf("Token %s", client.APIToken))
// Set Content-Type if body is present
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
return c.HTTPClient.Do(req)
return client.HTTPClient.Do(req)
}
// GetAllTags retrieves all tags from the Paperless-NGX API
func (c *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error) {
func (client *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error) {
tagIDMapping := make(map[string]int)
path := "api/tags/"
for path != "" {
resp, err := c.Do(ctx, "GET", path, nil)
resp, err := client.Do(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
@ -120,8 +120,8 @@ func (c *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error
// Extract relative path from the Next URL
if tagsResponse.Next != "" {
nextURL := tagsResponse.Next
if strings.HasPrefix(nextURL, c.BaseURL) {
nextURL = strings.TrimPrefix(nextURL, c.BaseURL+"/")
if strings.HasPrefix(nextURL, client.BaseURL) {
nextURL = strings.TrimPrefix(nextURL, client.BaseURL+"/")
}
path = nextURL
} else {
@ -133,15 +133,15 @@ func (c *PaperlessClient) GetAllTags(ctx context.Context) (map[string]int, error
}
// GetDocumentsByTags retrieves documents that match the specified tags
func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string) ([]Document, error) {
func (client *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string, pageSize int) ([]Document, error) {
tagQueries := make([]string, len(tags))
for i, tag := range tags {
tagQueries[i] = fmt.Sprintf("tags__name__iexact=%s", tag)
}
searchQuery := strings.Join(tagQueries, "&")
path := fmt.Sprintf("api/documents/?%s", urlEncode(searchQuery))
path := fmt.Sprintf("api/documents/?%s&page_size=%d", urlEncode(searchQuery), pageSize)
resp, err := c.Do(ctx, "GET", path, nil)
resp, err := client.Do(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
@ -158,7 +158,12 @@ func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string)
return nil, err
}
allTags, err := c.GetAllTags(ctx)
allTags, err := client.GetAllTags(ctx)
if err != nil {
return nil, err
}
allCorrespondents, err := client.GetAllCorrespondents(ctx)
if err != nil {
return nil, err
}
@ -175,11 +180,22 @@ func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string)
}
}
correspondentName := ""
if result.Correspondent != 0 {
for name, id := range allCorrespondents {
if result.Correspondent == id {
correspondentName = name
break
}
}
}
documents = append(documents, Document{
ID: result.ID,
Title: result.Title,
Content: result.Content,
Tags: tagNames,
ID: result.ID,
Title: result.Title,
Content: result.Content,
Correspondent: correspondentName,
Tags: tagNames,
})
}
@ -187,9 +203,9 @@ func (c *PaperlessClient) GetDocumentsByTags(ctx context.Context, tags []string)
}
// DownloadPDF downloads the PDF file of the specified document
func (c *PaperlessClient) DownloadPDF(ctx context.Context, document Document) ([]byte, error) {
func (client *PaperlessClient) DownloadPDF(ctx context.Context, document Document) ([]byte, error) {
path := fmt.Sprintf("api/documents/%d/download/", document.ID)
resp, err := c.Do(ctx, "GET", path, nil)
resp, err := client.Do(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
@ -203,9 +219,9 @@ func (c *PaperlessClient) DownloadPDF(ctx context.Context, document Document) ([
return io.ReadAll(resp.Body)
}
func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Document, error) {
func (client *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Document, error) {
path := fmt.Sprintf("api/documents/%d/", documentID)
resp, err := c.Do(ctx, "GET", path, nil)
resp, err := client.Do(ctx, "GET", path, nil)
if err != nil {
return Document{}, err
}
@ -222,11 +238,17 @@ func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Docu
return Document{}, err
}
allTags, err := c.GetAllTags(ctx)
allTags, err := client.GetAllTags(ctx)
if err != nil {
return Document{}, err
}
allCorrespondents, err := client.GetAllCorrespondents(ctx)
if err != nil {
return Document{}, err
}
// Match tag IDs to tag names
tagNames := make([]string, len(documentResponse.Tags))
for i, resultTagID := range documentResponse.Tags {
for tagName, tagID := range allTags {
@ -237,23 +259,51 @@ func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Docu
}
}
// Match correspondent ID to correspondent name
correspondentName := ""
for name, id := range allCorrespondents {
if documentResponse.Correspondent == id {
correspondentName = name
break
}
}
return Document{
ID: documentResponse.ID,
Title: documentResponse.Title,
Content: documentResponse.Content,
Tags: tagNames,
ID: documentResponse.ID,
Title: documentResponse.Title,
Content: documentResponse.Content,
Correspondent: correspondentName,
Tags: tagNames,
}, nil
}
// UpdateDocuments updates the specified documents with suggested changes
func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []DocumentSuggestion, db *gorm.DB, isUndo bool) error {
func (client *PaperlessClient) UpdateDocuments(ctx context.Context, documents []DocumentSuggestion, db *gorm.DB, isUndo bool) error {
// Fetch all available tags
availableTags, err := c.GetAllTags(ctx)
availableTags, err := client.GetAllTags(ctx)
if err != nil {
log.Errorf("Error fetching available tags: %v", err)
return err
}
documentsContainSuggestedCorrespondent := false
for _, document := range documents {
if document.SuggestedCorrespondent != "" {
documentsContainSuggestedCorrespondent = true
break
}
}
availableCorrespondents := make(map[string]int)
if documentsContainSuggestedCorrespondent {
availableCorrespondents, err = client.GetAllCorrespondents(ctx)
if err != nil {
log.Errorf("Error fetching available correspondents: %v",
err)
return err
}
}
for _, document := range documents {
documentID := document.ID
@ -284,8 +334,6 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum
// remove autoTag to prevent infinite loop - this is required in case of undo
tags = removeTagFromList(tags, autoTag)
// keep previous tags
tags = append(tags, originalTags...)
// remove duplicates
slices.Sort(tags)
tags = slices.Compact(tags)
@ -306,12 +354,27 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum
}
newTags = append(newTags, tagID)
} else {
log.Warnf("Tag '%s' does not exist in paperless-ngx, skipping.", tagName)
log.Errorf("Suggested tag '%s' does not exist in paperless-ngx, skipping.", tagName)
}
}
updatedFields["tags"] = newTags
// Map suggested correspondent names to IDs
if document.SuggestedCorrespondent != "" {
if correspondentID, exists := availableCorrespondents[document.SuggestedCorrespondent]; exists {
updatedFields["correspondent"] = correspondentID
} else {
newCorrespondent := instantiateCorrespondent(document.SuggestedCorrespondent)
newCorrespondentID, err := client.CreateCorrespondent(context.Background(), newCorrespondent)
if err != nil {
log.Errorf("Error creating correspondent with name %s: %v\n", document.SuggestedCorrespondent, err)
return err
}
log.Infof("Created correspondent with name %s and ID %d\n", document.SuggestedCorrespondent, newCorrespondentID)
updatedFields["correspondent"] = newCorrespondentID
}
}
suggestedTitle := document.SuggestedTitle
if len(suggestedTitle) > 128 {
suggestedTitle = suggestedTitle[:128]
@ -341,7 +404,7 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum
// Send the update request using the generic Do method
path := fmt.Sprintf("api/documents/%d/", documentID)
resp, err := c.Do(ctx, "PATCH", path, bytes.NewBuffer(jsonData))
resp, err := client.Do(ctx, "PATCH", path, bytes.NewBuffer(jsonData))
if err != nil {
log.Errorf("Error updating document %d: %v", documentID, err)
return err
@ -399,9 +462,9 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum
// DownloadDocumentAsImages downloads the PDF file of the specified document and converts it to images
// If limitPages > 0, only the first N pages will be processed
func (c *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, documentId int, limitPages int) ([]string, error) {
func (client *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, documentId int, limitPages int) ([]string, error) {
// Create a directory named after the document ID
docDir := filepath.Join(c.GetCacheFolder(), fmt.Sprintf("/document-%d", documentId))
docDir := filepath.Join(client.GetCacheFolder(), fmt.Sprintf("document-%d", documentId))
if _, err := os.Stat(docDir); os.IsNotExist(err) {
err = os.MkdirAll(docDir, 0755)
if err != nil {
@ -429,7 +492,7 @@ func (c *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, document
// Proceed with downloading and converting the document to images
path := fmt.Sprintf("api/documents/%d/download/", documentId)
resp, err := c.Do(ctx, "GET", path, nil)
resp, err := client.Do(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
@ -526,14 +589,97 @@ func (c *PaperlessClient) DownloadDocumentAsImages(ctx context.Context, document
}
// GetCacheFolder returns the cache folder for the PaperlessClient
func (c *PaperlessClient) GetCacheFolder() string {
if c.CacheFolder == "" {
c.CacheFolder = filepath.Join(os.TempDir(), "paperless-gpt")
func (client *PaperlessClient) GetCacheFolder() string {
if client.CacheFolder == "" {
client.CacheFolder = filepath.Join(os.TempDir(), "paperless-gpt")
}
return c.CacheFolder
return client.CacheFolder
}
// urlEncode encodes a string for safe URL usage
func urlEncode(s string) string {
return strings.ReplaceAll(s, " ", "+")
}
// instantiateCorrespondent creates a new Correspondent object with default values
func instantiateCorrespondent(name string) Correspondent {
return Correspondent{
Name: name,
MatchingAlgorithm: 0,
Match: "",
IsInsensitive: true,
Owner: nil,
}
}
// CreateCorrespondent creates a new correspondent in Paperless-NGX
func (client *PaperlessClient) CreateCorrespondent(ctx context.Context, correspondent Correspondent) (int, error) {
url := "api/correspondents/"
// Marshal the correspondent data to JSON
jsonData, err := json.Marshal(correspondent)
if err != nil {
return 0, err
}
// Send the POST request
resp, err := client.Do(ctx, "POST", url, bytes.NewBuffer(jsonData))
if err != nil {
return 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusCreated {
bodyBytes, _ := io.ReadAll(resp.Body)
return 0, fmt.Errorf("error creating correspondent: %d, %s", resp.StatusCode, string(bodyBytes))
}
// Decode the response body to get the ID of the created correspondent
var createdCorrespondent struct {
ID int `json:"id"`
}
err = json.NewDecoder(resp.Body).Decode(&createdCorrespondent)
if err != nil {
return 0, err
}
return createdCorrespondent.ID, nil
}
// CorrespondentResponse represents the response structure for correspondents
type CorrespondentResponse struct {
Results []struct {
ID int `json:"id"`
Name string `json:"name"`
} `json:"results"`
}
// GetAllCorrespondents retrieves all correspondents from the Paperless-NGX API
func (client *PaperlessClient) GetAllCorrespondents(ctx context.Context) (map[string]int, error) {
correspondentIDMapping := make(map[string]int)
path := "api/correspondents/?page_size=9999"
resp, err := client.Do(ctx, "GET", path, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("error fetching correspondents: %d, %s", resp.StatusCode, string(bodyBytes))
}
var correspondentsResponse CorrespondentResponse
err = json.NewDecoder(resp.Body).Decode(&correspondentsResponse)
if err != nil {
return nil, err
}
for _, correspondent := range correspondentsResponse.Results {
correspondentIDMapping[correspondent.Name] = correspondent.ID
}
return correspondentIDMapping, nil
}

53
paperless_test.go
View file

@ -55,6 +55,12 @@ func newTestEnv(t *testing.T) *testEnv {
env.client = NewPaperlessClient(env.server.URL, "test-token")
env.client.HTTPClient = env.server.Client()
// Add mock response for /api/correspondents/
env.setMockResponse("/api/correspondents/", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"results": [{"id": 1, "name": "Alpha"}, {"id": 2, "name": "Beta"}]}`))
})
return env
}
@ -176,7 +182,7 @@ func TestGetDocumentsByTags(t *testing.T) {
documentsResponse := GetDocumentsApiResponse{
Results: []struct {
ID int `json:"id"`
Correspondent interface{} `json:"correspondent"`
Correspondent int `json:"correspondent"`
DocumentType interface{} `json:"document_type"`
StoragePath interface{} `json:"storage_path"`
Title string `json:"title"`
@ -200,16 +206,18 @@ func TestGetDocumentsByTags(t *testing.T) {
} `json:"__search_hit__"`
}{
{
ID: 1,
Title: "Document 1",
Content: "Content 1",
Tags: []int{1, 2},
ID: 1,
Title: "Document 1",
Content: "Content 1",
Tags: []int{1, 2},
Correspondent: 1,
},
{
ID: 2,
Title: "Document 2",
Content: "Content 2",
Tags: []int{2, 3},
ID: 2,
Title: "Document 2",
Content: "Content 2",
Tags: []int{2, 3},
Correspondent: 2,
},
},
}
@ -227,7 +235,7 @@ func TestGetDocumentsByTags(t *testing.T) {
// Set mock responses
env.setMockResponse("/api/documents/", func(w http.ResponseWriter, r *http.Request) {
// Verify query parameters
expectedQuery := "tags__name__iexact=tag1&tags__name__iexact=tag2"
expectedQuery := "tags__name__iexact=tag1&tags__name__iexact=tag2&page_size=25"
assert.Equal(t, expectedQuery, r.URL.RawQuery)
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(documentsResponse)
@ -240,21 +248,23 @@ func TestGetDocumentsByTags(t *testing.T) {
ctx := context.Background()
tags := []string{"tag1", "tag2"}
documents, err := env.client.GetDocumentsByTags(ctx, tags)
documents, err := env.client.GetDocumentsByTags(ctx, tags, 25)
require.NoError(t, err)
expectedDocuments := []Document{
{
ID: 1,
Title: "Document 1",
Content: "Content 1",
Tags: []string{"tag1", "tag2"},
ID: 1,
Title: "Document 1",
Content: "Content 1",
Tags: []string{"tag1", "tag2"},
Correspondent: "Alpha",
},
{
ID: 2,
Title: "Document 2",
Content: "Content 2",
Tags: []string{"tag2", "tag3"},
ID: 2,
Title: "Document 2",
Content: "Content 2",
Tags: []string{"tag2", "tag3"},
Correspondent: "Beta",
},
}
@ -348,7 +358,8 @@ func TestUpdateDocuments(t *testing.T) {
// Expected updated fields
expectedFields := map[string]interface{}{
"title": "New Title",
"tags": []interface{}{float64(idTag1), float64(idTag2), float64(idTag3)}, // keep also previous tags
// do not keep previous tags since the tag generation will already take care to include old ones:
"tags": []interface{}{float64(idTag2), float64(idTag3)},
}
assert.Equal(t, expectedFields, updatedFields)
@ -412,7 +423,7 @@ func TestDownloadDocumentAsImages_ManyPages(t *testing.T) {
ID: 321,
}
// Get sample PDF from tests/pdf/sample.pdf
// Get sample PDF from tests/pdf/many-pages.pdf
pdfFile := "tests/pdf/many-pages.pdf"
pdfContent, err := os.ReadFile(pdfFile)
require.NoError(t, err)

51
types.go
View file

@ -11,7 +11,7 @@ type GetDocumentsApiResponse struct {
All []int `json:"all"`
Results []struct {
ID int `json:"id"`
Correspondent interface{} `json:"correspondent"`
Correspondent int `json:"correspondent"`
DocumentType interface{} `json:"document_type"`
StoragePath interface{} `json:"storage_path"`
Title string `json:"title"`
@ -38,7 +38,7 @@ type GetDocumentsApiResponse struct {
type GetDocumentApiResponse struct {
ID int `json:"id"`
Correspondent interface{} `json:"correspondent"`
Correspondent int `json:"correspondent"`
DocumentType interface{} `json:"document_type"`
StoragePath interface{} `json:"storage_path"`
Title string `json:"title"`
@ -59,25 +59,46 @@ type GetDocumentApiResponse struct {
// Document is a stripped down version of the document object from paperless-ngx.
// Response payload for /documents endpoint and part of request payload for /generate-suggestions endpoint
type Document struct {
ID int `json:"id"`
Title string `json:"title"`
Content string `json:"content"`
Tags []string `json:"tags"`
ID int `json:"id"`
Title string `json:"title"`
Content string `json:"content"`
Tags []string `json:"tags"`
Correspondent string `json:"correspondent"`
}
// GenerateSuggestionsRequest is the request payload for generating suggestions for /generate-suggestions endpoint
type GenerateSuggestionsRequest struct {
Documents []Document `json:"documents"`
GenerateTitles bool `json:"generate_titles,omitempty"`
GenerateTags bool `json:"generate_tags,omitempty"`
Documents []Document `json:"documents"`
GenerateTitles bool `json:"generate_titles,omitempty"`
GenerateTags bool `json:"generate_tags,omitempty"`
GenerateCorrespondents bool `json:"generate_correspondents,omitempty"`
}
// DocumentSuggestion is the response payload for /generate-suggestions endpoint and the request payload for /update-documents endpoint (as an array)
type DocumentSuggestion struct {
ID int `json:"id"`
OriginalDocument Document `json:"original_document"`
SuggestedTitle string `json:"suggested_title,omitempty"`
SuggestedTags []string `json:"suggested_tags,omitempty"`
SuggestedContent string `json:"suggested_content,omitempty"`
RemoveTags []string `json:"remove_tags,omitempty"`
ID int `json:"id"`
OriginalDocument Document `json:"original_document"`
SuggestedTitle string `json:"suggested_title,omitempty"`
SuggestedTags []string `json:"suggested_tags,omitempty"`
SuggestedContent string `json:"suggested_content,omitempty"`
SuggestedCorrespondent string `json:"suggested_correspondent,omitempty"`
RemoveTags []string `json:"remove_tags,omitempty"`
}
type Correspondent struct {
Name string `json:"name"`
MatchingAlgorithm int `json:"matching_algorithm"`
Match string `json:"match"`
IsInsensitive bool `json:"is_insensitive"`
Owner *int `json:"owner"`
SetPermissions struct {
View struct {
Users []int `json:"users"`
Groups []int `json:"groups"`
} `json:"view"`
Change struct {
Users []int `json:"users"`
Groups []int `json:"groups"`
} `json:"change"`
} `json:"set_permissions"`
}

17
web-app/src/DocumentProcessor.tsx
View file

@ -11,12 +11,14 @@ export interface Document {
title: string;
content: string;
tags: string[];
correspondent: string;
}
export interface GenerateSuggestionsRequest {
documents: Document[];
generate_titles?: boolean;
generate_tags?: boolean;
generate_correspondents?: boolean;
}
export interface DocumentSuggestion {
@ -25,6 +27,7 @@ export interface DocumentSuggestion {
suggested_title?: string;
suggested_tags?: string[];
suggested_content?: string;
suggested_correspondent?: string;
}
export interface TagOption {
@ -43,6 +46,7 @@ const DocumentProcessor: React.FC = () => {
const [filterTag, setFilterTag] = useState<string | null>(null);
const [generateTitles, setGenerateTitles] = useState(true);
const [generateTags, setGenerateTags] = useState(true);
const [generateCorrespondents, setGenerateCorrespondents] = useState(true);
const [error, setError] = useState<string | null>(null);
// Custom hook to fetch initial data
@ -81,6 +85,7 @@ const DocumentProcessor: React.FC = () => {
documents,
generate_titles: generateTitles,
generate_tags: generateTags,
generate_correspondents: generateCorrespondents,
};
const { data } = await axios.post<DocumentSuggestion[]>(
@ -137,6 +142,7 @@ const DocumentProcessor: React.FC = () => {
);
};
const handleTitleChange = (docId: number, title: string) => {
setSuggestions((prevSuggestions) =>
prevSuggestions.map((doc) =>
@ -145,6 +151,14 @@ const DocumentProcessor: React.FC = () => {
);
};
const handleCorrespondentChange = (docId: number, correspondent: string) => {
setSuggestions((prevSuggestions) =>
prevSuggestions.map((doc) =>
doc.id === docId ? { ...doc, suggested_correspondent: correspondent } : doc
)
);
}
const resetSuggestions = () => {
setSuggestions([]);
};
@ -214,6 +228,8 @@ const DocumentProcessor: React.FC = () => {
setGenerateTitles={setGenerateTitles}
generateTags={generateTags}
setGenerateTags={setGenerateTags}
generateCorrespondents={generateCorrespondents}
setGenerateCorrespondents={setGenerateCorrespondents}
onProcess={handleProcessDocuments}
processing={processing}
onReload={reloadDocuments}
@ -225,6 +241,7 @@ const DocumentProcessor: React.FC = () => {
onTitleChange={handleTitleChange}
onTagAddition={handleTagAddition}
onTagDeletion={handleTagDeletion}
onCorrespondentChange={handleCorrespondentChange}
onBack={resetSuggestions}
onUpdate={handleUpdateDocuments}
updating={updating}

6
web-app/src/components/DocumentCard.tsx
View file

@ -13,6 +13,9 @@ const DocumentCard: React.FC<DocumentCardProps> = ({ document }) => (
? `${document.content.substring(0, 100)}...`
: document.content}
</p>
<p className="text-sm text-gray-600 dark:text-gray-400 mt-2">
Correspondent: <span className="font-bold text-blue-600 dark:text-blue-400">{document.correspondent}</span>
</p>
<div className="mt-4">
{document.tags.map((tag) => (
<span
@ -27,6 +30,9 @@ const DocumentCard: React.FC<DocumentCardProps> = ({ document }) => (
<div className="text-sm text-white p-2 bg-gray-800 dark:bg-gray-900 rounded-md w-full max-h-full overflow-y-auto">
<h3 className="text-lg font-semibold text-white">{document.title}</h3>
<p className="mt-2 whitespace-pre-wrap">{document.content}</p>
<p className="mt-2">
Correspondent: <span className="font-bold text-blue-400">{document.correspondent}</span>
</p>
<div className="mt-4">
{document.tags.map((tag) => (
<span

13
web-app/src/components/DocumentsToProcess.tsx
View file

@ -9,6 +9,8 @@ interface DocumentsToProcessProps {
setGenerateTitles: React.Dispatch<React.SetStateAction<boolean>>;
generateTags: boolean;
setGenerateTags: React.Dispatch<React.SetStateAction<boolean>>;
generateCorrespondents: boolean;
setGenerateCorrespondents: React.Dispatch<React.SetStateAction<boolean>>;
onProcess: () => void;
processing: boolean;
onReload: () => void;
@ -20,6 +22,8 @@ const DocumentsToProcess: React.FC<DocumentsToProcessProps> = ({
setGenerateTitles,
generateTags,
setGenerateTags,
generateCorrespondents,
setGenerateCorrespondents,
onProcess,
processing,
onReload,
@ -64,6 +68,15 @@ const DocumentsToProcess: React.FC<DocumentsToProcessProps> = ({
/>
<span className="text-gray-700 dark:text-gray-200">Generate Tags</span>
</label>
<label className="flex items-center space-x-2">
<input
type="checkbox"
checked={generateCorrespondents}
onChange={(e) => setGenerateCorrespondents(e.target.checked)}
className="dark:bg-gray-700 dark:border-gray-600"
/>
<span className="text-gray-700 dark:text-gray-200">Generate Correspondents</span>
</label>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 gap-4">

20
web-app/src/components/SuggestionCard.tsx
View file

@ -8,6 +8,7 @@ interface SuggestionCardProps {
onTitleChange: (docId: number, title: string) => void;
onTagAddition: (docId: number, tag: TagOption) => void;
onTagDeletion: (docId: number, index: number) => void;
onCorrespondentChange: (docId: number, correspondent: string) => void;
}
const SuggestionCard: React.FC<SuggestionCardProps> = ({
@ -16,6 +17,7 @@ const SuggestionCard: React.FC<SuggestionCardProps> = ({
onTitleChange,
onTagAddition,
onTagDeletion,
onCorrespondentChange,
}) => {
const sortedAvailableTags = availableTags.sort((a, b) => a.name.localeCompare(b.name));
const document = suggestion.original_document;
@ -49,6 +51,9 @@ const SuggestionCard: React.FC<SuggestionCardProps> = ({
</div>
</div>
<div className="mt-4">
<label className="block text-sm font-medium text-gray-700 dark:text-gray-300">
Suggested Title
</label>
<input
type="text"
value={suggestion.suggested_title || ""}
@ -56,6 +61,9 @@ const SuggestionCard: React.FC<SuggestionCardProps> = ({
className="w-full border border-gray-300 dark:border-gray-600 rounded px-2 py-1 mt-2 focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-gray-200"
/>
<div className="mt-4">
<label className="block text-sm font-medium text-gray-700 dark:text-gray-300">
Suggested Tags
</label>
<ReactTags
selected={
suggestion.suggested_tags?.map((tag, index) => ({
@ -99,6 +107,18 @@ const SuggestionCard: React.FC<SuggestionCardProps> = ({
}}
/>
</div>
<div className="mt-4">
<label className="block text-sm font-medium text-gray-700 dark:text-gray-300">
Suggested Correspondent
</label>
<input
type="text"
value={suggestion.suggested_correspondent || ""}
onChange={(e) => onCorrespondentChange(suggestion.id, e.target.value)}
className="w-full border border-gray-300 dark:border-gray-600 rounded px-2 py-1 mt-2 focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-gray-200"
placeholder="Correspondent"
/>
</div>
</div>
</div>
);

3
web-app/src/components/SuggestionsReview.tsx
View file

@ -8,6 +8,7 @@ interface SuggestionsReviewProps {
onTitleChange: (docId: number, title: string) => void;
onTagAddition: (docId: number, tag: TagOption) => void;
onTagDeletion: (docId: number, index: number) => void;
onCorrespondentChange: (docId: number, correspondent: string) => void;
onBack: () => void;
onUpdate: () => void;
updating: boolean;
@ -19,6 +20,7 @@ const SuggestionsReview: React.FC<SuggestionsReviewProps> = ({
onTitleChange,
onTagAddition,
onTagDeletion,
onCorrespondentChange,
onBack,
onUpdate,
updating,
@ -36,6 +38,7 @@ const SuggestionsReview: React.FC<SuggestionsReviewProps> = ({
onTitleChange={onTitleChange}
onTagAddition={onTagAddition}
onTagDeletion={onTagDeletion}
onCorrespondentChange={onCorrespondentChange}
/>
))}
</div>

2
web-app/tsconfig.app.tsbuildinfo
View file

@ -1 +1 @@
{"root":["./src/app.tsx","./src/documentprocessor.tsx","./src/experimentalocr.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/documentcard.tsx","./src/components/documentstoprocess.tsx","./src/components/nodocuments.tsx","./src/components/successmodal.tsx","./src/components/suggestioncard.tsx","./src/components/suggestionsreview.tsx"],"version":"5.6.2"}
{"root":["./src/app.tsx","./src/documentprocessor.tsx","./src/experimentalocr.tsx","./src/history.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/documentcard.tsx","./src/components/documentstoprocess.tsx","./src/components/nodocuments.tsx","./src/components/sidebar.tsx","./src/components/successmodal.tsx","./src/components/suggestioncard.tsx","./src/components/suggestionsreview.tsx","./src/components/undocard.tsx"],"version":"5.7.2"}

2
web-app/tsconfig.node.tsbuildinfo
View file

@ -1 +1 @@
{"root":["./vite.config.ts"],"version":"5.6.2"}
{"root":["./vite.config.ts"],"version":"5.7.2"}