Mirrored-Projects/paperless-gpt
1
0
Fork 0
mirror of https://github.com/icereed/paperless-gpt.git synced 2025-03-13 05:08:01 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

refactor: enhance logging in document suggestion and title generation processes

Browse source
This commit is contained in:
Dominik Schröter 2025-01-13 08:02:18 +01:00
parent 287d0e5c7c
commit d8be5d9db5
3 changed files with 22 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app_http_handlers.go
View file

@ -119,7 +119,7 @@ func (app *App) generateSuggestionsHandler(c *gin.Context) {
return return
} }
results, err := app.generateDocumentSuggestions(ctx, suggestionRequest) results, err := app.generateDocumentSuggestions(ctx, suggestionRequest, log.WithContext(ctx))
if err != nil { if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error processing documents: %v", err)}) c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error processing documents: %v", err)})
log.Errorf("Error processing documents: %v", err) log.Errorf("Error processing documents: %v", err)

30
app_llm.go
View file

@ -16,7 +16,12 @@ import (
) )
// getSuggestedTags generates suggested tags for a document using the LLM // getSuggestedTags generates suggested tags for a document using the LLM
func (app *App) getSuggestedTags(ctx context.Context, content string, suggestedTitle string, availableTags []string) ([]string, error) { func (app *App) getSuggestedTags(
ctx context.Context,
content string,
suggestedTitle string,
availableTags []string,
logger *logrus.Entry) ([]string, error) {
likelyLanguage := getLikelyLanguage() likelyLanguage := getLikelyLanguage()
templateMutex.RLock() templateMutex.RLock()
@ -30,11 +35,12 @@ func (app *App) getSuggestedTags(ctx context.Context, content string, suggestedT
"Content": content, "Content": content,
}) })
if err != nil { if err != nil {
logger.Errorf("Error executing tag template: %v", err)
return nil, fmt.Errorf("error executing tag template: %v", err) return nil, fmt.Errorf("error executing tag template: %v", err)
} }
prompt := promptBuffer.String() prompt := promptBuffer.String()
log.Debugf("Tag suggestion prompt: %s", prompt) logger.Debugf("Tag suggestion prompt: %s", prompt)
completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{ completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{
{ {
@ -47,6 +53,7 @@ func (app *App) getSuggestedTags(ctx context.Context, content string, suggestedT
}, },
}) })
if err != nil { if err != nil {
logger.Errorf("Error getting response from LLM: %v", err)
return nil, fmt.Errorf("error getting response from LLM: %v", err) return nil, fmt.Errorf("error getting response from LLM: %v", err)
} }
@ -130,7 +137,7 @@ func (app *App) doOCRViaLLM(ctx context.Context, jpegBytes []byte, logger *logru
} }
// getSuggestedTitle generates a suggested title for a document using the LLM // getSuggestedTitle generates a suggested title for a document using the LLM
func (app *App) getSuggestedTitle(ctx context.Context, content string) (string, error) { func (app *App) getSuggestedTitle(ctx context.Context, content string, logger *logrus.Entry) (string, error) {
likelyLanguage := getLikelyLanguage() likelyLanguage := getLikelyLanguage()
templateMutex.RLock() templateMutex.RLock()
@ -147,7 +154,7 @@ func (app *App) getSuggestedTitle(ctx context.Context, content string) (string,
prompt := promptBuffer.String() prompt := promptBuffer.String()
log.Debugf("Title suggestion prompt: %s", prompt) logger.Debugf("Title suggestion prompt: %s", prompt)
completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{ completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{
{ {
@ -195,7 +202,8 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
go func(doc Document) { go func(doc Document) {
defer wg.Done() defer wg.Done()
documentID := doc.ID documentID := doc.ID
logger.Printf("Processing Document ID %d...", documentID) docLogger := documentLogger(documentID)
docLogger.Printf("Processing Document ID %d...", documentID)
content := doc.Content content := doc.Content
if len(content) > 5000 { if len(content) > 5000 {
@ -206,18 +214,18 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
var suggestedTags []string var suggestedTags []string
if suggestionRequest.GenerateTitles { if suggestionRequest.GenerateTitles {
suggestedTitle, err = app.getSuggestedTitle(ctx, content) suggestedTitle, err = app.getSuggestedTitle(ctx, content, docLogger)
if err != nil { if err != nil {
mu.Lock() mu.Lock()
errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err))
mu.Unlock() mu.Unlock()
logger.Errorf("Error processing document %d: %v", documentID, err) docLogger.Errorf("Error processing document %d: %v", documentID, err)
return return
} }
} }
if suggestionRequest.GenerateTags { if suggestionRequest.GenerateTags {
suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames) suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, docLogger)
if err != nil { if err != nil {
mu.Lock() mu.Lock()
errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err))
@ -234,7 +242,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
} }
// Titles // Titles
if suggestionRequest.GenerateTitles { if suggestionRequest.GenerateTitles {
logger.Printf("Suggested title for document %d: %s", documentID, suggestedTitle) docLogger.Printf("Suggested title for document %d: %s", documentID, suggestedTitle)
suggestion.SuggestedTitle = suggestedTitle suggestion.SuggestedTitle = suggestedTitle
} else { } else {
suggestion.SuggestedTitle = doc.Title suggestion.SuggestedTitle = doc.Title
@ -242,14 +250,14 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque
// Tags // Tags
if suggestionRequest.GenerateTags { if suggestionRequest.GenerateTags {
logger.Printf("Suggested tags for document %d: %v", documentID, suggestedTags) docLogger.Printf("Suggested tags for document %d: %v", documentID, suggestedTags)
suggestion.SuggestedTags = suggestedTags suggestion.SuggestedTags = suggestedTags
} else { } else {
suggestion.SuggestedTags = removeTagFromList(doc.Tags, manualTag) suggestion.SuggestedTags = removeTagFromList(doc.Tags, manualTag)
} }
documentSuggestions = append(documentSuggestions, suggestion) documentSuggestions = append(documentSuggestions, suggestion)
mu.Unlock() mu.Unlock()
logger.Printf("Document %d processed successfully.", documentID) docLogger.Printf("Document %d processed successfully.", documentID)
}(documents[i]) }(documents[i])
} }

4
main.go
View file

@ -362,7 +362,7 @@ func (app *App) processAutoTagDocuments() (int, error) {
return 0, fmt.Errorf("error generating suggestions for document %d: %w", documents[0].ID, err) return 0, fmt.Errorf("error generating suggestions for document %d: %w", documents[0].ID, err)
} }
err = app.Client.UpdateDocuments(ctx, suggestions, app.Database, docLogger) err = app.Client.UpdateDocuments(ctx, suggestions, app.Database, false)
if err != nil { if err != nil {
return 0, fmt.Errorf("error updating document %d: %w", documents[0].ID, err) return 0, fmt.Errorf("error updating document %d: %w", documents[0].ID, err)
} }
@ -403,7 +403,7 @@ func (app *App) processAutoOcrTagDocuments() (int, error) {
OriginalDocument: documents[0], OriginalDocument: documents[0],
SuggestedContent: ocrContent, SuggestedContent: ocrContent,
}, },
}, app.Database, docLogger) }, app.Database, false)
if err != nil { if err != nil {
return 0, fmt.Errorf("error updating document %d after OCR: %w", documents[0].ID, err) return 0, fmt.Errorf("error updating document %d after OCR: %w", documents[0].ID, err)
} }