From d8be5d9db557428a003f737132c1adbcb0999905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Schr=C3=B6ter?= Date: Mon, 13 Jan 2025 08:02:18 +0100 Subject: [PATCH] refactor: enhance logging in document suggestion and title generation processes --- app_http_handlers.go | 2 +- app_llm.go | 30 +++++++++++++++++++----------- main.go | 4 ++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/app_http_handlers.go b/app_http_handlers.go index 27ea742..909edbd 100644 --- a/app_http_handlers.go +++ b/app_http_handlers.go @@ -119,7 +119,7 @@ func (app *App) generateSuggestionsHandler(c *gin.Context) { return } - results, err := app.generateDocumentSuggestions(ctx, suggestionRequest) + results, err := app.generateDocumentSuggestions(ctx, suggestionRequest, log.WithContext(ctx)) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error processing documents: %v", err)}) log.Errorf("Error processing documents: %v", err) diff --git a/app_llm.go b/app_llm.go index 4f0bb27..6e0dc2c 100644 --- a/app_llm.go +++ b/app_llm.go @@ -16,7 +16,12 @@ import ( ) // getSuggestedTags generates suggested tags for a document using the LLM -func (app *App) getSuggestedTags(ctx context.Context, content string, suggestedTitle string, availableTags []string) ([]string, error) { +func (app *App) getSuggestedTags( + ctx context.Context, + content string, + suggestedTitle string, + availableTags []string, + logger *logrus.Entry) ([]string, error) { likelyLanguage := getLikelyLanguage() templateMutex.RLock() @@ -30,11 +35,12 @@ func (app *App) getSuggestedTags(ctx context.Context, content string, suggestedT "Content": content, }) if err != nil { + logger.Errorf("Error executing tag template: %v", err) return nil, fmt.Errorf("error executing tag template: %v", err) } prompt := promptBuffer.String() - log.Debugf("Tag suggestion prompt: %s", prompt) + logger.Debugf("Tag suggestion prompt: %s", prompt) completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{ { @@ -47,6 +53,7 @@ func (app *App) getSuggestedTags(ctx context.Context, content string, suggestedT }, }) if err != nil { + logger.Errorf("Error getting response from LLM: %v", err) return nil, fmt.Errorf("error getting response from LLM: %v", err) } @@ -130,7 +137,7 @@ func (app *App) doOCRViaLLM(ctx context.Context, jpegBytes []byte, logger *logru } // getSuggestedTitle generates a suggested title for a document using the LLM -func (app *App) getSuggestedTitle(ctx context.Context, content string) (string, error) { +func (app *App) getSuggestedTitle(ctx context.Context, content string, logger *logrus.Entry) (string, error) { likelyLanguage := getLikelyLanguage() templateMutex.RLock() @@ -147,7 +154,7 @@ func (app *App) getSuggestedTitle(ctx context.Context, content string) (string, prompt := promptBuffer.String() - log.Debugf("Title suggestion prompt: %s", prompt) + logger.Debugf("Title suggestion prompt: %s", prompt) completion, err := app.LLM.GenerateContent(ctx, []llms.MessageContent{ { @@ -195,7 +202,8 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque go func(doc Document) { defer wg.Done() documentID := doc.ID - logger.Printf("Processing Document ID %d...", documentID) + docLogger := documentLogger(documentID) + docLogger.Printf("Processing Document ID %d...", documentID) content := doc.Content if len(content) > 5000 { @@ -206,18 +214,18 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque var suggestedTags []string if suggestionRequest.GenerateTitles { - suggestedTitle, err = app.getSuggestedTitle(ctx, content) + suggestedTitle, err = app.getSuggestedTitle(ctx, content, docLogger) if err != nil { mu.Lock() errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) mu.Unlock() - logger.Errorf("Error processing document %d: %v", documentID, err) + docLogger.Errorf("Error processing document %d: %v", documentID, err) return } } if suggestionRequest.GenerateTags { - suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames) + suggestedTags, err = app.getSuggestedTags(ctx, content, suggestedTitle, availableTagNames, docLogger) if err != nil { mu.Lock() errorsList = append(errorsList, fmt.Errorf("Document %d: %v", documentID, err)) @@ -234,7 +242,7 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque } // Titles if suggestionRequest.GenerateTitles { - logger.Printf("Suggested title for document %d: %s", documentID, suggestedTitle) + docLogger.Printf("Suggested title for document %d: %s", documentID, suggestedTitle) suggestion.SuggestedTitle = suggestedTitle } else { suggestion.SuggestedTitle = doc.Title @@ -242,14 +250,14 @@ func (app *App) generateDocumentSuggestions(ctx context.Context, suggestionReque // Tags if suggestionRequest.GenerateTags { - logger.Printf("Suggested tags for document %d: %v", documentID, suggestedTags) + docLogger.Printf("Suggested tags for document %d: %v", documentID, suggestedTags) suggestion.SuggestedTags = suggestedTags } else { suggestion.SuggestedTags = removeTagFromList(doc.Tags, manualTag) } documentSuggestions = append(documentSuggestions, suggestion) mu.Unlock() - logger.Printf("Document %d processed successfully.", documentID) + docLogger.Printf("Document %d processed successfully.", documentID) }(documents[i]) } diff --git a/main.go b/main.go index 22af13f..9b2b134 100644 --- a/main.go +++ b/main.go @@ -362,7 +362,7 @@ func (app *App) processAutoTagDocuments() (int, error) { return 0, fmt.Errorf("error generating suggestions for document %d: %w", documents[0].ID, err) } - err = app.Client.UpdateDocuments(ctx, suggestions, app.Database, docLogger) + err = app.Client.UpdateDocuments(ctx, suggestions, app.Database, false) if err != nil { return 0, fmt.Errorf("error updating document %d: %w", documents[0].ID, err) } @@ -403,7 +403,7 @@ func (app *App) processAutoOcrTagDocuments() (int, error) { OriginalDocument: documents[0], SuggestedContent: ocrContent, }, - }, app.Database, docLogger) + }, app.Database, false) if err != nil { return 0, fmt.Errorf("error updating document %d after OCR: %w", documents[0].ID, err) }