From b788f091858823e374f882bbcb6b2d9b56ae7295 Mon Sep 17 00:00:00 2001 From: ccrlawrence Date: Fri, 13 Dec 2024 15:48:09 +0000 Subject: [PATCH] UNDO feature - easily track changes (#54) --- .gitignore | 3 +- Dockerfile | 11 +- README.md | 2 + app_http_handlers.go | 89 +++++++- go.mod | 9 +- go.sum | 18 +- local_db.go | 79 +++++++ main.go | 22 +- paperless.go | 101 ++++++++- paperless_test.go | 26 ++- web-app/package-lock.json | 311 +++++++++++++++++++++++++++- web-app/package.json | 8 +- web-app/src/App.tsx | 16 +- web-app/src/DocumentProcessor.tsx | 18 +- web-app/src/History.tsx | 126 +++++++++++ web-app/src/assets/logo.svg | 12 ++ web-app/src/components/Sidebar.css | 75 +++++++ web-app/src/components/Sidebar.tsx | 81 ++++++++ web-app/src/components/UndoCard.tsx | 193 +++++++++++++++++ 19 files changed, 1152 insertions(+), 48 deletions(-) create mode 100644 local_db.go create mode 100644 web-app/src/History.tsx create mode 100644 web-app/src/assets/logo.svg create mode 100644 web-app/src/components/Sidebar.css create mode 100644 web-app/src/components/Sidebar.tsx create mode 100644 web-app/src/components/UndoCard.tsx diff --git a/.gitignore b/.gitignore index 296cc9f..e2c91ba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .DS_Store prompts/ tests/tmp -tmp/ \ No newline at end of file +tmp/ +db/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b83b642..92e32c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,11 +18,14 @@ COPY go.mod go.sum ./ # Download dependencies RUN go mod download -# Copy the rest of the application code -COPY . . +# Pre-compile go-sqlite3 to avoid doing this every time +RUN CGO_ENABLED=1 go build -tags musl -o /dev/null github.com/mattn/go-sqlite3 -# Build the Go binary with the musl build tag -RUN go build -tags musl -o paperless-gpt . +# Now copy the actual source files +COPY *.go . + +# Build the binary using caching for both go modules and build cache +RUN CGO_ENABLED=1 GOMAXPROCS=$(nproc) go build -tags musl -o paperless-gpt . # Stage 2: Build Vite frontend FROM node:20-alpine AS frontend diff --git a/README.md b/README.md index 6b00991..ca10f02 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ services: environment: PAPERLESS_BASE_URL: 'http://paperless-ngx:8000' PAPERLESS_API_TOKEN: 'your_paperless_api_token' + PAPERLESS_PUBLIC_URL: 'http://paperless.mydomain.com' # Optional, your public link to access Paperless LLM_PROVIDER: 'openai' # or 'ollama' LLM_MODEL: 'gpt-4o' # or 'llama2' OPENAI_API_KEY: 'your_openai_api_key' # Required if using OpenAI @@ -137,6 +138,7 @@ If you prefer to run the application manually: |-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|----------| | `PAPERLESS_BASE_URL` | The base URL of your paperless-ngx instance (e.g., `http://paperless-ngx:8000`). | Yes | | `PAPERLESS_API_TOKEN` | API token for accessing paperless-ngx. You can generate one in the paperless-ngx admin interface. | Yes | +| `PAPERLESS_PUBLIC_URL` | The public URL for your Paperless instance, if it is different to your `PAPERLESS_BASE_URL` - say if you are running in Docker Compose | No | | `LLM_PROVIDER` | The LLM provider to use (`openai` or `ollama`). | Yes | | `LLM_MODEL` | The model name to use (e.g., `gpt-4o`, `gpt-3.5-turbo`, `llama2`). | Yes | | `OPENAI_API_KEY` | Your OpenAI API key. Required if using OpenAI as the LLM provider. | Cond. | diff --git a/app_http_handlers.go b/app_http_handlers.go index c9c243f..27ea742 100644 --- a/app_http_handlers.go +++ b/app_http_handlers.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "net/http" "os" @@ -138,7 +139,7 @@ func (app *App) updateDocumentsHandler(c *gin.Context) { return } - err := app.Client.UpdateDocuments(ctx, documents) + err := app.Client.UpdateDocuments(ctx, documents, app.Database, false) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error updating documents: %v", err)}) log.Errorf("Error updating documents: %v", err) @@ -237,8 +238,94 @@ func (app *App) getDocumentHandler() gin.HandlerFunc { document, err := app.Client.GetDocument(c, parsedID) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + log.Errorf("Error fetching document: %v", err) return } c.JSON(http.StatusOK, document) } } + +// Section for local-db actions + +func (app *App) getModificationHistoryHandler(c *gin.Context) { + modifications, err := GetAllModifications(app.Database) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve modification history"}) + log.Errorf("Failed to retrieve modification history: %v", err) + return + } + c.JSON(http.StatusOK, modifications) +} + +func (app *App) undoModificationHandler(c *gin.Context) { + id := c.Param("id") + modID, err := strconv.Atoi(id) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid modification ID"}) + log.Errorf("Invalid modification ID: %v", err) + return + } + + modification, err := GetModification(app.Database, uint(modID)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve modification"}) + log.Errorf("Failed to retrieve modification: %v", err) + return + } + + if modification.Undone { + c.JSON(http.StatusBadRequest, gin.H{"error": "Modification has already been undone"}) + log.Errorf("Modification has already been undone: %v", id) + return + } + + // Ok, we're actually doing the update: + ctx := c.Request.Context() + + // Make the document suggestions for UpdateDocuments + var suggestion DocumentSuggestion + suggestion.ID = int(modification.DocumentID) + suggestion.OriginalDocument, err = app.Client.GetDocument(ctx, int(modification.DocumentID)) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve original document"}) + log.Errorf("Failed to retrieve original document: %v", err) + return + } + switch modification.ModField { + case "title": + suggestion.SuggestedTitle = modification.PreviousValue + case "tags": + var tags []string + err := json.Unmarshal([]byte(modification.PreviousValue), &tags) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to unmarshal previous tags"}) + log.Errorf("Failed to unmarshal previous tags: %v", err) + return + } + suggestion.SuggestedTags = tags + case "content": + suggestion.SuggestedContent = modification.PreviousValue + default: + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid modification field"}) + log.Errorf("Invalid modification field: %v", modification.ModField) + return + } + + // Update the document + err = app.Client.UpdateDocuments(ctx, []DocumentSuggestion{suggestion}, app.Database, true) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update document"}) + log.Errorf("Failed to update document: %v", err) + return + } + + // Successful, so set modification as undone + err = SetModificationUndone(app.Database, modification) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to mark modification as undone"}) + return + } + + // Else all was ok + c.Status(http.StatusOK) +} diff --git a/go.mod b/go.mod index babd910..ddeb457 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/tmc/langchaingo v0.1.12 - golang.org/x/sync v0.7.0 + golang.org/x/sync v0.9.0 + gorm.io/driver/sqlite v1.5.6 + gorm.io/gorm v1.25.12 ) require ( @@ -33,11 +35,14 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jupiterrider/ffi v0.2.0 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-sqlite3 v1.14.24 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -53,7 +58,7 @@ require ( golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/text v0.20.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 76d584c..f914d5b 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,10 @@ github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jupiterrider/ffi v0.2.0 h1:tMM70PexgYNmV+WyaYhJgCvQAvtTCs3wXeILPutihnA= @@ -60,6 +64,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= @@ -119,8 +125,8 @@ golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -139,8 +145,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -157,6 +163,10 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/local_db.go b/local_db.go new file mode 100644 index 0000000..931d7fb --- /dev/null +++ b/local_db.go @@ -0,0 +1,79 @@ +package main + +import ( + "os" + "path/filepath" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// ModificationHistory represents the schema of the modification_history table +type ModificationHistory struct { + ID uint `gorm:"primaryKey"` // Auto-incrementing primary key + DocumentID uint `gorm:"not null"` // Foreign key to documents table (if applicable) + DateChanged string `gorm:"not null"` // Date and time of modification + ModField string `gorm:"size:255;not null"` // Field being modified + PreviousValue string `gorm:"size:1048576"` // Previous value of the field + NewValue string `gorm:"size:1048576"` // New value of the field + Undone bool `gorm:"not null;default:false"` // Whether the modification has been undone + UndoneDate string `gorm:"default:null"` // Date and time of undoing the modification +} + +// InitializeDB initializes the SQLite database and migrates the schema +func InitializeDB() *gorm.DB { + // Ensure db directory exists + dbDir := "db" + if err := os.MkdirAll(dbDir, os.ModePerm); err != nil { + log.Fatalf("Failed to create db directory: %v", err) + } + + dbPath := filepath.Join(dbDir, "modification_history.db") + + // Connect to SQLite database + db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{}) + if err != nil { + log.Fatalf("Failed to connect to database: %v", err) + } + + // Migrate the schema (create the table if it doesn't exist) + err = db.AutoMigrate(&ModificationHistory{}) + if err != nil { + log.Fatalf("Failed to migrate database schema: %v", err) + } + + return db +} + +// InsertModification inserts a new modification record into the database +func InsertModification(db *gorm.DB, record *ModificationHistory) error { + log.Debugf("Passed modification record: %+v", record) + record.DateChanged = time.Now().Format(time.RFC3339) // Set the DateChanged field to the current time + log.Debugf("Inserting modification record: %+v", record) + result := db.Create(&record) // GORM's Create method + log.Debugf("Insertion result: %+v", result) + return result.Error +} + +// GetModification retrieves a modification record by its ID +func GetModification(db *gorm.DB, id uint) (*ModificationHistory, error) { + var record ModificationHistory + result := db.First(&record, id) // GORM's First method retrieves the first record matching the ID + return &record, result.Error +} + +// GetAllModifications retrieves all modification records from the database +func GetAllModifications(db *gorm.DB) ([]ModificationHistory, error) { + var records []ModificationHistory + result := db.Order("date_changed DESC").Find(&records) // GORM's Find method retrieves all records + return records, result.Error +} + +// UndoModification marks a modification record as undone and sets the undo date +func SetModificationUndone(db *gorm.DB, record *ModificationHistory) error { + record.Undone = true + record.UndoneDate = time.Now().Format(time.RFC3339) + result := db.Save(&record) // GORM's Save method + return result.Error +} diff --git a/main.go b/main.go index 5c8d3ef..4448832 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "github.com/tmc/langchaingo/llms" "github.com/tmc/langchaingo/llms/ollama" "github.com/tmc/langchaingo/llms/openai" + "gorm.io/gorm" ) // Global Variables and Constants @@ -73,6 +74,7 @@ Be very selective and only choose the most relevant tags since too many tags wil // App struct to hold dependencies type App struct { Client *PaperlessClient + Database *gorm.DB LLM llms.Model VisionLLM llms.Model } @@ -87,6 +89,9 @@ func main() { // Initialize PaperlessClient client := NewPaperlessClient(paperlessBaseURL, paperlessAPIToken) + // Initialize Database + database := InitializeDB() + // Load Templates loadTemplates() @@ -105,6 +110,7 @@ func main() { // Initialize App with dependencies app := &App{ Client: client, + Database: database, LLM: llm, VisionLLM: visionLlm, } @@ -165,6 +171,20 @@ func main() { enabled := isOcrEnabled() c.JSON(http.StatusOK, gin.H{"enabled": enabled}) }) + + // Local db actions + api.GET("/modifications", app.getModificationHistoryHandler) + api.POST("/undo-modification/:id", app.undoModificationHandler) + + // Get public Paperless environment (as set in environment variables) + api.GET("/paperless-url", func(c *gin.Context) { + baseUrl := os.Getenv("PAPERLESS_PUBLIC_URL") + if baseUrl == "" { + baseUrl = os.Getenv("PAPERLESS_BASE_URL") + } + baseUrl = strings.TrimRight(baseUrl, "/") + c.JSON(http.StatusOK, gin.H{"url": baseUrl}) + }) } // Serve static files for the frontend under /assets @@ -268,7 +288,7 @@ func (app *App) processAutoTagDocuments() (int, error) { return 0, fmt.Errorf("error generating suggestions: %w", err) } - err = app.Client.UpdateDocuments(ctx, suggestions) + err = app.Client.UpdateDocuments(ctx, suggestions, app.Database, false) if err != nil { return 0, fmt.Errorf("error updating documents: %w", err) } diff --git a/paperless.go b/paperless.go index 212e73e..6cff47c 100644 --- a/paperless.go +++ b/paperless.go @@ -11,11 +11,13 @@ import ( "os" "path/filepath" "slices" + "sort" "strings" "sync" "github.com/gen2brain/go-fitz" "golang.org/x/sync/errgroup" + "gorm.io/gorm" ) // PaperlessClient struct to interact with the Paperless-NGX API @@ -26,6 +28,32 @@ type PaperlessClient struct { CacheFolder string } +func hasSameTags(original, suggested []string) bool { + if len(original) != len(suggested) { + return false + } + + // Create copies to avoid modifying original slices + orig := make([]string, len(original)) + sugg := make([]string, len(suggested)) + + copy(orig, original) + copy(sugg, suggested) + + // Sort both slices + sort.Strings(orig) + sort.Strings(sugg) + + // Compare elements + for i := range orig { + if orig[i] != sugg[i] { + return false + } + } + + return true +} + // NewPaperlessClient creates a new instance of PaperlessClient with a default HTTP client func NewPaperlessClient(baseURL, apiToken string) *PaperlessClient { cacheFolder := os.Getenv("PAPERLESS_GPT_CACHE_DIR") @@ -218,7 +246,7 @@ func (c *PaperlessClient) GetDocument(ctx context.Context, documentID int) (Docu } // UpdateDocuments updates the specified documents with suggested changes -func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []DocumentSuggestion) error { +func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []DocumentSuggestion, db *gorm.DB, isUndo bool) error { // Fetch all available tags availableTags, err := c.GetAllTags(ctx) if err != nil { @@ -229,21 +257,43 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum for _, document := range documents { documentID := document.ID + // Original fields will store any updated fields to store records for + originalFields := make(map[string]interface{}) updatedFields := make(map[string]interface{}) newTags := []int{} tags := document.SuggestedTags - if len(tags) == 0 { - tags = document.OriginalDocument.Tags + originalTags := document.OriginalDocument.Tags + + originalTagsJSON, err := json.Marshal(originalTags) + if err != nil { + log.Errorf("Error marshalling JSON for document %d: %v", documentID, err) + return err } + // remove autoTag to prevent infinite loop (even if it is in the original tags) - tags = removeTagFromList(tags, autoTag) + originalTags = removeTagFromList(originalTags, autoTag) + + if len(tags) == 0 { + tags = originalTags + } else { + // We have suggested tags to change + originalFields["tags"] = originalTags + // remove autoTag to prevent infinite loop - this is required in case of undo + tags = removeTagFromList(tags, autoTag) + } + + updatedTagsJSON, err := json.Marshal(tags) + if err != nil { + log.Errorf("Error marshalling JSON for document %d: %v", documentID, err) + return err + } // Map suggested tag names to IDs for _, tagName := range tags { if tagID, exists := availableTags[tagName]; exists { // Skip the tag that we are filtering - if tagName == manualTag { + if !isUndo && tagName == manualTag { continue } newTags = append(newTags, tagID) @@ -259,6 +309,7 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum suggestedTitle = suggestedTitle[:128] } if suggestedTitle != "" { + originalFields["title"] = document.OriginalDocument.Title updatedFields["title"] = suggestedTitle } else { log.Warnf("No valid title found for document %d, skipping.", documentID) @@ -267,8 +318,11 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum // Suggested Content suggestedContent := document.SuggestedContent if suggestedContent != "" { + originalFields["content"] = document.OriginalDocument.Content updatedFields["content"] = suggestedContent } + log.Debugf("Document %d: Original fields: %v", documentID, originalFields) + log.Debugf("Document %d: Updated fields: %v Tags: %v", documentID, updatedFields, tags) // Marshal updated fields to JSON jsonData, err := json.Marshal(updatedFields) @@ -290,6 +344,43 @@ func (c *PaperlessClient) UpdateDocuments(ctx context.Context, documents []Docum bodyBytes, _ := io.ReadAll(resp.Body) log.Errorf("Error updating document %d: %d, %s", documentID, resp.StatusCode, string(bodyBytes)) return fmt.Errorf("error updating document %d: %d, %s", documentID, resp.StatusCode, string(bodyBytes)) + } else { + for field, value := range originalFields { + log.Printf("Document %d: Updated %s from %v to %v", documentID, field, originalFields[field], value) + // Insert the modification record into the database + var modificationRecord ModificationHistory + if field == "tags" { + // Make sure we only store changes where tags are changed - not the same before and after + // And we have to use tags, not updatedFields as they are IDs not fields + if !hasSameTags(document.OriginalDocument.Tags, tags) { + modificationRecord = ModificationHistory{ + DocumentID: uint(documentID), + ModField: field, + PreviousValue: string(originalTagsJSON), + NewValue: string(updatedTagsJSON), + } + } + } else { + // Only store mod if field actually changed + if originalFields[field] != updatedFields[field] { + modificationRecord = ModificationHistory{ + DocumentID: uint(documentID), + ModField: field, + PreviousValue: fmt.Sprintf("%v", originalFields[field]), + NewValue: fmt.Sprintf("%v", updatedFields[field]), + } + } + } + + // Only store if we have a valid modification record + if (modificationRecord != ModificationHistory{}) { + err = InsertModification(db, &modificationRecord) + } + if err != nil { + log.Errorf("Error inserting modification record for document %d: %v", documentID, err) + return err + } + } } log.Printf("Document %d updated successfully.", documentID) diff --git a/paperless_test.go b/paperless_test.go index 17cddde..c75049f 100644 --- a/paperless_test.go +++ b/paperless_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) // Helper struct to hold common test data and methods @@ -22,6 +24,7 @@ type testEnv struct { client *PaperlessClient requestCount int mockResponses map[string]http.HandlerFunc + db *gorm.DB } // newTestEnv initializes a new test environment @@ -31,6 +34,11 @@ func newTestEnv(t *testing.T) *testEnv { mockResponses: make(map[string]http.HandlerFunc), } + // Initialize test database + db, err := InitializeTestDB() + require.NoError(t, err) + env.db = db + // Create a mock server with a handler that dispatches based on URL path env.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { env.requestCount++ @@ -50,6 +58,22 @@ func newTestEnv(t *testing.T) *testEnv { return env } +func InitializeTestDB() (*gorm.DB, error) { + // Use in-memory SQLite for testing + db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) + if err != nil { + return nil, err + } + + // Migrate schema + err = db.AutoMigrate(&ModificationHistory{}) + if err != nil { + return nil, err + } + + return db, nil +} + // teardown closes the mock server func (env *testEnv) teardown() { env.server.Close() @@ -327,7 +351,7 @@ func TestUpdateDocuments(t *testing.T) { }) ctx := context.Background() - err := env.client.UpdateDocuments(ctx, documents) + err := env.client.UpdateDocuments(ctx, documents, env.db, false) require.NoError(t, err) } diff --git a/web-app/package-lock.json b/web-app/package-lock.json index 31ecd9b..a342dba 100644 --- a/web-app/package-lock.json +++ b/web-app/package-lock.json @@ -10,17 +10,23 @@ "dependencies": { "@headlessui/react": "^2.1.8", "@heroicons/react": "^2.1.5", + "@mdi/js": "^7.4.47", + "@mdi/react": "^1.6.1", "axios": "^1.7.7", "classnames": "^2.5.1", + "date-fns": "^4.1.0", "prop-types": "^15.8.1", "react": "^18.3.1", "react-dom": "^18.3.1", "react-icons": "^5.3.0", "react-router-dom": "^6.27.0", - "react-tag-autocomplete": "^7.3.0" + "react-tag-autocomplete": "^7.3.0", + "react-tooltip": "^5.28.0", + "winston": "^3.17.0" }, "devDependencies": { "@eslint/js": "^9.9.0", + "@types/node": "^22.10.1", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react-swc": "^3.5.0", @@ -48,6 +54,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", @@ -723,6 +747,19 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mdi/js": { + "version": "7.4.47", + "resolved": "https://registry.npmjs.org/@mdi/js/-/js-7.4.47.tgz", + "integrity": "sha512-KPnNOtm5i2pMabqZxpUz7iQf+mfrYZyKCZ8QNz85czgEt7cuHcGorWfdzUMWYA0SD+a6Hn4FmJ+YhzzzjkTZrQ==" + }, + "node_modules/@mdi/react": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@mdi/react/-/react-1.6.1.tgz", + "integrity": "sha512-4qZeDcluDFGFTWkHs86VOlHkm6gnKaMql13/gpIcUQ8kzxHgpj31NuCkD8abECVfbULJ3shc7Yt4HJ6Wu6SN4w==", + "dependencies": { + "prop-types": "^15.7.2" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1313,6 +1350,15 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/node": { + "version": "22.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", + "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", + "dev": true, + "dependencies": { + "undici-types": "~6.20.0" + } + }, "node_modules/@types/prop-types": { "version": "15.7.13", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.13.tgz", @@ -1338,6 +1384,11 @@ "@types/react": "*" } }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.6.0.tgz", @@ -1660,6 +1711,11 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1887,6 +1943,15 @@ "node": ">=6" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1902,8 +1967,38 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } }, "node_modules/combined-stream": { "version": "1.0.8", @@ -1963,6 +2058,15 @@ "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "dev": true }, + "node_modules/date-fns": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-4.1.0.tgz", + "integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -2024,6 +2128,11 @@ "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -2305,6 +2414,11 @@ "reusify": "^1.0.4" } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -2364,6 +2478,11 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "node_modules/follow-redirects": { "version": "1.15.9", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", @@ -2577,6 +2696,16 @@ "node": ">=0.8.19" } }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -2652,6 +2781,17 @@ "node": ">=8" } }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -2726,6 +2866,11 @@ "json-buffer": "3.0.1" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2775,6 +2920,22 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -2857,8 +3018,7 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/mz": { "version": "2.7.0", @@ -2936,6 +3096,14 @@ "node": ">= 6" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3362,6 +3530,19 @@ "react": "^18.0.0" } }, + "node_modules/react-tooltip": { + "version": "5.28.0", + "resolved": "https://registry.npmjs.org/react-tooltip/-/react-tooltip-5.28.0.tgz", + "integrity": "sha512-R5cO3JPPXk6FRbBHMO0rI9nkUG/JKfalBSQfZedZYzmqaZQgq7GLzF8vcCWx6IhUCKg0yPqJhXIzmIO5ff15xg==", + "dependencies": { + "@floating-ui/dom": "^1.6.1", + "classnames": "^2.3.0" + }, + "peerDependencies": { + "react": ">=16.14.0", + "react-dom": ">=16.14.0" + } + }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", @@ -3371,6 +3552,19 @@ "pify": "^2.3.0" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -3477,6 +3671,33 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "engines": { + "node": ">=10" + } + }, "node_modules/scheduler": { "version": "0.23.2", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", @@ -3530,6 +3751,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -3539,6 +3768,22 @@ "node": ">=0.10.0" } }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "engines": { + "node": "*" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", @@ -3729,6 +3974,11 @@ "node": ">=14.0.0" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -3768,6 +4018,14 @@ "node": ">=8.0" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/ts-api-utils": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", @@ -3839,6 +4097,12 @@ } } }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true + }, "node_modules/update-browserslist-db": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.0.tgz", @@ -3881,8 +4145,7 @@ "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/vite": { "version": "5.4.7", @@ -3958,6 +4221,40 @@ "node": ">= 8" } }, + "node_modules/winston": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", diff --git a/web-app/package.json b/web-app/package.json index 09b6007..c8cd1ef 100644 --- a/web-app/package.json +++ b/web-app/package.json @@ -13,17 +13,23 @@ "dependencies": { "@headlessui/react": "^2.1.8", "@heroicons/react": "^2.1.5", + "@mdi/js": "^7.4.47", + "@mdi/react": "^1.6.1", "axios": "^1.7.7", "classnames": "^2.5.1", + "date-fns": "^4.1.0", "prop-types": "^15.8.1", "react": "^18.3.1", "react-dom": "^18.3.1", "react-icons": "^5.3.0", "react-router-dom": "^6.27.0", - "react-tag-autocomplete": "^7.3.0" + "react-tag-autocomplete": "^7.3.0", + "react-tooltip": "^5.28.0", + "winston": "^3.17.0" }, "devDependencies": { "@eslint/js": "^9.9.0", + "@types/node": "^22.10.1", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react-swc": "^3.5.0", diff --git a/web-app/src/App.tsx b/web-app/src/App.tsx index 4b89f18..d5a2052 100644 --- a/web-app/src/App.tsx +++ b/web-app/src/App.tsx @@ -1,16 +1,24 @@ // App.tsx or App.jsx import React from 'react'; import { Route, BrowserRouter as Router, Routes } from 'react-router-dom'; +import Sidebar from './components/Sidebar'; import DocumentProcessor from './DocumentProcessor'; import ExperimentalOCR from './ExperimentalOCR'; // New component +import History from './History'; const App: React.FC = () => { return ( - - } /> - } /> - +
+ console.log(page)} /> +
+ + } /> + } /> + } /> + +
+
); }; diff --git a/web-app/src/DocumentProcessor.tsx b/web-app/src/DocumentProcessor.tsx index d83709c..093a5e1 100644 --- a/web-app/src/DocumentProcessor.tsx +++ b/web-app/src/DocumentProcessor.tsx @@ -1,6 +1,5 @@ import axios from "axios"; import React, { useCallback, useEffect, useState } from "react"; -import { Link } from "react-router-dom"; import "react-tag-autocomplete/example/src/styles.css"; // Ensure styles are loaded import DocumentsToProcess from "./components/DocumentsToProcess"; import NoDocuments from "./components/NoDocuments"; @@ -46,22 +45,17 @@ const DocumentProcessor: React.FC = () => { const [generateTags, setGenerateTags] = useState(true); const [error, setError] = useState(null); - // Temporary feature flags - const [ocrEnabled, setOcrEnabled] = useState(false); - // Custom hook to fetch initial data const fetchInitialData = useCallback(async () => { try { - const [filterTagRes, documentsRes, tagsRes, ocrEnabledRes] = await Promise.all([ + const [filterTagRes, documentsRes, tagsRes] = await Promise.all([ axios.get<{ tag: string }>("/api/filter-tag"), axios.get("/api/documents"), axios.get>("/api/tags"), - axios.get<{enabled: boolean}>("/api/experimental/ocr"), ]); setFilterTag(filterTagRes.data.tag); setDocuments(documentsRes.data); - setOcrEnabled(ocrEnabledRes.data.enabled); const tags = Object.keys(tagsRes.data).map((tag) => ({ id: tag, name: tag, @@ -199,16 +193,6 @@ const DocumentProcessor: React.FC = () => {

Paperless GPT

- {ocrEnabled && ( -
- - OCR via LLMs (Experimental) - -
- )}
{error && ( diff --git a/web-app/src/History.tsx b/web-app/src/History.tsx new file mode 100644 index 0000000..e68fb57 --- /dev/null +++ b/web-app/src/History.tsx @@ -0,0 +1,126 @@ +import React, { useEffect, useState } from 'react'; +import UndoCard from './components/UndoCard'; + +interface ModificationHistory { + ID: number; + DocumentID: number; + DateChanged: string; + ModField: string; + PreviousValue: string; + NewValue: string; + Undone: boolean; + UndoneDate: string | null; +} + +const History: React.FC = () => { + const [modifications, setModifications] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [paperlessUrl, setPaperlessUrl] = useState(''); + + // Get Paperless URL + useEffect(() => { + const fetchUrl = async () => { + try { + const response = await fetch('/api/paperless-url'); + if (!response.ok) { + throw new Error('Failed to fetch public URL'); + } + const { url } = await response.json(); + setPaperlessUrl(url); + } catch (err) { + console.error('Error fetching Paperless URL:', err); + } + }; + + fetchUrl(); + }, []); + + // Get all modifications + useEffect(() => { + fetchModifications(); + }, []); + + const fetchModifications = async () => { + try { + const response = await fetch('/api/modifications'); + if (!response.ok) { + throw new Error('Failed to fetch modifications'); + } + const data = await response.json(); + setModifications(data); + } catch (err) { + setError(err instanceof Error ? err.message : 'Unknown error occurred'); + } finally { + setLoading(false); + } + }; + + const handleUndo = async (id: number) => { + try { + const response = await fetch(`/api/undo-modification/${id}`, { + method: 'POST', + }); + + if (!response.ok) { + throw new Error('Failed to undo modification'); + } + + // Use ISO 8601 format for consistency + const now = new Date().toISOString(); + + setModifications(mods => mods.map(mod => + mod.ID === id + ? { ...mod, Undone: true, UndoneDate: now } + : mod + )); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to undo modification'); + } + }; + + if (loading) { + return ( +
+
+
+ ); + } + + if (error) { + return ( +
+ Error: {error} +
+ ); + } + + return ( +
+

+ Modification History +

+
+ Note: when undoing tag changes, this will not re-add 'paperless-gpt-auto' +
+ {modifications.length === 0 ? ( +

+ No modifications found +

+ ) : ( +
+ {modifications.map((modification) => ( + + ))} +
+ )} +
+ ); +}; + +export default History; \ No newline at end of file diff --git a/web-app/src/assets/logo.svg b/web-app/src/assets/logo.svg new file mode 100644 index 0000000..347b1e7 --- /dev/null +++ b/web-app/src/assets/logo.svg @@ -0,0 +1,12 @@ + + + + + + diff --git a/web-app/src/components/Sidebar.css b/web-app/src/components/Sidebar.css new file mode 100644 index 0000000..2459a05 --- /dev/null +++ b/web-app/src/components/Sidebar.css @@ -0,0 +1,75 @@ +.sidebar { + width: 250px; + background-color: #2c3e50; + color: #ecf0f1; + display: flex; + flex-direction: column; + transition: width 0.3s; + } + + .sidebar.collapsed { + width: 60px; + + } + + .sidebar-header { + display: flex; + align-items: center; + padding: 10px; + background-color: #34495e; + justify-content: space-between; + } + + .sidebar-header.collapsed { + justify-content: center; + } + + .logo { + height: 40px; + margin-right: 10px; + } + + .menu-items { + list-style: none; + padding: 0; + margin: 0; + } + + .menu-items li { + padding: 15px 20px; + cursor: pointer; + } + + .menu-items li.active { + background-color: darkslategray; + padding: 15px 20px; + cursor: pointer; + } + + .menu-items li:hover { + background-color: #1abc9c; + } + + .menu-items li a { + text-decoration: none; + color: inherit; + font-size: 18px; + } + + .toggle-btn { + background: none; + border: none; + color: white; + font-size: 24px; + cursor: pointer; + } + + .sidebar.collapsed .menu-items li a { + display: none; + } + + .sidebar.collapsed .logo { + height: 40px; + margin: auto; + } + \ No newline at end of file diff --git a/web-app/src/components/Sidebar.tsx b/web-app/src/components/Sidebar.tsx new file mode 100644 index 0000000..ae17d27 --- /dev/null +++ b/web-app/src/components/Sidebar.tsx @@ -0,0 +1,81 @@ +import axios from "axios"; +import React, { useCallback, useEffect, useState } from 'react'; +import "./Sidebar.css"; +import { Link, useLocation } from 'react-router-dom'; +import { Icon } from '@mdi/react'; +import { mdiHomeOutline, mdiTextBoxSearchOutline, mdiHistory } from '@mdi/js'; +import logo from "../assets/logo.svg"; + + +interface SidebarProps { + onSelectPage: (page: string) => void; +} + +const Sidebar: React.FC = ({ onSelectPage }) => { + const [collapsed, setCollapsed] = useState(false); + const location = useLocation(); + + const toggleSidebar = () => { + setCollapsed(!collapsed); + }; + + const handlePageClick = (page: string) => { + onSelectPage(page); + }; + + // Get whether experimental OCR is enabled + const [ocrEnabled, setOcrEnabled] = useState(false); + const fetchOcrEnabled = useCallback(async () => { + try { + const res = await axios.get<{ enabled: boolean }>("/api/experimental/ocr"); + setOcrEnabled(res.data.enabled); + } catch (err) { + console.error(err); + } + }, []); + + useEffect(() => { + fetchOcrEnabled(); + }, [fetchOcrEnabled]); + + const menuItems = [ + { name: 'home', path: '/', icon: mdiHomeOutline, title: 'Home' }, + { name: 'history', path: '/history', icon: mdiHistory, title: 'History' }, + ]; + + // If OCR is enabled, add the OCR menu item + if (ocrEnabled) { + menuItems.push({ name: 'ocr', path: '/experimental-ocr', icon: mdiTextBoxSearchOutline, title: 'OCR' }); + } + + return ( +
+
+ {!collapsed && Logo} + +
+
    + {menuItems.map((item) => ( +
  • + handlePageClick(item.name)} + style={{ display: 'flex', alignItems: 'center' }} + > + {/* + {!collapsed &&   {item.title}} */} +
    + +
    + {!collapsed && {item.title}} + +
    • + ))} +
+
+ ); +}; + +export default Sidebar; diff --git a/web-app/src/components/UndoCard.tsx b/web-app/src/components/UndoCard.tsx new file mode 100644 index 0000000..d3c2254 --- /dev/null +++ b/web-app/src/components/UndoCard.tsx @@ -0,0 +1,193 @@ +// UndoCard.tsx +import React from 'react'; +import { Tooltip } from 'react-tooltip' + +interface ModificationProps { + ID: number; + DocumentID: number; + DateChanged: string; + ModField: string; + PreviousValue: string; + NewValue: string; + Undone: boolean; + UndoneDate: string | null; + onUndo: (id: number) => void; + paperlessUrl: string; +} + +const formatDate = (dateString: string | null): string => { + if (!dateString) return ''; + + try { + const date = new Date(dateString); + // Check if date is valid + if (isNaN(date.getTime())) { + return 'Invalid date'; + } + return `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}-${String(date.getDate()).padStart(2, '0')} ${String(date.getHours()).padStart(2, '0')}:${String(date.getMinutes()).padStart(2, '0')}`; + } catch { + return 'Invalid date'; + } +}; + +const buildPaperlessUrl = (paperlessUrl: string, documentId: number): string => { + return `${paperlessUrl}/documents/${documentId}/details`; +}; + +const UndoCard: React.FC = ({ + ID, + DocumentID, + DateChanged, + ModField, + PreviousValue, + NewValue, + Undone, + UndoneDate, + onUndo, + paperlessUrl, +}) => { + const formatValue = (value: string, field: string) => { + if (field === 'tags') { + try { + const tags = JSON.parse(value) as string[]; + return ( +
+ {tags.map((tag) => ( + + {tag} + + ))} +
+ ); + } catch { + return value; + } + } else if (field.toLowerCase().includes('date')) { + return formatDate(value); + } + return value; + }; + + return ( +
+
+
{/* Left content */} +
+
+
+ Date Modified +
+
+ {DateChanged && formatDate(DateChanged)} +
+
+ + +
+
+ Modified Field +
+
+ {ModField} +
+
+
+
+
+
+ Previous:   + 100 ? { + 'data-tooltip-id': `tooltip-${ID}-prev` + } : {})} + > + {formatValue(PreviousValue, ModField)} + +
+
+ New:   + 100 ? { + 'data-tooltip-id': `tooltip-${ID}-new` + } : {})} + > + {formatValue(NewValue, ModField)} + +
+
+ + {PreviousValue} + + + {NewValue} + +
+
+
{/* Button content */} + +
+
+
+ ); +}; + +export default UndoCard; \ No newline at end of file