diff --git a/.gitignore b/.gitignore index 3323b34..f679422 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env .DS_Store +prompts/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 13b8366..b4b81ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ RUN go mod download COPY . . # Build the Go binary -RUN CGO_ENABLED=0 GOOS=linux go build -o paperless-gpt main.go +RUN CGO_ENABLED=0 GOOS=linux go build -o paperless-gpt . # Stage 2: Build Vite frontend FROM node:20 AS frontend diff --git a/README.md b/README.md index 81c3fdb..fa030b4 100644 --- a/README.md +++ b/README.md @@ -5,15 +5,16 @@ ![Screenshot](./paperless-gpt-screenshot.png) -**paperless-gpt** is a tool designed to generate accurate and meaningful document titles for [paperless-ngx](https://github.com/paperless-ngx/paperless-ngx) using Large Language Models (LLMs). It supports multiple LLM providers, including **OpenAI** and **Ollama**. With paperless-gpt, you can streamline your document management by automatically suggesting appropriate titles and tags based on the content of your scanned documents. +**paperless-gpt** is a tool designed to generate accurate and meaningful document titles and tags for [paperless-ngx](https://github.com/paperless-ngx/paperless-ngx) using Large Language Models (LLMs). It supports multiple LLM providers, including **OpenAI** and **Ollama**. With paperless-gpt, you can streamline your document management by automatically suggesting appropriate titles and tags based on the content of your scanned documents. [![Demo](./demo.gif)](./demo.gif) ## Features -- **Multiple LLM Support**: Choose between OpenAI and Ollama for generating document titles. +- **Multiple LLM Support**: Choose between OpenAI and Ollama for generating document titles and tags. +- **Customizable Prompts**: Modify the prompt templates to suit your specific needs. - **Easy Integration**: Works seamlessly with your existing paperless-ngx setup. -- **User-Friendly Interface**: Intuitive web interface for reviewing and applying suggested titles. +- **User-Friendly Interface**: Intuitive web interface for reviewing and applying suggested titles and tags. - **Dockerized Deployment**: Simple setup using Docker and Docker Compose. ## Table of Contents @@ -28,6 +29,11 @@ - [Manual Setup](#manual-setup) - [Configuration](#configuration) - [Environment Variables](#environment-variables) + - [Custom Prompt Templates](#custom-prompt-templates) + - [Prompt Templates Directory](#prompt-templates-directory) + - [Mounting the Prompts Directory](#mounting-the-prompts-directory) + - [Editing the Prompt Templates](#editing-the-prompt-templates) + - [Template Syntax and Variables](#template-syntax-and-variables) - [Usage](#usage) - [Contributing](#contributing) - [License](#license) @@ -64,7 +70,9 @@ services: LLM_MODEL: 'gpt-4o' # or 'llama2' OPENAI_API_KEY: 'your_openai_api_key' # Required if using OpenAI LLM_LANGUAGE: 'English' # Optional, default is 'English' - OLLAMA_HOST: http://host.docker.internal:11434 # Useful if using Ollama + OLLAMA_HOST: 'http://host.docker.internal:11434' # If using Ollama + volumes: + - ./prompts:/app/prompts # Mount the prompts directory ports: - '8080:8080' depends_on: @@ -84,13 +92,19 @@ If you prefer to run the application manually: cd paperless-gpt ``` -2. **Build the Docker Image:** +2. **Create a `prompts` Directory:** + + ```bash + mkdir prompts + ``` + +3. **Build the Docker Image:** ```bash docker build -t paperless-gpt . ``` -3. **Run the Container:** +4. **Run the Container:** ```bash docker run -d \ @@ -100,6 +114,7 @@ If you prefer to run the application manually: -e LLM_MODEL='gpt-4o' \ -e OPENAI_API_KEY='your_openai_api_key' \ -e LLM_LANGUAGE='English' \ + -v $(pwd)/prompts:/app/prompts \ # Mount the prompts directory -p 8080:8080 \ paperless-gpt ``` @@ -108,18 +123,118 @@ If you prefer to run the application manually: ### Environment Variables -| Variable | Description | Required | -|-----------------------|-----------------------------------------------------------------------------------------------------|----------| -| `PAPERLESS_BASE_URL` | The base URL of your paperless-ngx instance (e.g., `http://paperless-ngx:8000`). | Yes | -| `PAPERLESS_API_TOKEN` | API token for accessing paperless-ngx. You can generate one in the paperless-ngx admin interface. | Yes | -| `LLM_PROVIDER` | The LLM provider to use (`openai` or `ollama`). | Yes | -| `LLM_MODEL` | The model name to use (e.g., `gpt-4`, `gpt-3.5-turbo`, `llama2`). | Yes | -| `OPENAI_API_KEY` | Your OpenAI API key. Required if using OpenAI as the LLM provider. | Cond. | -| `LLM_LANGUAGE` | The likely language of your documents (e.g., `English`, `German`). Default is `English`. | No | -| `OLLAMA_HOST` | The URL of the Ollama server (e.g., `http://host.docker.internal:11434`). Useful if using Ollama. Default is `http://127.0.0.1:11434` | No | +| Variable | Description | Required | +|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| `PAPERLESS_BASE_URL` | The base URL of your paperless-ngx instance (e.g., `http://paperless-ngx:8000`). | Yes | +| `PAPERLESS_API_TOKEN` | API token for accessing paperless-ngx. You can generate one in the paperless-ngx admin interface. | Yes | +| `LLM_PROVIDER` | The LLM provider to use (`openai` or `ollama`). | Yes | +| `LLM_MODEL` | The model name to use (e.g., `gpt-4o`, `gpt-3.5-turbo`, `llama2`). | Yes | +| `OPENAI_API_KEY` | Your OpenAI API key. Required if using OpenAI as the LLM provider. | Cond. | +| `LLM_LANGUAGE` | The likely language of your documents (e.g., `English`, `German`). Default is `English`. | No | +| `OLLAMA_HOST` | The URL of the Ollama server (e.g., `http://host.docker.internal:11434`). Useful if using Ollama. Default is `http://127.0.0.1:11434`. | No | **Note:** When using Ollama, ensure that the Ollama server is running and accessible from the paperless-gpt container. +### Custom Prompt Templates + +You can customize the prompt templates used by paperless-gpt to generate titles and tags. By default, the application uses built-in templates, but you can modify them by editing the template files. + +#### Prompt Templates Directory + +The prompt templates are stored in the `prompts` directory inside the application. The two main template files are: + +- `title_prompt.tmpl`: Template used for generating document titles. +- `tag_prompt.tmpl`: Template used for generating document tags. + +#### Mounting the Prompts Directory + +To modify the prompt templates, you need to mount a local `prompts` directory into the container. + +**Docker Compose Example:** + +```yaml +services: + paperless-gpt: + image: icereed/paperless-gpt:latest + # ... (other configurations) + volumes: + - ./prompts:/app/prompts # Mount the prompts directory +``` + +**Docker Run Command Example:** + +```bash +docker run -d \ + # ... (other configurations) + -v $(pwd)/prompts:/app/prompts \ + paperless-gpt +``` + +#### Editing the Prompt Templates + +1. **Start the Container:** + + When you first start the container with the `prompts` directory mounted, it will automatically create the default template files in your local `prompts` directory if they do not exist. + +2. **Edit the Template Files:** + + - Open `prompts/title_prompt.tmpl` and `prompts/tag_prompt.tmpl` with your favorite text editor. + - Modify the templates using Go's `text/template` syntax. + - Save the changes. + +3. **Restart the Container (if necessary):** + + The application automatically reloads the templates when it starts. If the container is already running, you may need to restart it to apply the changes. + +#### Template Syntax and Variables + +The templates use Go's `text/template` syntax and have access to the following variables: + +- **For `title_prompt.tmpl`:** + + - `{{.Language}}`: The language specified in `LLM_LANGUAGE` (default is `English`). + - `{{.Content}}`: The content of the document. + +- **For `tag_prompt.tmpl`:** + + - `{{.Language}}`: The language specified in `LLM_LANGUAGE`. + - `{{.AvailableTags}}`: A list (array) of available tags from paperless-ngx. + - `{{.Title}}`: The suggested title for the document. + - `{{.Content}}`: The content of the document. + +**Example `title_prompt.tmpl`:** + +```text +I will provide you with the content of a document that has been partially read by OCR (so it may contain errors). +Your task is to find a suitable document title that I can use as the title in the paperless-ngx program. +Respond only with the title, without any additional information. The content is likely in {{.Language}}. + +Be sure to add one fitting emoji at the beginning of the title to make it more visually appealing. + +Content: +{{.Content}} +``` + +**Example `tag_prompt.tmpl`:** + +```text +I will provide you with the content and the title of a document. Your task is to select appropriate tags for the document from the list of available tags I will provide. Only select tags from the provided list. Respond only with the selected tags as a comma-separated list, without any additional information. The content is likely in {{.Language}}. + +Available Tags: +{{.AvailableTags | join ","}} + +Title: +{{.Title}} + +Content: +{{.Content}} + +Please concisely select the {{.Language}} tags from the list above that best describe the document. +Be very selective and only choose the most relevant tags since too many tags will make the document less discoverable. +``` + +**Note:** Advanced users can utilize additional functions from the [Sprig](http://masterminds.github.io/sprig/) template library, as it is included in the application. + ## Usage 1. **Tag Documents in paperless-ngx:** diff --git a/go.mod b/go.mod index 25ab1c8..698838e 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,14 @@ go 1.22.0 toolchain go1.22.2 require ( + github.com/Masterminds/sprig/v3 v3.2.3 github.com/gin-gonic/gin v1.10.0 github.com/tmc/langchaingo v0.1.12 ) require ( + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect @@ -22,14 +25,20 @@ require ( github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/huandu/xstrings v1.3.3 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkoukk/tiktoken-go v0.1.6 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect diff --git a/go.sum b/go.sum index b313bca..39f6dd5 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -30,8 +36,14 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -42,6 +54,10 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -53,11 +69,17 @@ github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAc github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -71,26 +93,57 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/main.go b/main.go index c408c64..387c911 100644 --- a/main.go +++ b/main.go @@ -9,72 +9,18 @@ import ( "log" "net/http" "os" + "path/filepath" "strings" "sync" - "time" + "text/template" + "github.com/Masterminds/sprig/v3" "github.com/gin-gonic/gin" "github.com/tmc/langchaingo/llms" "github.com/tmc/langchaingo/llms/ollama" "github.com/tmc/langchaingo/llms/openai" ) -type GetDocumentsApiResponse struct { - Count int `json:"count"` - Next interface{} `json:"next"` - Previous interface{} `json:"previous"` - All []int `json:"all"` - Results []struct { - ID int `json:"id"` - Correspondent interface{} `json:"correspondent"` - DocumentType interface{} `json:"document_type"` - StoragePath interface{} `json:"storage_path"` - Title string `json:"title"` - Content string `json:"content"` - Tags []int `json:"tags"` - Created time.Time `json:"created"` - CreatedDate string `json:"created_date"` - Modified time.Time `json:"modified"` - Added time.Time `json:"added"` - ArchiveSerialNumber interface{} `json:"archive_serial_number"` - OriginalFileName string `json:"original_file_name"` - ArchivedFileName string `json:"archived_file_name"` - Owner int `json:"owner"` - UserCanChange bool `json:"user_can_change"` - Notes []interface{} `json:"notes"` - SearchHit struct { - Score float64 `json:"score"` - Highlights string `json:"highlights"` - NoteHighlights string `json:"note_highlights"` - Rank int `json:"rank"` - } `json:"__search_hit__"` - } `json:"results"` -} - -// Document is a stripped down version of the document object from paperless-ngx. -// Response payload for /documents endpoint and part of request payload for /generate-suggestions endpoint -type Document struct { - ID int `json:"id"` - Title string `json:"title"` - Content string `json:"content"` - Tags []string `json:"tags"` -} - -// GenerateSuggestionsRequest is the request payload for generating suggestions for /generate-suggestions endpoint -type GenerateSuggestionsRequest struct { - Documents []Document `json:"documents"` - GenerateTitles bool `json:"generate_titles,omitempty"` - GenerateTags bool `json:"generate_tags,omitempty"` -} - -// DocumentSuggestion is the response payload for /generate-suggestions endpoint and the request payload for /update-documents endpoint (as an array) -type DocumentSuggestion struct { - ID int `json:"id"` - OriginalDocument Document `json:"original_document"` - SuggestedTitle string `json:"suggested_title,omitempty"` - SuggestedTags []string `json:"suggested_tags,omitempty"` -} - var ( paperlessBaseURL = os.Getenv("PAPERLESS_BASE_URL") paperlessAPIToken = os.Getenv("PAPERLESS_API_TOKEN") @@ -82,6 +28,35 @@ var ( tagToFilter = "paperless-gpt" llmProvider = os.Getenv("LLM_PROVIDER") llmModel = os.Getenv("LLM_MODEL") + + // Templates + titleTemplate *template.Template + tagTemplate *template.Template + templateMutex sync.RWMutex + + // Default templates + defaultTitleTemplate = `I will provide you with the content of a document that has been partially read by OCR (so it may contain errors). +Your task is to find a suitable document title that I can use as the title in the paperless-ngx program. +Respond only with the title, without any additional information. The content is likely in {{.Language}}. + +Content: +{{.Content}} +` + + defaultTagTemplate = `I will provide you with the content and the title of a document. Your task is to select appropriate tags for the document from the list of available tags I will provide. Only select tags from the provided list. Respond only with the selected tags as a comma-separated list, without any additional information. The content is likely in {{.Language}}. + +Available Tags: +{{.AvailableTags | join ", "}} + +Title: +{{.Title}} + +Content: +{{.Content}} + +Please concisely select the {{.Language}} tags from the list above that best describe the document. +Be very selective and only choose the most relevant tags since too many tags will make the document less discoverable. +` ) func main() { @@ -97,6 +72,8 @@ func main() { log.Fatal("Please set the OPENAI_API_KEY environment variable for OpenAI provider.") } + loadTemplates() + // Create a Gin router with default middleware (logger and recovery) router := gin.Default() @@ -122,6 +99,8 @@ func main() { c.JSON(http.StatusOK, tags) }) + api.GET("/prompts", getPromptsHandler) + api.POST("/prompts", updatePromptsHandler) } // Serve static files for the frontend under /static @@ -139,6 +118,113 @@ func main() { } } +func getPromptsHandler(c *gin.Context) { + templateMutex.RLock() + defer templateMutex.RUnlock() + + // Read the templates from files or use default content + titleTemplateContent, err := os.ReadFile("title_prompt.tmpl") + if err != nil { + titleTemplateContent = []byte(defaultTitleTemplate) + } + + tagTemplateContent, err := os.ReadFile("tag_prompt.tmpl") + if err != nil { + tagTemplateContent = []byte(defaultTagTemplate) + } + + c.JSON(http.StatusOK, gin.H{ + "title_template": string(titleTemplateContent), + "tag_template": string(tagTemplateContent), + }) +} + +func updatePromptsHandler(c *gin.Context) { + var req struct { + TitleTemplate string `json:"title_template"` + TagTemplate string `json:"tag_template"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request payload"}) + return + } + + templateMutex.Lock() + defer templateMutex.Unlock() + + // Update title template + if req.TitleTemplate != "" { + t, err := template.New("title").Parse(req.TitleTemplate) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid title template: %v", err)}) + return + } + titleTemplate = t + err = os.WriteFile("title_prompt.tmpl", []byte(req.TitleTemplate), 0644) + if err != nil { + log.Printf("Failed to write title_prompt.tmpl: %v", err) + } + } + + // Update tag template + if req.TagTemplate != "" { + t, err := template.New("tag").Parse(req.TagTemplate) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid tag template: %v", err)}) + return + } + tagTemplate = t + err = os.WriteFile("tag_prompt.tmpl", []byte(req.TagTemplate), 0644) + if err != nil { + log.Printf("Failed to write tag_prompt.tmpl: %v", err) + } + } + + c.Status(http.StatusOK) +} + +func loadTemplates() { + templateMutex.Lock() + defer templateMutex.Unlock() + + // Ensure prompts directory exists + promptsDir := "prompts" + if err := os.MkdirAll(promptsDir, os.ModePerm); err != nil { + log.Fatalf("Failed to create prompts directory: %v", err) + } + + // Load title template + titleTemplatePath := filepath.Join(promptsDir, "title_prompt.tmpl") + titleTemplateContent, err := os.ReadFile(titleTemplatePath) + if err != nil { + log.Printf("Could not read %s, using default template: %v", titleTemplatePath, err) + titleTemplateContent = []byte(defaultTitleTemplate) + if err := os.WriteFile(titleTemplatePath, titleTemplateContent, os.ModePerm); err != nil { + log.Fatalf("Failed to write default title template to disk: %v", err) + } + } + titleTemplate, err = template.New("title").Funcs(sprig.FuncMap()).Parse(string(titleTemplateContent)) + if err != nil { + log.Fatalf("Failed to parse title template: %v", err) + } + + // Load tag template + tagTemplatePath := filepath.Join(promptsDir, "tag_prompt.tmpl") + tagTemplateContent, err := os.ReadFile(tagTemplatePath) + if err != nil { + log.Printf("Could not read %s, using default template: %v", tagTemplatePath, err) + tagTemplateContent = []byte(defaultTagTemplate) + if err := os.WriteFile(tagTemplatePath, tagTemplateContent, os.ModePerm); err != nil { + log.Fatalf("Failed to write default tag template to disk: %v", err) + } + } + tagTemplate, err = template.New("tag").Funcs(sprig.FuncMap()).Parse(string(tagTemplateContent)) + if err != nil { + log.Fatalf("Failed to parse tag template: %v", err) + } +} + // createLLM creates the appropriate LLM client based on the provider func createLLM() (llms.Model, error) { switch strings.ToLower(llmProvider) { @@ -165,40 +251,47 @@ func createLLM() (llms.Model, error) { } func getAllTags(ctx context.Context, baseURL, apiToken string) (map[string]int, error) { + tagIDMapping := make(map[string]int) url := fmt.Sprintf("%s/api/tags/", baseURL) - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", fmt.Sprintf("Token %s", apiToken)) client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - bodyBytes, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("Error fetching tags: %d, %s", resp.StatusCode, string(bodyBytes)) - } + for url != "" { + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Token %s", apiToken)) - var tagsResponse struct { - Results []struct { - ID int `json:"id"` - Name string `json:"name"` - } `json:"results"` - } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() - err = json.NewDecoder(resp.Body).Decode(&tagsResponse) - if err != nil { - return nil, err - } + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("Error fetching tags: %d, %s", resp.StatusCode, string(bodyBytes)) + } - tagIDMapping := make(map[string]int) - for _, tag := range tagsResponse.Results { - tagIDMapping[tag.Name] = tag.ID + var tagsResponse struct { + Results []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"results"` + Next string `json:"next"` + } + + err = json.NewDecoder(resp.Body).Decode(&tagsResponse) + if err != nil { + return nil, err + } + + for _, tag := range tagsResponse.Results { + tagIDMapping[tag.Name] = tag.ID + } + + url = tagsResponse.Next } return tagIDMapping, nil @@ -430,20 +523,22 @@ func removeTagFromList(tags []string, tagToRemove string) []string { func getSuggestedTags(ctx context.Context, llm llms.Model, content string, suggestedTitle string, availableTags []string) ([]string, error) { likelyLanguage := getLikelyLanguage() - prompt := fmt.Sprintf(`I will provide you with the content and the title of a document. Your task is to select appropriate tags for the document from the list of available tags I will provide. Only select tags from the provided list. Respond only with the selected tags as a comma-separated list, without any additional information. The content is likely in %s. + templateMutex.RLock() + defer templateMutex.RUnlock() -Available Tags: -%s + var promptBuffer bytes.Buffer + err := tagTemplate.Execute(&promptBuffer, map[string]interface{}{ + "Language": likelyLanguage, + "AvailableTags": availableTags, + "Title": suggestedTitle, + "Content": content, + }) + if err != nil { + return nil, fmt.Errorf("error executing tag template: %v", err) + } -Title: -%s - -Content: -%s - -Please concisely select the %s tags from the list above that best describe the document. -Be very selective and only choose the most relevant tags since too many tags will make the document less discoverable. -`, likelyLanguage, strings.Join(availableTags, ", "), suggestedTitle, content, likelyLanguage) + prompt := promptBuffer.String() + log.Printf("Tag suggestion prompt: %s", prompt) completion, err := llm.GenerateContent(ctx, []llms.MessageContent{ { @@ -456,7 +551,7 @@ Be very selective and only choose the most relevant tags since too many tags wil }, }) if err != nil { - return nil, fmt.Errorf("Error getting response from LLM: %v", err) + return nil, fmt.Errorf("error getting response from LLM: %v", err) } response := strings.TrimSpace(completion.Choices[0].Content) @@ -490,13 +585,22 @@ func getLikelyLanguage() string { func getSuggestedTitle(ctx context.Context, llm llms.Model, content string) (string, error) { likelyLanguage := getLikelyLanguage() - prompt := fmt.Sprintf(`I will provide you with the content of a document that has been partially read by OCR (so it may contain errors). -Your task is to find a suitable document title that I can use as the title in the paperless-ngx program. -Respond only with the title, without any additional information. The content is likely in %s. + templateMutex.RLock() + defer templateMutex.RUnlock() + + var promptBuffer bytes.Buffer + err := titleTemplate.Execute(&promptBuffer, map[string]interface{}{ + "Language": likelyLanguage, + "Content": content, + }) + if err != nil { + return "", fmt.Errorf("error executing title template: %v", err) + } + + prompt := promptBuffer.String() + + log.Printf("Title suggestion prompt: %s", prompt) -Content: -%s -`, likelyLanguage, content) completion, err := llm.GenerateContent(ctx, []llms.MessageContent{ { Parts: []llms.ContentPart{ @@ -508,7 +612,7 @@ Content: }, }) if err != nil { - return "", fmt.Errorf("Error getting response from LLM: %v", err) + return "", fmt.Errorf("error getting response from LLM: %v", err) } return strings.TrimSpace(strings.Trim(completion.Choices[0].Content, "\"")), nil diff --git a/types.go b/types.go new file mode 100644 index 0000000..0320027 --- /dev/null +++ b/types.go @@ -0,0 +1,61 @@ +package main + +import ( + "time" +) + +type GetDocumentsApiResponse struct { + Count int `json:"count"` + Next interface{} `json:"next"` + Previous interface{} `json:"previous"` + All []int `json:"all"` + Results []struct { + ID int `json:"id"` + Correspondent interface{} `json:"correspondent"` + DocumentType interface{} `json:"document_type"` + StoragePath interface{} `json:"storage_path"` + Title string `json:"title"` + Content string `json:"content"` + Tags []int `json:"tags"` + Created time.Time `json:"created"` + CreatedDate string `json:"created_date"` + Modified time.Time `json:"modified"` + Added time.Time `json:"added"` + ArchiveSerialNumber interface{} `json:"archive_serial_number"` + OriginalFileName string `json:"original_file_name"` + ArchivedFileName string `json:"archived_file_name"` + Owner int `json:"owner"` + UserCanChange bool `json:"user_can_change"` + Notes []interface{} `json:"notes"` + SearchHit struct { + Score float64 `json:"score"` + Highlights string `json:"highlights"` + NoteHighlights string `json:"note_highlights"` + Rank int `json:"rank"` + } `json:"__search_hit__"` + } `json:"results"` +} + +// Document is a stripped down version of the document object from paperless-ngx. +// Response payload for /documents endpoint and part of request payload for /generate-suggestions endpoint +type Document struct { + ID int `json:"id"` + Title string `json:"title"` + Content string `json:"content"` + Tags []string `json:"tags"` +} + +// GenerateSuggestionsRequest is the request payload for generating suggestions for /generate-suggestions endpoint +type GenerateSuggestionsRequest struct { + Documents []Document `json:"documents"` + GenerateTitles bool `json:"generate_titles,omitempty"` + GenerateTags bool `json:"generate_tags,omitempty"` +} + +// DocumentSuggestion is the response payload for /generate-suggestions endpoint and the request payload for /update-documents endpoint (as an array) +type DocumentSuggestion struct { + ID int `json:"id"` + OriginalDocument Document `json:"original_document"` + SuggestedTitle string `json:"suggested_title,omitempty"` + SuggestedTags []string `json:"suggested_tags,omitempty"` +} diff --git a/web-app/src/components/SuggestionCard.tsx b/web-app/src/components/SuggestionCard.tsx index 25da077..361ce25 100644 --- a/web-app/src/components/SuggestionCard.tsx +++ b/web-app/src/components/SuggestionCard.tsx @@ -17,6 +17,7 @@ const SuggestionCard: React.FC = ({ onTagAddition, onTagDeletion, }) => { + const sortedAvailableTags = availableTags.sort((a, b) => a.name.localeCompare(b.name)); const document = suggestion.original_document; return (
@@ -64,7 +65,7 @@ const SuggestionCard: React.FC = ({ value: index.toString(), })) || [] } - suggestions={availableTags.map((tag) => ({ + suggestions={sortedAvailableTags.map((tag) => ({ id: tag.id, name: tag.name, label: tag.name,