fix: allow for caps in file name suffixes #206

@schinkelg ran aground of this one and I took the opportunity to add a
test to catch this sort of thing for next time.

docs/changelog.rst

@@ -1,6 +1,18 @@
 Changelog
 #########
 
+* 0.4.1
+  * Fix for `#206`_ wherein the pluggable parser didn't recognise files with
+    all-caps suffixes like ``.PDF``
+
+* 0.4.0
+  * Introducing reminders.  See `#199`_ for more information, but the short
+    explanation is that you can now attach simple notes & times to documents
+    which are made available via the API.  Currently, the default API
+    (basically just the Django admin) doesn't really make use of this, but
+    `Thomas Brueggemann`_ over at `Paperless Desktop`_ has said that he would
+    like to make use of this feature in his project.
+
 * 0.3.6
   * Fix for `#200`_ (!!) where the API wasn't configured to allow updating the
     correspondent or the tags for a document.
@@ -173,6 +185,8 @@ Changelog
 .. _Tim White: https://github.com/timwhite
 .. _Florian Harr: https://github.com/evils
 .. _Justin Snyman: https://github.com/stringlytyped
+.. _Thomas Brueggemann: https://github.com/thomasbrueggemann
+.. _Paperless Desktop: https://github.com/thomasbrueggemann/paperless-desktop
 
 .. _#20: https://github.com/danielquinn/paperless/issues/20
 .. _#44: https://github.com/danielquinn/paperless/issues/44
@@ -199,4 +213,6 @@ Changelog
 .. _#171: https://github.com/danielquinn/paperless/issues/171
 .. _#172: https://github.com/danielquinn/paperless/issues/172
 .. _#179: https://github.com/danielquinn/paperless/pull/179
+.. _#199: https://github.com/danielquinn/paperless/issues/199
 .. _#200: https://github.com/danielquinn/paperless/issues/200
+.. _#206: https://github.com/danielquinn/paperless/issues/206

src/documents/consumer.py

@@ -102,7 +102,7 @@ class Consumer(object):
             parser_class = self._get_parser_class(doc)
             if not parser_class:
                 self.log(
-                    "info", "No parsers could be found for {}".format(doc))
+                    "error", "No parsers could be found for {}".format(doc))
                 self._ignore.append(doc)
                 continue
 
@@ -160,6 +160,16 @@ class Consumer(object):
             if result:
                 options.append(result)
 
+        self.log(
+            "info",
+            "Parsers available: {}".format(
+                ", ".join([str(o["parser"].__name__) for o in options])
+            )
+        )
+
+        if not options:
+            return None
+
         # Return the parser with the highest weight.
         return sorted(
             options, key=lambda _: _["weight"], reverse=True)[0]["parser"]

src/documents/filters.py

@@ -8,7 +8,7 @@ class CorrespondentFilterSet(FilterSet):
     class Meta(object):
         model = Correspondent
         fields = {
-            'name': [
+            "name": [
                 "startswith", "endswith", "contains",
                 "istartswith", "iendswith", "icontains"
             ],
@@ -21,7 +21,7 @@ class TagFilterSet(FilterSet):
     class Meta(object):
         model = Tag
         fields = {
-            'name': [
+            "name": [
                 "startswith", "endswith", "contains",
                 "istartswith", "iendswith", "icontains"
             ],

src/documents/migrations/0016_auto_20170325_1558.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-03-25 15:58
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('documents', '0015_add_insensitive_to_match'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='document',
+            name='content',
+            field=models.TextField(blank=True, db_index=True, help_text='The raw, text-only data of the document.  This field is primarily used for searching.'),
+        ),
+    ]

src/documents/mixins.py

@@ -1,8 +1,3 @@
-from django.contrib.auth.mixins import AccessMixin
-from django.contrib.auth import authenticate, login
-import base64
-
-
 class Renderable(object):
     """
     A handy mixin to make it easier/cleaner to print output based on a
@@ -12,46 +7,3 @@ class Renderable(object):
     def _render(self, text, verbosity):
         if self.verbosity >= verbosity:
             print(text)
-
-
-class SessionOrBasicAuthMixin(AccessMixin):
-    """
-    Session or Basic Authentication mixin for Django.
-    It determines if the requester is already logged in or if they have
-    provided proper http-authorization and returning the view if all goes
-    well, otherwise responding with a 401.
-
-    Base for mixin found here: https://djangosnippets.org/snippets/3073/
-    """
-
-    def dispatch(self, request, *args, **kwargs):
-
-        # check if user is authenticated via the session
-        if request.user.is_authenticated:
-
-            # Already logged in, just return the view.
-            return super(SessionOrBasicAuthMixin, self).dispatch(
-                request, *args, **kwargs
-            )
-
-        # apparently not authenticated via session, maybe via HTTP Basic?
-        if 'HTTP_AUTHORIZATION' in request.META:
-            auth = request.META['HTTP_AUTHORIZATION'].split()
-            if len(auth) == 2:
-                # NOTE: Support for only basic authentication
-                if auth[0].lower() == "basic":
-                    authString = base64.b64decode(auth[1]).decode('utf-8')
-                    uname, passwd = authString.split(':')
-                    user = authenticate(username=uname, password=passwd)
-                    if user is not None:
-                        if user.is_active:
-                            login(request, user)
-                            request.user = user
-                            return super(
-                                SessionOrBasicAuthMixin, self
-                            ).dispatch(
-                                request, *args, **kwargs
-                            )
-
-        # nope, really not authenticated
-        return self.handle_no_permission()

src/documents/static/paperless.css

@@ -10,3 +10,14 @@ td a.tag {
   margin: 1px;
   display: inline-block;
 }
+
+#result_list th.column-note {
+  text-align: right;
+}
+#result_list td.field-note {
+  text-align: right;
+}
+#result_list td textarea {
+  width: 90%;
+  height: 5em;
+}

src/documents/tests/test_consumer.py

@@ -1,8 +1,56 @@
 from django.test import TestCase
+from unittest import mock
 
+from ..consumer import Consumer
 from ..models import FileInfo
 
 
+class TestConsumer(TestCase):
+
+    class DummyParser(object):
+        pass
+
+    def test__get_parser_class_1_parser(self):
+        self.assertEqual(
+            self._get_consumer()._get_parser_class("doc.pdf"),
+            self.DummyParser
+        )
+
+    @mock.patch("documents.consumer.os.makedirs")
+    @mock.patch("documents.consumer.os.path.exists", return_value=True)
+    @mock.patch("documents.consumer.document_consumer_declaration.send")
+    def test__get_parser_class_n_parsers(self, m, *args):
+
+        class DummyParser1(object):
+            pass
+
+        class DummyParser2(object):
+            pass
+
+        m.return_value = (
+            (None, lambda _: {"weight": 0, "parser": DummyParser1}),
+            (None, lambda _: {"weight": 1, "parser": DummyParser2}),
+        )
+
+        self.assertEqual(Consumer()._get_parser_class("doc.pdf"), DummyParser2)
+
+    @mock.patch("documents.consumer.os.makedirs")
+    @mock.patch("documents.consumer.os.path.exists", return_value=True)
+    @mock.patch("documents.consumer.document_consumer_declaration.send")
+    def test__get_parser_class_0_parsers(self, m, *args):
+        m.return_value = ((None, lambda _: None),)
+        self.assertIsNone(Consumer()._get_parser_class("doc.pdf"))
+
+    @mock.patch("documents.consumer.os.makedirs")
+    @mock.patch("documents.consumer.os.path.exists", return_value=True)
+    @mock.patch("documents.consumer.document_consumer_declaration.send")
+    def _get_consumer(self, m, *args):
+        m.return_value = (
+            (None, lambda _: {"weight": 0, "parser": self.DummyParser}),
+        )
+        return Consumer()
+
+
 class TestAttributes(TestCase):
 
     TAGS = ("tag1", "tag2", "tag3")

src/documents/views.py

@@ -2,15 +2,16 @@ from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import DetailView, FormView, TemplateView
 from django_filters.rest_framework import DjangoFilterBackend
-from rest_framework.filters import SearchFilter, OrderingFilter
 from paperless.db import GnuPG
+from paperless.mixins import SessionOrBasicAuthMixin
+from paperless.views import StandardPagination
+from rest_framework.filters import OrderingFilter, SearchFilter
 from rest_framework.mixins import (
     DestroyModelMixin,
     ListModelMixin,
     RetrieveModelMixin,
     UpdateModelMixin
 )
-from rest_framework.pagination import PageNumberPagination
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.viewsets import (
     GenericViewSet,
@@ -27,7 +28,6 @@ from .serialisers import (
     LogSerializer,
     TagSerializer
 )
-from .mixins import SessionOrBasicAuthMixin
 
 
 class IndexView(TemplateView):
@@ -92,12 +92,6 @@ class PushView(SessionOrBasicAuthMixin, FormView):
         return HttpResponse("0")
 
 
-class StandardPagination(PageNumberPagination):
-    page_size = 25
-    page_size_query_param = "page-size"
-    max_page_size = 100000
-
-
 class CorrespondentViewSet(ModelViewSet):
     model = Correspondent
     queryset = Correspondent.objects.all()

src/paperless/mixins.py

@@ -0,0 +1,46 @@
+from django.contrib.auth.mixins import AccessMixin
+from django.contrib.auth import authenticate, login
+import base64
+
+
+class SessionOrBasicAuthMixin(AccessMixin):
+    """
+    Session or Basic Authentication mixin for Django.
+    It determines if the requester is already logged in or if they have
+    provided proper http-authorization and returning the view if all goes
+    well, otherwise responding with a 401.
+
+    Base for mixin found here: https://djangosnippets.org/snippets/3073/
+    """
+
+    def dispatch(self, request, *args, **kwargs):
+
+        # check if user is authenticated via the session
+        if request.user.is_authenticated:
+
+            # Already logged in, just return the view.
+            return super(SessionOrBasicAuthMixin, self).dispatch(
+                request, *args, **kwargs
+            )
+
+        # apparently not authenticated via session, maybe via HTTP Basic?
+        if 'HTTP_AUTHORIZATION' in request.META:
+            auth = request.META['HTTP_AUTHORIZATION'].split()
+            if len(auth) == 2:
+                # NOTE: Support for only basic authentication
+                if auth[0].lower() == "basic":
+                    authString = base64.b64decode(auth[1]).decode('utf-8')
+                    uname, passwd = authString.split(':')
+                    user = authenticate(username=uname, password=passwd)
+                    if user is not None:
+                        if user.is_active:
+                            login(request, user)
+                            request.user = user
+                            return super(
+                                SessionOrBasicAuthMixin, self
+                            ).dispatch(
+                                request, *args, **kwargs
+                            )
+
+        # nope, really not authenticated
+        return self.handle_no_permission()

src/paperless/settings.py

@@ -61,6 +61,7 @@ INSTALLED_APPS = [
     "django_extensions",
 
     "documents.apps.DocumentsConfig",
+    "reminders.apps.RemindersConfig",
     "paperless_tesseract.apps.PaperlessTesseractConfig",
 
     "flat_responsive",

src/paperless/urls.py

@@ -24,12 +24,14 @@ from documents.views import (
     IndexView, FetchView, PushView,
     CorrespondentViewSet, TagViewSet, DocumentViewSet, LogViewSet
 )
+from reminders.views import ReminderViewSet
 
 router = DefaultRouter()
-router.register(r'correspondents', CorrespondentViewSet)
+router.register(r"correspondents", CorrespondentViewSet)
-router.register(r'tags', TagViewSet)
+router.register(r"documents", DocumentViewSet)
-router.register(r'documents', DocumentViewSet)
+router.register(r"logs", LogViewSet)
-router.register(r'logs', LogViewSet)
+router.register(r"reminders", ReminderViewSet)
+router.register(r"tags", TagViewSet)
 
 urlpatterns = [
 

src/paperless/views.py

@@ -0,0 +1,7 @@
+from rest_framework.pagination import PageNumberPagination
+
+
+class StandardPagination(PageNumberPagination):
+    page_size = 25
+    page_size_query_param = "page-size"
+    max_page_size = 100000

src/paperless_tesseract/signals.py

@@ -5,7 +5,7 @@ from .parsers import RasterisedDocumentParser
 
 class ConsumerDeclaration(object):
 
-    MATCHING_FILES = re.compile("^.*\.(pdf|jpg|gif|png|tiff|pnm|bmp)$")
+    MATCHING_FILES = re.compile("^.*\.(pdf|jpg|gif|png|tiff?|pnm|bmp)$")
 
     @classmethod
     def handle(cls, sender, **kwargs):
@@ -14,7 +14,7 @@ class ConsumerDeclaration(object):
     @classmethod
     def test(cls, doc):
 
-        if cls.MATCHING_FILES.match(doc):
+        if cls.MATCHING_FILES.match(doc.lower()):
             return {
                 "parser": RasterisedDocumentParser,
                 "weight": 0

src/paperless_tesseract/tests/test_signals.py

@@ -0,0 +1,36 @@
+from django.test import TestCase
+
+from ..signals import ConsumerDeclaration
+
+
+class SignalsTestCase(TestCase):
+
+    def test_test_handles_various_file_names_true(self):
+
+        prefixes = (
+            "doc", "My Document", "Μυ Γρεεκ Δοψθμεντ", "Doc -with - tags",
+            "A document with a . in it", "Doc with -- in it"
+        )
+        suffixes = (
+            "pdf", "jpg", "gif", "png", "tiff", "tif", "pnm", "bmp",
+            "PDF", "JPG", "GIF", "PNG", "TIFF", "TIF", "PNM", "BMP",
+            "pDf", "jPg", "gIf", "pNg", "tIff", "tIf", "pNm", "bMp",
+        )
+
+        for prefix in prefixes:
+            for suffix in suffixes:
+                name = "{}.{}".format(prefix, suffix)
+                self.assertTrue(ConsumerDeclaration.test(name))
+
+    def test_test_handles_various_file_names_false(self):
+
+        prefixes = ("doc",)
+        suffixes = ("txt", "markdown", "",)
+
+        for prefix in prefixes:
+            for suffix in suffixes:
+                name = "{}.{}".format(prefix, suffix)
+                self.assertFalse(ConsumerDeclaration.test(name))
+
+        self.assertFalse(ConsumerDeclaration.test(""))
+        self.assertFalse(ConsumerDeclaration.test("doc"))

src/reminders/admin.py

@@ -0,0 +1,20 @@
+from django.conf import settings
+from django.contrib import admin
+
+from .models import Reminder
+
+
+class ReminderAdmin(admin.ModelAdmin):
+
+    class Media:
+        css = {
+            "all": ("paperless.css",)
+        }
+
+    list_per_page = settings.PAPERLESS_LIST_PER_PAGE
+    list_display = ("date", "document", "note")
+    list_filter = ("date",)
+    list_editable = ("note",)
+
+
+admin.site.register(Reminder, ReminderAdmin)

src/reminders/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class RemindersConfig(AppConfig):
+    name = "reminders"

src/reminders/filters.py

@@ -0,0 +1,14 @@
+from django_filters.rest_framework import CharFilter, FilterSet
+
+from .models import Reminder
+
+
+class ReminderFilterSet(FilterSet):
+
+    class Meta(object):
+        model = Reminder
+        fields = {
+            "document": ["exact"],
+            "date": ["gt", "lt", "gte", "lte", "exact"],
+            "note": ["istartswith", "iendswith", "icontains"]
+        }

src/reminders/migrations/0001_initial.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-03-25 15:58
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('documents', '0016_auto_20170325_1558'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Reminder',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('date', models.DateTimeField()),
+                ('note', models.TextField(blank=True)),
+                ('document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='documents.Document')),
+            ],
+        ),
+    ]

src/reminders/models.py

@@ -0,0 +1,8 @@
+from django.db import models
+
+
+class Reminder(models.Model):
+
+    document = models.ForeignKey("documents.Document")
+    date = models.DateTimeField()
+    note = models.TextField(blank=True)

src/reminders/serialisers.py

@@ -0,0 +1,14 @@
+from documents.models import Document
+from rest_framework import serializers
+
+from .models import Reminder
+
+
+class ReminderSerializer(serializers.HyperlinkedModelSerializer):
+
+    document = serializers.HyperlinkedRelatedField(
+        view_name="drf:document-detail", queryset=Document.objects)
+
+    class Meta(object):
+        model = Reminder
+        fields = ("id", "document", "date", "note")

src/reminders/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

src/reminders/views.py

@@ -0,0 +1,22 @@
+from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework.filters import OrderingFilter
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.viewsets import (
+    ModelViewSet,
+)
+
+from .filters import ReminderFilterSet
+from .models import Reminder
+from .serialisers import ReminderSerializer
+from paperless.views import StandardPagination
+
+
+class ReminderViewSet(ModelViewSet):
+    model = Reminder
+    queryset = Reminder.objects
+    serializer_class = ReminderSerializer
+    pagination_class = StandardPagination
+    permission_classes = (IsAuthenticated,)
+    filter_backends = (DjangoFilterBackend, OrderingFilter)
+    filter_class = ReminderFilterSet
+    ordering_fields = ("date", "document")