diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml
index fc15cda27e..830ffb5c01 100644
--- a/.github/workflows/cleanup.yml
+++ b/.github/workflows/cleanup.yml
@@ -61,7 +61,7 @@ jobs:
               'cleanup.yml',
               'close-duplicates.yml',
               'closed-issues.yml',
-              'debug-info-actions.yml',
+              'issues-reply-old-versions.yml',
               'depsreview.yml',
               'label-actions.yml',
               'lock.yml',
diff --git a/.github/workflows/debug-info-actions.yml b/.github/workflows/debug-info-actions.yml
deleted file mode 100644
index 394102e535..0000000000
--- a/.github/workflows/debug-info-actions.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: 'Issues - Respond to hidden commands'
-
-# the workflow to execute on is comments that are newly created
-on:
-  issues:
-    types: [ opened, edited ]
-  issue_comment:
-    types: [ created ]
-
-# permissions needed for reacting to IssueOps commands on issues and PRs
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-  checks: read
-
-jobs:
-  respond:
-    runs-on: ubuntu-latest
-    steps:
-      - run: |
-          ISSUE_BODY=$(gh issue view $NUMBER --json body)
-          if [[ $ISSUE_BODY == *".eOxNZAmyGz6CXMyf"* ]]; then
-            gh issue comment "$NUMBER" --body "$V2_ISSUE_REPLY_BODY"
-            gh issue close "$NUMBER" --reason completed
-          fi
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GH_REPO: ${{ github.repository }}
-          NUMBER: ${{ github.event.issue.number }}
-          V2_ISSUE_REPLY_BODY: ${{ secrets.V2_ISSUE_REPLY_BODY }}
-          LABELS: v2-layout-issue
diff --git a/.github/workflows/issues-reply-old-versions.yml b/.github/workflows/issues-reply-old-versions.yml
new file mode 100644
index 0000000000..862ede4b75
--- /dev/null
+++ b/.github/workflows/issues-reply-old-versions.yml
@@ -0,0 +1,51 @@
+name: 'Issues - Respond to old versions'
+
+# the workflow to execute on is comments that are newly created
+on:
+  issues:
+    types: [ opened ]
+
+# permissions needed for reacting to IssueOps commands on issues and PRs
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+  checks: read
+
+jobs:
+  respond:
+    runs-on: ubuntu-latest
+    steps:
+      - id: ff3version
+        uses: pozetroninc/github-action-get-latest-release@master
+        with:
+          owner: firefly-iii
+          repo: firefly-iii
+          excludes: prerelease, draft
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - id: importerversion
+        uses: pozetroninc/github-action-get-latest-release@master
+        with:
+          owner: firefly-iii
+          repo: data-importer
+          excludes: prerelease, draft
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - run: |
+          ISSUE_BODY=$(gh issue view $NUMBER --json body)
+          if [[ $ISSUE_BODY != *${{ steps.ff3version.outputs.release }}* && $ISSUE_BODY != *${{ steps.importerversion.outputs.release }}* ]]; then
+
+            MESSAGE="Hi there!
+
+          This is an automated reply. \`Share and enjoy\`
+
+          You triggered an automated reply, because it looks like you're not running Firefly III version **${{ steps.ff3version.outputs.release }}** or version **${{ steps.importerversion.outputs.release }}** of the data importer.
+
+          Please be so kind as to include the output from the \`/debug\`-page, if you have not done so already. If you already included the debug information, plase make sure to upgrade to the latest version of Firefly III (or the data importer) *first*. This may already solve your issue.
+
+          If you are running the latest version, and this message is wrong, my apologies for the intrusion."
+            gh issue comment "$NUMBER" --body "$MESSAGE"
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+          NUMBER: ${{ github.event.issue.number }}