Updated export routine.

app/Export/Collector/UploadCollector.php

@@ -15,6 +15,7 @@ namespace FireflyIII\Export\Collector;
 
 use Crypt;
 use Illuminate\Contracts\Encryption\DecryptException;
+use Illuminate\Contracts\Filesystem\FileNotFoundException;
 use Log;
 use Storage;
 
@@ -50,14 +51,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface
     public function run(): bool
     {
         Log::debug('Going to collect attachments', ['key' => $this->job->key]);
-
-        // file names associated with the old import routine.
-        $this->vintageFormat = sprintf('csv-upload-%d-', $this->job->user->id);
-
-        // collect old upload files (names beginning with "csv-upload".
-        $this->collectVintageUploads();
-
-        // then collect current upload files:
         $this->collectModernUploads();
 
         return true;
@@ -70,7 +63,8 @@ class UploadCollector extends BasicCollector implements CollectorInterface
      */
     private function collectModernUploads(): bool
     {
-        $set  = $this->job->user->importJobs()->where('status', 'import_complete')->get(['import_jobs.*']);
+        $set = $this->job->user->importJobs()->whereIn('status', ['import_complete', 'finished'])->get(['import_jobs.*']);
+        Log::debug(sprintf('Found %d import jobs', $set->count()));
         $keys = [];
         if ($set->count() > 0) {
             $keys = $set->pluck('key')->toArray();
@@ -83,59 +77,6 @@ class UploadCollector extends BasicCollector implements CollectorInterface
         return true;
     }
 
-    /**
-     * This method collects all the uploads that are uploaded using the "old" importer. So from before the summer of 2016.
-     *
-     * @return bool
-     */
-    private function collectVintageUploads(): bool
-    {
-        // grab upload directory.
-        $files = $this->uploadDisk->files();
-
-        foreach ($files as $entry) {
-            $this->processVintageUpload($entry);
-        }
-
-        return true;
-    }
-
-    /**
-     * This method tells you when the vintage upload file was actually uploaded.
-     *
-     * @param string $entry
-     *
-     * @return string
-     */
-    private function getVintageUploadDate(string $entry): string
-    {
-        // this is an original upload.
-        $parts          = explode('-', str_replace(['.csv.encrypted', $this->vintageFormat], '', $entry));
-        $originalUpload = intval($parts[1]);
-        $date           = date('Y-m-d \a\t H-i-s', $originalUpload);
-
-        return $date;
-    }
-
-    /**
-     * Tells you if a file name is a vintage upload.
-     *
-     * @param string $entry
-     *
-     * @return bool
-     */
-    private function isVintageImport(string $entry): bool
-    {
-        $len = strlen($this->vintageFormat);
-        // file is part of the old import routine:
-        if (substr($entry, 0, $len) === $this->vintageFormat) {
-
-            return true;
-        }
-
-        return false;
-    }
-
     /**
      * @param string $key
      *
@@ -153,7 +94,7 @@ class UploadCollector extends BasicCollector implements CollectorInterface
         $content = '';
         try {
             $content = Crypt::decrypt($this->uploadDisk->get(sprintf('%s.upload', $key)));
-        } catch (DecryptException $e) {
+        } catch (FileNotFoundException | DecryptException $e) {
             Log::error(sprintf('Could not decrypt old import file "%s". Skipped because: %s', $key, $e->getMessage()));
         }
 
@@ -168,47 +109,4 @@ class UploadCollector extends BasicCollector implements CollectorInterface
         return true;
     }
 
-    /**
-     * If the file is a vintage upload, process it.
-     *
-     * @param string $entry
-     *
-     * @return bool
-     */
-    private function processVintageUpload(string $entry): bool
-    {
-        if ($this->isVintageImport($entry)) {
-            $this->saveVintageImportFile($entry);
-
-            return true;
-        }
-
-        return false;
-    }
-
-
-    /**
-     * This will store the content of the old vintage upload somewhere.
-     *
-     * @param string $entry
-     */
-    private function saveVintageImportFile(string $entry)
-    {
-        $content = '';
-        try {
-            $content = Crypt::decrypt($this->uploadDisk->get($entry));
-        } catch (DecryptException $e) {
-            Log::error('Could not decrypt old CSV import file ' . $entry . '. Skipped because ' . $e->getMessage());
-        }
-
-        if (strlen($content) > 0) {
-            // add to export disk.
-            $date = $this->getVintageUploadDate($entry);
-            $file = $this->job->key . '-Old import dated ' . $date . '.csv';
-            $this->exportDisk->put($file, $content);
-            $this->getEntries()->push($file);
-        }
-    }
-
-
 }

app/Export/Entry/Entry.php

@@ -13,6 +13,7 @@ declare(strict_types=1);
 
 namespace FireflyIII\Export\Entry;
 
+use FireflyIII\Models\Transaction;
 use Steam;
 
 /**
@@ -37,24 +38,43 @@ final class Entry
 {
     // @formatter:off
     public $journal_id;
+    public $transaction_id = 0;
+
     public $date;
     public $description;
 
     public $currency_code;
     public $amount;
+    public $foreign_currency_code = '';
+    public $foreign_amount        = '0';
 
     public $transaction_type;
 
     public $asset_account_id;
     public $asset_account_name;
+    public $asset_account_iban;
+    public $asset_account_bic;
+    public $asset_account_number;
+    public $asset_currency_code;
 
     public $opposing_account_id;
     public $opposing_account_name;
+    public $opposing_account_iban;
+    public $opposing_account_bic;
+    public $opposing_account_number;
+    public $opposing_currency_code;
 
     public $budget_id;
     public $budget_name;
+
     public $category_id;
     public $category_name;
+
+    public $bill_id;
+    public $bill_name;
+
+    public $notes;
+    public $tags;
     // @formatter:on
 
     /**
@@ -95,5 +115,72 @@ final class Entry
         return $entry;
     }
 
+    /**
+     * Converts a given transaction (as collected by the collector) into an export entry.
+     *
+     * @param Transaction $transaction
+     *
+     * @return Entry
+     */
+    public static function fromTransaction(Transaction $transaction): Entry
+    {
+        $entry                 = new self;
+        $entry->journal_id     = $transaction->journal_id;
+        $entry->transaction_id = $transaction->id;
+        $entry->date           = $transaction->date->format('Ymd');
+        $entry->description    = $transaction->description;
+        if (strlen(strval($transaction->transaction_description)) > 0) {
+            $entry->description = $transaction->transaction_description . '(' . $transaction->description . ')';
+        }
+        $entry->currency_code = $transaction->transactionCurrency->code;
+        $entry->amount        = round($transaction->transaction_amount, $transaction->transactionCurrency->decimal_places);
+
+        $entry->foreign_currency_code = is_null($transaction->foreign_currency_id) ? null : $transaction->foreignCurrency->code;
+        $entry->foreign_amount        = is_null($transaction->foreign_currency_id)
+            ? null
+            : round(
+                $transaction->transaction_foreign_amount, $transaction->foreignCurrency->decimal_places
+            );
+
+        $entry->transaction_type     = $transaction->transaction_type_type;
+        $entry->asset_account_id     = $transaction->account_id;
+        $entry->asset_account_name   = app('steam')->tryDecrypt($transaction->account_name);
+        $entry->asset_account_iban   = $transaction->account_iban;
+        $entry->asset_account_number = $transaction->account_number;
+        $entry->asset_account_bic    = $transaction->account_bic;
+        // asset_currency_code
+        $entry->opposing_account_id     = $transaction->opposing_account_id;
+        $entry->opposing_account_name   = app('steam')->tryDecrypt($transaction->opposing_account_name);
+        $entry->opposing_account_iban   = $transaction->opposing_account_iban;
+        $entry->opposing_account_number = $transaction->opposing_account_number;
+        $entry->opposing_account_bic    = $transaction->opposing_account_bic;
+        // opposing currency code
+
+        /** budget */
+        $entry->budget_id   = $transaction->transaction_budget_id;
+        $entry->budget_name = app('steam')->tryDecrypt($transaction->transaction_budget_name);
+        if (is_null($transaction->transaction_budget_id)) {
+            $entry->budget_id   = $transaction->transaction_journal_budget_id;
+            $entry->budget_name = app('steam')->tryDecrypt($transaction->transaction_journal_budget_name);
+        }
+
+        /** category */
+        $entry->category_id   = $transaction->transaction_category_id;
+        $entry->category_name = app('steam')->tryDecrypt($transaction->transaction_category_name);
+        if (is_null($transaction->transaction_category_id)) {
+            $entry->category_id   = $transaction->transaction_journal_category_id;
+            $entry->category_name = app('steam')->tryDecrypt($transaction->transaction_journal_category_name);
+        }
+
+        /** budget */
+        $entry->bill_id   = $transaction->bill_id;
+        $entry->bill_name = app('steam')->tryDecrypt($transaction->bill_name);
+
+        $entry->tags  = $transaction->tags;
+        $entry->notes = $transaction->notes;
+
+        return $entry;
+    }
+
 
 }

app/Export/ExpandedProcessor.php

@@ -0,0 +1,308 @@
+<?php
+/**
+ * ExpandedProcessor.php
+ * Copyright (c) 2017 thegrumpydictator@gmail.com
+ * This software may be modified and distributed under the terms of the
+ * Creative Commons Attribution-ShareAlike 4.0 International License.
+ *
+ * See the LICENSE file for details.
+ */
+
+declare(strict_types=1);
+
+namespace FireflyIII\Export;
+
+
+use Crypt;
+use DB;
+use FireflyIII\Exceptions\FireflyException;
+use FireflyIII\Export\Collector\AttachmentCollector;
+use FireflyIII\Export\Collector\UploadCollector;
+use FireflyIII\Export\Entry\Entry;
+use FireflyIII\Helpers\Collector\JournalCollectorInterface;
+use FireflyIII\Helpers\Filter\InternalTransferFilter;
+use FireflyIII\Models\AccountMeta;
+use FireflyIII\Models\ExportJob;
+use FireflyIII\Models\Transaction;
+use FireflyIII\Models\TransactionJournalMeta;
+use Illuminate\Support\Collection;
+use Log;
+use Storage;
+use ZipArchive;
+
+/**
+ * Class ExpandedProcessor
+ *
+ * @package FireflyIII\Export
+ */
+class ExpandedProcessor implements ProcessorInterface
+{
+
+    /** @var Collection */
+    public $accounts;
+    /** @var  string */
+    public $exportFormat;
+    /** @var  bool */
+    public $includeAttachments;
+    /** @var  bool */
+    public $includeOldUploads;
+    /** @var  ExportJob */
+    public $job;
+    /** @var array */
+    public $settings;
+    /** @var  Collection */
+    private $exportEntries;
+    /** @var  Collection */
+    private $files;
+    /** @var  Collection */
+    private $journals;
+
+    /**
+     * Processor constructor.
+     */
+    public function __construct()
+    {
+        $this->journals      = new Collection;
+        $this->exportEntries = new Collection;
+        $this->files         = new Collection;
+    }
+
+    /**
+     * @return bool
+     */
+    public function collectAttachments(): bool
+    {
+        /** @var AttachmentCollector $attachmentCollector */
+        $attachmentCollector = app(AttachmentCollector::class);
+        $attachmentCollector->setJob($this->job);
+        $attachmentCollector->setDates($this->settings['startDate'], $this->settings['endDate']);
+        $attachmentCollector->run();
+        $this->files = $this->files->merge($attachmentCollector->getEntries());
+
+        return true;
+    }
+
+    /**
+     * @return bool
+     */
+    public function collectJournals(): bool
+    {
+        // use journal collector thing.
+        /** @var JournalCollectorInterface $collector */
+        $collector = app(JournalCollectorInterface::class);
+        $collector->setAccounts($this->accounts)->setRange($this->settings['startDate'], $this->settings['endDate'])
+                  ->withOpposingAccount()->withBudgetInformation()->withCategoryInformation()
+                  ->removeFilter(InternalTransferFilter::class);
+        $transactions = $collector->getJournals();
+        // get some more meta data for each entry:
+        $ids         = $transactions->pluck('journal_id')->toArray();
+        $assetIds    = $transactions->pluck('account_id')->toArray();
+        $opposingIds = $transactions->pluck('opposing_account_id')->toArray();
+        $notes       = $this->getNotes($ids);
+        $tags        = $this->getTags($ids);
+        $ibans       = $this->getIbans($assetIds) + $this->getIbans($opposingIds);
+        $transactions->each(
+            function (Transaction $transaction) use ($notes, $tags, $ibans) {
+                $journalId                            = intval($transaction->journal_id);
+                $accountId                            = intval($transaction->account_id);
+                $opposingId                           = intval($transaction->opposing_account_id);
+                $transaction->notes                   = $notes[$journalId] ?? '';
+                $transaction->tags                    = join(',', $tags[$journalId] ?? []);
+                $transaction->account_number          = $ibans[$accountId]['accountNumber'] ?? '';
+                $transaction->account_bic             = $ibans[$accountId]['BIC'] ?? '';
+                $transaction->opposing_account_number = $ibans[$opposingId]['accountNumber'] ?? '';
+                $transaction->opposing_account_bic    = $ibans[$opposingId]['BIC'] ?? '';
+
+            }
+        );
+
+        $this->journals = $transactions;
+
+        return true;
+    }
+
+    /**
+     * @return bool
+     */
+    public function collectOldUploads(): bool
+    {
+        /** @var UploadCollector $uploadCollector */
+        $uploadCollector = app(UploadCollector::class);
+        $uploadCollector->setJob($this->job);
+        $uploadCollector->run();
+
+        $this->files = $this->files->merge($uploadCollector->getEntries());
+
+        return true;
+    }
+
+    /**
+     * @return bool
+     */
+    public function convertJournals(): bool
+    {
+        $this->journals->each(
+            function (Transaction $transaction) {
+                $this->exportEntries->push(Entry::fromTransaction($transaction));
+            }
+        );
+        Log::debug(sprintf('Count %d entries in exportEntries (convertJournals)', $this->exportEntries->count()));
+
+        return true;
+    }
+
+    /**
+     * @return bool
+     * @throws FireflyException
+     */
+    public function createZipFile(): bool
+    {
+        $zip      = new ZipArchive;
+        $file     = $this->job->key . '.zip';
+        $fullPath = storage_path('export') . '/' . $file;
+
+        if ($zip->open($fullPath, ZipArchive::CREATE) !== true) {
+            throw new FireflyException('Cannot store zip file.');
+        }
+        // for each file in the collection, add it to the zip file.
+        $disk = Storage::disk('export');
+        foreach ($this->getFiles() as $entry) {
+            // is part of this job?
+            $zipFileName = str_replace($this->job->key . '-', '', $entry);
+            $zip->addFromString($zipFileName, $disk->get($entry));
+        }
+
+        $zip->close();
+
+        // delete the files:
+        $this->deleteFiles();
+
+        return true;
+    }
+
+    /**
+     * @return bool
+     */
+    public function exportJournals(): bool
+    {
+        $exporterClass = config('firefly.export_formats.' . $this->exportFormat);
+        $exporter      = app($exporterClass);
+        $exporter->setJob($this->job);
+        $exporter->setEntries($this->exportEntries);
+        $exporter->run();
+        $this->files->push($exporter->getFileName());
+
+        return true;
+    }
+
+    /**
+     * @return Collection
+     */
+    public function getFiles(): Collection
+    {
+        return $this->files;
+    }
+
+    /**
+     * Save export job settings to class.
+     *
+     * @param array $settings
+     */
+    public function setSettings(array $settings)
+    {
+        // save settings
+        $this->settings           = $settings;
+        $this->accounts           = $settings['accounts'];
+        $this->exportFormat       = $settings['exportFormat'];
+        $this->includeAttachments = $settings['includeAttachments'];
+        $this->includeOldUploads  = $settings['includeOldUploads'];
+        $this->job                = $settings['job'];
+    }
+
+    /**
+     *
+     */
+    private function deleteFiles()
+    {
+        $disk = Storage::disk('export');
+        foreach ($this->getFiles() as $file) {
+            $disk->delete($file);
+        }
+    }
+
+    /**
+     * Get all IBAN / SWIFT / account numbers
+     *
+     * @param array $array
+     *
+     * @return array
+     */
+    private function getIbans(array $array): array
+    {
+        $array  = array_unique($array);
+        $return = [];
+        $set    = AccountMeta::whereIn('account_id', $array)
+                             ->leftJoin('accounts', 'accounts.id', 'account_meta.account_id')
+                             ->where('accounts.user_id', $this->job->user_id)
+                             ->whereIn('account_meta.name', ['accountNumber', 'BIC', 'currency_id'])
+                             ->get(['account_meta.id', 'account_meta.account_id', 'account_meta.name', 'account_meta.data']);
+        /** @var AccountMeta $meta */
+        foreach ($set as $meta) {
+            $id                       = intval($meta->account_id);
+            $return[$id][$meta->name] = $meta->data;
+        }
+
+        return $return;
+    }
+
+    /**
+     * Returns, if present, for the given journal ID's the notes.
+     *
+     * @param array $array
+     *
+     * @return array
+     */
+    private function getNotes(array $array): array
+    {
+        $array  = array_unique($array);
+        $set    = TransactionJournalMeta::whereIn('journal_meta.transaction_journal_id', $array)
+                                        ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'journal_meta.transaction_journal_id')
+                                        ->where('transaction_journals.user_id', $this->job->user_id)
+                                        ->where('journal_meta.name', 'notes')->get(
+                ['journal_meta.transaction_journal_id', 'journal_meta.data', 'journal_meta.id']
+            );
+        $return = [];
+        /** @var TransactionJournalMeta $meta */
+        foreach ($set as $meta) {
+            $id          = intval($meta->transaction_journal_id);
+            $return[$id] = $meta->data;
+        }
+
+        return $return;
+    }
+
+    /**
+     * Returns a comma joined list of all the users tags linked to these journals.
+     *
+     * @param array $array
+     *
+     * @return array
+     */
+    private function getTags(array $array): array
+    {
+        $set    = DB::table('tag_transaction_journal')
+                    ->whereIn('tag_transaction_journal.transaction_journal_id', $array)
+                    ->leftJoin('tags', 'tag_transaction_journal.tag_id', '=', 'tags.id')
+                    ->leftJoin('transaction_journals', 'transaction_journals.id', '=', 'tag_transaction_journal.transaction_journal_id')
+                    ->where('transaction_journals.user_id', $this->job->user_id)
+                    ->get(['tag_transaction_journal.transaction_journal_id', 'tags.tag']);
+        $result = [];
+        foreach ($set as $entry) {
+            $id            = intval($entry->transaction_journal_id);
+            $result[$id]   = isset($result[$id]) ? $result[$id] : [];
+            $result[$id][] = Crypt::decrypt($entry->tag);
+        }
+
+        return $result;
+    }
+}

app/Export/Exporter/CsvExporter.php

@@ -58,8 +58,12 @@ class CsvExporter extends BasicExporter implements ExporterInterface
 
         // get field names for header row:
         $first   = $this->getEntries()->first();
-        $headers = array_keys(get_object_vars($first));
-        $rows[]  = $headers;
+        $headers = [];
+        if (!is_null($first)) {
+            $headers = array_keys(get_object_vars($first));
+        }
+
+        $rows[] = $headers;
 
         /** @var Entry $entry */
         foreach ($this->getEntries() as $entry) {