gh-mirrors/firefly-iii
1
0
Fork 0
mirror of https://github.com/firefly-iii/firefly-iii.git synced 2026-01-10 12:24:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved code for new import + some tests.

Browse Source
This commit is contained in:
James Cole
2018-05-04 20:21:27 +02:00
parent b541f7b944
commit 1c0da454db
22 changed files with 2770 additions and 2281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1650
app/Import/Routine/BunqRoutine.php
View File

File diff suppressed because it is too large Load Diff

535
app/Import/Routine/FileRoutine.php
View File

@@ -24,6 +24,7 @@ namespace FireflyIII\Import\Routine;
use Carbon\Carbon;
use DB;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Import\FileProcessor\FileProcessorInterface;
use FireflyIII\Import\Storage\ImportStorage;
use FireflyIII\Models\ImportJob;
@@ -38,271 +39,295 @@ use Log;
*/
class FileRoutine implements RoutineInterface
{
/** @var Collection */
public $errors;
/** @var Collection */
public $journals;
/** @var int */
public $lines = 0;
/** @var ImportJob */
private $job;
/** @var ImportJobRepositoryInterface */
private $repository;
/**
* ImportRoutine constructor.
*/
public function __construct()
{
$this->journals = new Collection;
$this->errors = new Collection;
}
/**
* @return Collection
*/
public function getErrors(): Collection
{
return $this->errors;
}
/**
* @return Collection
*/
public function getJournals(): Collection
{
return $this->journals;
}
/**
* @return int
*/
public function getLines(): int
{
return $this->lines;
}
// /** @var Collection */
// public $errors;
// /** @var Collection */
// public $journals;
// /** @var int */
// public $lines = 0;
// /** @var ImportJob */
// private $job;
//
// /** @var ImportJobRepositoryInterface */
// private $repository;
//
// /**
// * ImportRoutine constructor.
// */
// public function __construct()
// {
// $this->journals = new Collection;
// $this->errors = new Collection;
// }
//
// /**
// * @return Collection
// */
// public function getErrors(): Collection
// {
// return $this->errors;
// }
//
// /**
// * @return Collection
// */
// public function getJournals(): Collection
// {
// return $this->journals;
// }
//
// /**
// * @return int
// */
// public function getLines(): int
// {
// return $this->lines;
// }
//
// /**
// *
// */
// public function run(): bool
// {
// if ('configured' !== $this->getStatus()) {
// Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
//
// return false;
// }
// set_time_limit(0);
// Log::info(sprintf('Start with import job %s', $this->job->key));
//
// // total steps: 6
// $this->setTotalSteps(6);
//
// $importObjects = $this->getImportObjects();
// $this->lines = $importObjects->count();
// $this->addStep();
//
// // total steps can now be extended. File has been scanned. 7 steps per line:
// $this->addTotalSteps(7 * $this->lines);
//
// // once done, use storage thing to actually store them:
// Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
//
// $storage = $this->storeObjects($importObjects);
// $this->addStep();
// Log::debug('Back in run()');
//
// Log::debug('Updated job...');
// Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
// $this->journals = $storage->journals;
// $this->errors = $storage->errors;
//
// Log::debug('Going to call createImportTag()');
//
// // create tag, link tag to all journals:
// $this->createImportTag();
// $this->addStep();
//
// // update job:
// $this->setStatus('finished');
//
// Log::info(sprintf('Done with import job %s', $this->job->key));
//
// return true;
// }
//
// /**
// * @param ImportJob $job
// */
// public function setJob(ImportJob $job)
// {
// $this->job = $job;
// $this->repository = app(ImportJobRepositoryInterface::class);
// $this->repository->setUser($job->user);
// }
//
// /**
// * @return Collection
// */
// protected function getImportObjects(): Collection
// {
// $objects = new Collection;
// $fileType = $this->getConfig()['file-type'] ?? 'csv';
// // will only respond to "file"
// $class = config(sprintf('import.options.file.processors.%s', $fileType));
// /** @var FileProcessorInterface $processor */
// $processor = app($class);
// $processor->setJob($this->job);
//
// if ('configured' === $this->getStatus()) {
// // set job as "running"...
// $this->setStatus('running');
//
// Log::debug('Job is configured, start with run()');
// $processor->run();
// $objects = $processor->getObjects();
// }
//
// return $objects;
// }
//
// /**
// * Shorthand method.
// */
// private function addStep()
// {
// $this->repository->addStepsDone($this->job, 1);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function addTotalSteps(int $steps)
// {
// $this->repository->addTotalSteps($this->job, $steps);
// }
//
// /**
// *
// */
// private function createImportTag(): Tag
// {
// Log::debug('Now in createImportTag()');
//
// if ($this->journals->count() < 1) {
// Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
//
// return new Tag;
// }
// $this->addTotalSteps($this->journals->count() + 2);
//
// /** @var TagRepositoryInterface $repository */
// $repository = app(TagRepositoryInterface::class);
// $repository->setUser($this->job->user);
// $data = [
// 'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
// 'date' => new Carbon,
// 'description' => null,
// 'latitude' => null,
// 'longitude' => null,
// 'zoomLevel' => null,
// 'tagMode' => 'nothing',
// ];
// $tag = $repository->store($data);
// $this->addStep();
// $extended = $this->getExtendedStatus();
// $extended['tag'] = $tag->id;
// $this->setExtendedStatus($extended);
//
// Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
// Log::debug('Looping journals...');
// $journalIds = $this->journals->pluck('id')->toArray();
// $tagId = $tag->id;
// foreach ($journalIds as $journalId) {
// Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
// DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
// $this->addStep();
// }
// Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
// $this->addStep();
//
// return $tag;
// }
//
// /**
// * Shorthand method
// *
// * @return array
// */
// private function getConfig(): array
// {
// return $this->repository->getConfiguration($this->job);
// }
//
// /**
// * @return array
// */
// private function getExtendedStatus(): array
// {
// return $this->repository->getExtendedStatus($this->job);
// }
//
// /**
// * Shorthand method.
// *
// * @return string
// */
// private function getStatus(): string
// {
// return $this->repository->getStatus($this->job);
// }
//
// /**
// * @param array $extended
// */
// private function setExtendedStatus(array $extended): void
// {
// $this->repository->setExtendedStatus($this->job, $extended);
// }
//
// /**
// * Shorthand
// *
// * @param string $status
// */
// private function setStatus(string $status): void
// {
// $this->repository->setStatus($this->job, $status);
// }
//
// /**
// * Shorthand
// *
// * @param int $steps
// */
// private function setTotalSteps(int $steps)
// {
// $this->repository->setTotalSteps($this->job, $steps);
// }
//
// /**
// * @param Collection $objects
// *
// * @return ImportStorage
// */
// private function storeObjects(Collection $objects): ImportStorage
// {
// $config = $this->getConfig();
// $storage = new ImportStorage;
// $storage->setJob($this->job);
// $storage->setDateFormat($config['date-format']);
// $storage->setObjects($objects);
// $storage->store();
// Log::info('Back in storeObjects()');
//
// return $storage;
// }
/**
* At the end of each run(), the import routine must set the job to the expected status.
*
* The final status of the routine must be "provider_finished".
*
* @return bool
* @throws FireflyException
*/
public function run(): bool
public function run(): void
{
if ('configured' !== $this->getStatus()) {
Log::error(sprintf('Job %s is in state "%s" so it cannot be started.', $this->job->key, $this->getStatus()));
return false;
}
set_time_limit(0);
Log::info(sprintf('Start with import job %s', $this->job->key));
// total steps: 6
$this->setTotalSteps(6);
$importObjects = $this->getImportObjects();
$this->lines = $importObjects->count();
$this->addStep();
// total steps can now be extended. File has been scanned. 7 steps per line:
$this->addTotalSteps(7 * $this->lines);
// once done, use storage thing to actually store them:
Log::info(sprintf('Returned %d valid objects from file processor', $this->lines));
$storage = $this->storeObjects($importObjects);
$this->addStep();
Log::debug('Back in run()');
Log::debug('Updated job...');
Log::debug(sprintf('%d journals in $storage->journals', $storage->journals->count()));
$this->journals = $storage->journals;
$this->errors = $storage->errors;
Log::debug('Going to call createImportTag()');
// create tag, link tag to all journals:
$this->createImportTag();
$this->addStep();
// update job:
$this->setStatus('finished');
Log::info(sprintf('Done with import job %s', $this->job->key));
return true;
// TODO: Implement run() method.
throw new NotImplementedException;
}
/**
* @param ImportJob $job
*
* @return mixed
*/
public function setJob(ImportJob $job)
{
$this->job = $job;
$this->repository = app(ImportJobRepositoryInterface::class);
$this->repository->setUser($job->user);
}
/**
* @return Collection
*/
protected function getImportObjects(): Collection
{
$objects = new Collection;
$fileType = $this->getConfig()['file-type'] ?? 'csv';
// will only respond to "file"
$class = config(sprintf('import.options.file.processors.%s', $fileType));
/** @var FileProcessorInterface $processor */
$processor = app($class);
$processor->setJob($this->job);
if ('configured' === $this->getStatus()) {
// set job as "running"...
$this->setStatus('running');
Log::debug('Job is configured, start with run()');
$processor->run();
$objects = $processor->getObjects();
}
return $objects;
}
/**
* Shorthand method.
*/
private function addStep()
{
$this->repository->addStepsDone($this->job, 1);
}
/**
* Shorthand
*
* @param int $steps
*/
private function addTotalSteps(int $steps)
{
$this->repository->addTotalSteps($this->job, $steps);
}
/**
*
*/
private function createImportTag(): Tag
{
Log::debug('Now in createImportTag()');
if ($this->journals->count() < 1) {
Log::info(sprintf('Will not create tag, %d journals imported.', $this->journals->count()));
return new Tag;
}
$this->addTotalSteps($this->journals->count() + 2);
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->job->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->job->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
$this->addStep();
$extended = $this->getExtendedStatus();
$extended['tag'] = $tag->id;
$this->setExtendedStatus($extended);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $this->journals->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
$this->addStep();
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $this->journals->count(), $tag->id, $tag->tag));
$this->addStep();
return $tag;
}
/**
* Shorthand method
*
* @return array
*/
private function getConfig(): array
{
return $this->repository->getConfiguration($this->job);
}
/**
* @return array
*/
private function getExtendedStatus(): array
{
return $this->repository->getExtendedStatus($this->job);
}
/**
* Shorthand method.
*
* @return string
*/
private function getStatus(): string
{
return $this->repository->getStatus($this->job);
}
/**
* @param array $extended
*/
private function setExtendedStatus(array $extended): void
{
$this->repository->setExtendedStatus($this->job, $extended);
}
/**
* Shorthand
*
* @param string $status
*/
private function setStatus(string $status): void
{
$this->repository->setStatus($this->job, $status);
}
/**
* Shorthand
*
* @param int $steps
*/
private function setTotalSteps(int $steps)
{
$this->repository->setTotalSteps($this->job, $steps);
}
/**
* @param Collection $objects
*
* @return ImportStorage
*/
private function storeObjects(Collection $objects): ImportStorage
{
$config = $this->getConfig();
$storage = new ImportStorage;
$storage->setJob($this->job);
$storage->setDateFormat($config['date-format']);
$storage->setObjects($objects);
$storage->store();
Log::info('Back in storeObjects()');
return $storage;
// TODO: Implement setJob() method.
throw new NotImplementedException;
}
}

1070
app/Import/Routine/SpectreRoutine.php
View File

File diff suppressed because it is too large Load Diff

268
app/Import/Storage/ImportArrayStorage.php
View File

@@ -3,21 +3,21 @@
namespace FireflyIII\Import\Storage;
use Carbon\Carbon;
use DB;
use FireflyIII\Exceptions\FireflyException;
use FireflyIII\Factory\TransactionJournalFactory;
use FireflyIII\Helpers\Collector\JournalCollectorInterface;
use FireflyIII\Helpers\Filter\InternalTransferFilter;
use FireflyIII\Models\ImportJob;
use FireflyIII\Models\Tag;
use FireflyIII\Models\Rule;
use FireflyIII\Models\Transaction;
use FireflyIII\Models\TransactionJournalMeta;
use FireflyIII\Models\TransactionType;
use FireflyIII\Repositories\ImportJob\ImportJobRepositoryInterface;
use FireflyIII\Repositories\Tag\TagRepositoryInterface;
use FireflyIII\TransactionRules\Processor;
use Illuminate\Support\Collection;
use Illuminate\Support\MessageBag;
use Log;
use DB;
/**
* Creates new transactions based upon arrays. Will first check the array for duplicates.
@@ -40,11 +40,9 @@ class ImportArrayStorage
private $transfers;
/**
* ImportArrayStorage constructor.
*
* @param ImportJob $importJob
*/
public function __construct(ImportJob $importJob)
public function setJob(ImportJob $importJob): void
{
$this->importJob = $importJob;
$this->countTransfers();
@@ -56,104 +54,61 @@ class ImportArrayStorage
}
/**
* Actually does the storing.
* Actually does the storing. Does three things.
* - Store journals
* - Link to tag
* - Run rules (if set to)
*
* @return Collection
* @throws FireflyException
*/
public function store(): Collection
{
$count = count($this->importJob->transactions);
Log::debug(sprintf('Now in store(). Count of items is %d', $count));
$toStore = [];
foreach ($this->importJob->transactions as $index => $transaction) {
Log::debug(sprintf('Now at item %d out of %d', ($index + 1), $count));
$existingId = $this->hashExists($transaction);
if (null !== $existingId) {
$this->logDuplicateObject($transaction, $existingId);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. It already exists.',
$index, $transaction['description']
)
);
continue;
}
if ($this->checkForTransfers) {
if ($this->transferExists($transaction)) {
$this->logDuplicateTransfer($transaction);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
$index,
$transaction['description']
)
);
continue;
}
}
$toStore[] = $transaction;
// store transactions
$this->setStatus('storing_data');
$collection = $this->storeArray();
$this->setStatus('stored_data');
// link tag:
$this->setStatus('linking_to_tag');
$this->linkToTag($collection);
$this->setStatus('linked_to_tag');
// run rules, if configured to.
$config = $this->importJob->configuration;
if (isset($config['apply-rules']) && $config['apply-rules'] === true) {
$this->setStatus('applying_rules');
$this->applyRules($collection);
$this->setStatus('rules_applied');
}
if (count($toStore) === 0) {
Log::info('No transactions to store left!');
return new Collection;
}
Log::debug('Going to store...');
// now actually store them:
$collection = new Collection;
/** @var TransactionJournalFactory $factory */
$factory = app(TransactionJournalFactory::class);
$factory->setUser($this->importJob->user);
foreach ($toStore as $store) {
// convert the date to an object:
$store['date'] = Carbon::createFromFormat('Y-m-d', $store['date']);
// store the journal.
$collection->push($factory->create($store));
}
Log::debug('DONE storing!');
// create tag and append journals:
$this->createTag($collection);
return $collection;
}
/**
* @param Collection $collection
*
* @throws FireflyException
*/
private function createTag(Collection $collection): void
private function applyRules(Collection $collection): void
{
$rules = $this->getRules();
if ($rules->count() > 0) {
foreach ($collection as $journal) {
$rules->each(
function (Rule $rule) use ($journal) {
Log::debug(sprintf('Going to apply rule #%d to journal %d.', $rule->id, $journal->id));
$processor = Processor::make($rule);
$processor->handleTransactionJournal($journal);
if ($rule->stop_processing) {
return false;
}
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->importJob->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $collection->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
return true;
}
);
}
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $collection->count(), $tag->id, $tag->tag));
$this->repository->setTag($this->importJob, $tag);
}
/**
@@ -161,8 +116,10 @@ class ImportArrayStorage
*/
private function countTransfers(): void
{
/** @var array $array */
$array = $this->importJob->transactions;
$count = 0;
foreach ($this->importJob->transactions as $transaction) {
foreach ($array as $transaction) {
if (strtolower(TransactionType::TRANSFER) === $transaction['type']) {
$count++;
}
@@ -177,6 +134,29 @@ class ImportArrayStorage
}
/**
* @return Collection
*/
private function getRules(): Collection
{
/** @var Collection $set */
$set = Rule::distinct()
->where('rules.user_id', $this->importJob->user_id)
->leftJoin('rule_groups', 'rule_groups.id', '=', 'rules.rule_group_id')
->leftJoin('rule_triggers', 'rules.id', '=', 'rule_triggers.rule_id')
->where('rule_groups.active', 1)
->where('rule_triggers.trigger_type', 'user_action')
->where('rule_triggers.trigger_value', 'store-journal')
->where('rules.active', 1)
->orderBy('rule_groups.order', 'ASC')
->orderBy('rules.order', 'ASC')
->get(['rules.*', 'rule_groups.order']);
Log::debug(sprintf('Found %d user rules.', $set->count()));
return $set;
}
/**
* Get the users transfers, so they can be compared to whatever the user is trying to import.
*/
@@ -192,7 +172,6 @@ class ImportArrayStorage
}
/**
* @param array $transaction
*
@@ -222,6 +201,39 @@ class ImportArrayStorage
return (int)$entry->transaction_journal_id;
}
/**
* @param Collection $collection
*/
private function linkToTag(Collection $collection): void
{
/** @var TagRepositoryInterface $repository */
$repository = app(TagRepositoryInterface::class);
$repository->setUser($this->importJob->user);
$data = [
'tag' => trans('import.import_with_key', ['key' => $this->importJob->key]),
'date' => new Carbon,
'description' => null,
'latitude' => null,
'longitude' => null,
'zoomLevel' => null,
'tagMode' => 'nothing',
];
$tag = $repository->store($data);
Log::debug(sprintf('Created tag #%d ("%s")', $tag->id, $tag->tag));
Log::debug('Looping journals...');
$journalIds = $collection->pluck('id')->toArray();
$tagId = $tag->id;
foreach ($journalIds as $journalId) {
Log::debug(sprintf('Linking journal #%d to tag #%d...', $journalId, $tagId));
DB::table('tag_transaction_journal')->insert(['transaction_journal_id' => $journalId, 'tag_id' => $tagId]);
}
Log::info(sprintf('Linked %d journals to tag #%d ("%s")', $collection->count(), $tag->id, $tag->tag));
$this->repository->setTag($this->importJob, $tag);
}
/**
* @param array $transaction
* @param int $existingId
@@ -255,6 +267,84 @@ class ImportArrayStorage
);
}
/**
* Shorthand method to quickly set job status
*
* @param string $status
*/
private function setStatus(string $status): void
{
$this->repository->setStatus($this->importJob, $status);
}
/**
* Store array as journals.
*
* @return Collection
* @throws FireflyException
*/
private function storeArray(): Collection
{
/** @var array $array */
$array = $this->importJob->transactions;
$count = \count($array);
$toStore = [];
Log::debug(sprintf('Now in store(). Count of items is %d', $count));
foreach ($array as $index => $transaction) {
Log::debug(sprintf('Now at item %d out of %d', $index + 1, $count));
$existingId = $this->hashExists($transaction);
if (null !== $existingId) {
$this->logDuplicateObject($transaction, $existingId);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. It already exists.',
$index, $transaction['description']
)
);
continue;
}
if ($this->checkForTransfers) {
if ($this->transferExists($transaction)) {
$this->logDuplicateTransfer($transaction);
$this->repository->addErrorMessage(
$this->importJob, sprintf(
'Entry #%d ("%s") could not be imported. Such a transfer already exists.',
$index,
$transaction['description']
)
);
continue;
}
}
$toStore[] = $transaction;
}
if (\count($toStore) === 0) {
Log::info('No transactions to store left!');
return new Collection;
}
Log::debug('Going to store...');
// now actually store them:
$collection = new Collection;
/** @var TransactionJournalFactory $factory */
$factory = app(TransactionJournalFactory::class);
$factory->setUser($this->importJob->user);
foreach ($toStore as $store) {
// convert the date to an object:
$store['date'] = Carbon::createFromFormat('Y-m-d', $store['date']);
// store the journal.
$collection->push($factory->create($store));
}
Log::debug('DONE storing!');
return $collection;
}
/**
* Check if a transfer exists.
*