Laravel-excel: [QUESTION] how to catch failures and errors on ShouldQueue imports

[Naoray]

Prerequisites

• [x] Have read and understood: https://github.com/Maatwebsite/Laravel-Excel/blob/3.1/.github/SUPPORT.md
• [x] Checked if your Laravel Excel version is still supported: https://docs.laravel-excel.com/3.1/getting-started/support.html#supported-versions
• [x] Checked that your question isn't already asked before.
• [x] Filled in the entire issue template

Versions

• PHP version: 7.2.19
• Laravel version: 5.8
• Package version: 3.1.15

Description

I've read about row validation and was wondering whether this is also possible for imports implementing ShouldQueue?

Additional Information

// My import code

$importer->import($request->file('file'));

        \Log::debug('failure');
        \Log::debug($importer->failures());
        \Log::debug('errors');
        \Log::debug($importer->errors());

Obviously the code is executed synchronouly and there are no errors nor failures on the importer class when the debug statement are executed...

class CourseContentImport implements ToCollection, HeadingRow, WithBatchInserts, WithChunkReading, Formulas, ShouldQueue, SkipsOnFailure, SkipsOnError
{
    use Importable, SkipsFailures, SkipsErrors;
    //...
}

[patrickbrouwers]

The process is async, so you cannot catch the failures in a sync way. You can skip the failures/errors, but you won't be able to catch them in the way you show in your example. You probably want to use the failed hook to deal with the validation errors.

[patrickbrouwers]

Btw can you try to add a title to the issue, it's difficult to search on it now without a title.

[Naoray]

@patrickbrouwers sorry, didn't notice I didn't add a title

[Naoray]

Ah ofc... onFailure/onError is obviously the solution

[hotmeteor]

@Naoray Would love to see your solution for this, because I can't get it to work at all.

[Naoray]

@hotmeteor I can't remember why I did it the way I did it, but basically it's just using the SkipsErrors/SkipsFailures and checking after each import process is finished if some failures/errors occured (s. https://docs.laravel-excel.com/3.1/imports/validation.html).

I took it a bit further and created a dedicated class to import ToCollection imports and send a FailureNotification with all failures that occured during the import:

// ToCollectionImport
use Throwable;
use Illuminate\Support\Str;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Validator;
use Maatwebsite\Excel\Validators\Failure;
use Illuminate\Support\Facades\Notification;
use Maatwebsite\Excel\Concerns\SkipsOnError;
use Maatwebsite\Excel\Concerns\ToCollection;
use Illuminate\Validation\ValidationException;
use Maatwebsite\Excel\Concerns\SkipsOnFailure;
use Maatwebsite\Excel\Concerns\WithValidation;
use ...\Exports\FailureExport;
use ...\Contracts\NotifiesUserOnFailure;
use ...\Notifications\FailureNotification;

abstract class ToCollectionImport implements ToCollection
{
    /**
     * Set additional user resolve callback.
     *
     * @var \Illuminate\Support\Collection
     */
    protected static $additionalUsersCallback;

    abstract public function processImport(Collection $rows);

    abstract public function rules(): array;

    abstract public function getUser();

    /**
     * @param Collection $rows
     */
    public function collection(Collection $rows)
    {
        if ($this instanceof WithValidation) {
            $rows = $this->validate($rows);
        }

        try {
            $this->processImport($rows);
        } catch (Throwable $e) {
            $this->recordOrThrowErrors($e);
        }

        if ($this->failures()->count() > 0) {
            $name = Str::random(16) . now()->format('Y-M-d');
            $path = config('nova-import-card.import_failures_path') . $name . '.xlsx';

            (new FailureExport($this->failures()))->store($path);

            if ($this instanceof NotifiesUserOnFailure) {
                $this->notifyUsers($name);
            }
        }

        if ($this->errors()->count() > 0) {
            \Log::error($this->errors());
        }
    }

    /**
     * Validate given collection data.
     *
     * @param Collection $rows
     *
     * @throws ValidationException
     *
     * @return Collection
     */
    protected function validate(Collection $rows)
    {
        $validator = Validator::make($rows->toArray(), $this->rules());

        if (! $validator->fails()) {
            return $rows;
        }

        if ($this instanceof SkipsOnFailure) {
            $this->onFailure(
                ...$this->collectErrors($validator, $rows)
            );

            $keysCausingFailure = collect($validator->errors()->keys())->map(function ($key) {
                return Str::before($key, '.');
            })
            ->values()
            ->toArray();

            return $rows->except($keysCausingFailure);
        }

        throw new ValidationException($validator);
    }

    /**
     * Get all validation errors.
     *
     * @param $validator
     * @param Collection $rows
     *
     * @return array
     */
    protected function collectErrors($validator, Collection $rows)
    {
        $failures = [];

        foreach ($validator->errors()->messages() as $attribute => $messages) {
            $row = strtok($attribute, '.');

            $attributeName = strtok('');
            $attributeName = $attributeName;

            $failures[] = new Failure(
                $row,
                $attributeName,
                str_replace($attribute, $attributeName, $messages),
                $rows->toArray()[$row]
            );
        }

        return $failures;
    }

    /**
     * Records an error or throws its exception.
     *
     * @param Throwable $error
     *
     * @throws \Exception
     */
    protected function recordOrThrowErrors(Throwable $error)
    {
        if ($this instanceof SkipsOnError) {
            return $this->onError($error);
        }

        throw $error;
    }

    /**
     * Notify all users.
     *
     * @param string $name
     */
    public function notifyUsers($name)
    {
        $users = ! static::$additionalUsersCallback
            ? collect($this->getUser())
            : call_user_func(static::$additionalUsersCallback, $this->getUser());

        Notification::send($users, new FailureNotification($name));
    }

    /**
     * Set additional users callback.
     *
     * @param \Callable $callback
     *
     * @return self
     */
    public static function additionalUsers($callback)
    {
        static::$additionalUsersCallback = $callback;
    }
}


// the concrete usage of the above

use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\Importable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Maatwebsite\Excel\Concerns\SkipsErrors;
use Maatwebsite\Excel\Concerns\SkipsOnError;
use Maatwebsite\Excel\Concerns\SkipsFailures;
use ...\ToCollectionImport;
use Maatwebsite\Excel\Concerns\SkipsOnFailure;
use Maatwebsite\Excel\Concerns\WithValidation;
use ...\Concerns\NotifiesUser;
use ...\Contracts\NotifiesUserOnFailure;

class CourseContentImport extends ToCollectionImport implements
    ShouldQueue,
    SkipsOnError,
    SkipsOnFailure,
    WithValidation,
    NotifiesUserOnFailure
{
    use Importable, SkipsErrors, SkipsFailures, NotifiesUser;

    /**
     * @param Collection $rows
     */
    public function processImport(Collection $rows)
    {
        foreach ($rows as $row) {
            // get information from row and create corresponding models
        }
    }

    public function rules(): array
    {
        return [..];
    }
}

// FailureExport
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\Exportable;
use Maatwebsite\Excel\Concerns\WithHeadings;
use Maatwebsite\Excel\Concerns\FromCollection;
use Maatwebsite\Excel\Concerns\ShouldAutoSize;

class FailureExport implements FromCollection, WithHeadings, ShouldAutoSize
{
    use Exportable;

    protected $failures;

    public function __construct(Collection $failures)
    {
        $this->failures = $failures->map(function ($failure) {
            return array_merge(
                ['error' => implode(',', $failure->errors())],
                $failure->values()
            );
        });
    }

    /**
     * @return array
     */
    public function headings(): array
    {
        return array_keys($this->failures->first());
    }

    public function collection()
    {
        return $this->failures;
    }
}

If you have additional question feel free to ask!