Skip to content

Instantly share code, notes, and snippets.

@TheDeadCode

TheDeadCode/JsonModelStreamWriter.php

Created March 3, 2021 22:04
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save TheDeadCode/0bd4dbe9a2ff744b5a86663523292ce4 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save TheDeadCode/0bd4dbe9a2ff744b5a86663523292ce4 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. TheDeadCode created this gist Mar 3, 2021.
    118 changes: 118 additions & 0 deletions JsonModelStreamWriter.php
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,118 @@
    <?php

    /**
    * Stream-write arrays onto a file on disk as JSON format, to avoid memory leaks.
    * Written expressly for /u/devourment77 of reddit.
    *
    * Copyright (C) 2021 by Vynatu Cyberlabs, Inc. and Felix Lebel
    *
    * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
    * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
    * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
    * permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright
    * notice and this permission notice shall be included in all copies or substantial portions of the Software. THE
    * SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
    * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
    * OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
    * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    *
    * @author Felix Lebel <[email protected]>
    * @copyright 2019-2021 Vynatu Cyberlabs, Inc.
    * @license https://opensource.org/licenses/MIT MIT License
    */

    namespace App\Services;

    class JsonModelStreamWriter
    {
    const ARR_OPEN = "[";
    const ARR_CLOSE = "]";
    const OBJ_OPEN = "{";
    const OBJ_CLOSE = "}";
    const COLON = ":";
    const COMMA = ",";

    /**
    * The open file resource.
    *
    * @var resource
    */
    private $resource;

    /**
    * Whether the resource was closed. Prevents redundant is_resource() calls.
    *
    * @var bool
    */
    private $is_closed;

    private $wrote_at_least_once = false;

    public function __construct(string $filename)
    {
    $this->resource = fopen($filename, 'w+');
    fwrite($this->resource, self::ARR_OPEN);
    $this->is_closed = false;
    }

    /**
    * Write an array to the disk without buffering.
    *
    * @param array $array The key => value array to stream-write to the disk
    */
    public function writeArray(array $array)
    {
    if ($this->is_closed) {
    return;
    }

    $temp_string = "";

    if ($this->wrote_at_least_once) {
    $temp_string .= self::COMMA;
    }

    $temp_string .= self::OBJ_OPEN;

    $wrote_first_key = false;

    foreach ($array as $key => $value) {
    $json_key = json_encode((string)$key); // JSON Keys can only be strings.
    $json_value = $this->getValidJsonValue($value);

    if ($json_value === null) {
    continue;
    }

    if ($wrote_first_key) {
    $temp_string .= self::COMMA;
    }

    $temp_string .= $json_key . self::COLON . $json_value;

    $wrote_first_key = true;
    }

    if ($wrote_first_key) {
    $temp_string .= self::OBJ_CLOSE;
    fwrite($this->resource, $temp_string);
    $this->wrote_at_least_once = true;
    }
    }

    public function close()
    {
    fwrite($this->resource, self::ARR_CLOSE);
    fclose($this->resource);
    $this->is_closed = true;
    }

    private function getValidJsonValue($value)
    {
    if (is_object($value) || is_array($value)) {
    return null;
    }

    return json_encode($value);
    }
    }
    13 changes: 13 additions & 0 deletions Usage.php
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,13 @@
    <?php

    use App\Services\JsonModelStreamWriter; // Change the namespace of file above to your liking.

    $models = User::all(); // Or chunk

    $writer = new JsonModelStreamWriter(storage_path("some_file.json"));

    foreach ($models as $model) {
    $writer->writeArray($model->toArray());
    }

    $writer->close();