daw_json_lines_iterator.h

Go to the documentation of this file.

// Copyright (c) Darrell Wright
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Official repository: https://github.com/beached/daw_json_link
//
 
#pragma once
 
#include "daw/json/impl/version.h"
 
#include "daw/json/impl/daw_json_link_types_fwd.h"
#include "daw/json/impl/daw_json_parse_class.h"
#include "daw/json/impl/daw_json_value.h"
 
#include <daw/daw_cxmath.h>
#include <daw/daw_move.h>
#include <daw/daw_scope_guard.h>
#include <daw/daw_traits.h>
#include <daw/daw_utility.h>
 
#include <array>
#include <cstddef>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <string>
#include <string_view>
#include <type_traits>
 
namespace daw::json {
  inline namespace DAW_JSON_VER {
    template<typename JsonElement = json_value, auto... PolicyFlags>
    class json_lines_iterator {
      using ParseState = TryDefaultParsePolicy<BasicParsePolicy<
        options::details::make_parse_flags<PolicyFlags...>( ).value>>;
 
    public:
      using element_type = json_details::json_deduced_type<JsonElement>;
      static_assert( not std::is_same_v<element_type, void>,
                     "Unknown JsonElement type." );
      using value_type = json_details::json_result_t<element_type>;
      using reference = value_type;
      using pointer = json_details::arrow_proxy<value_type>;
      using difference_type = std::ptrdiff_t;
      // Can do forward iteration and be stored
      using iterator_category = std::input_iterator_tag;
 
    private:
      ParseState m_state = ParseState( );
      /***
       * This lets us fastpath and just skip n characters as we have already
       * parsed them
       */
      mutable char const *m_can_skip = nullptr;
 
    public:
      explicit json_lines_iterator( ) = default;
 
      explicit constexpr json_lines_iterator( daw::string_view json_lines_doc )
        : m_state( ParseState( std::data( json_lines_doc ),
                               daw::data_end( json_lines_doc ) ) ) {
 
        m_state.trim_left( );
      }
 
      [[nodiscard]] constexpr value_type operator*( ) const {
        daw_json_assert_weak(
          m_state.has_more( ), ErrorReason::UnexpectedEndOfData, m_state );
 
        auto tmp = m_state;
        auto const run_after_parse = daw::on_scope_exit( [&] {
          m_can_skip = tmp.first;
        } );
        (void)run_after_parse;
        return json_details::
          parse_value<element_type, false, element_type::expected_type>( tmp );
      }
 
      [[nodiscard]] constexpr pointer operator->( ) const {
        return pointer{ operator*( ) };
      }
 
      constexpr json_lines_iterator &operator++( ) {
        daw_json_assert_weak(
          m_state.has_more( ), ErrorReason::UnexpectedEndOfData, m_state );
        if( m_can_skip ) {
          m_state.first = m_can_skip;
          m_can_skip = nullptr;
        } else {
          (void)json_details::skip_known_value<element_type>( m_state );
        }
        m_state.move_next_member_or_end( );
        return *this;
      }
 
      constexpr void operator++( int ) & {
        (void)operator++( );
      }
 
      [[nodiscard]] constexpr bool good( ) const {
        return not m_state.is_null( ) and m_state.has_more( );
      }
 
      [[nodiscard]] explicit constexpr operator bool( ) const {
        return good( );
      }
 
      [[nodiscard]] constexpr bool
      operator==( json_lines_iterator const &rhs ) const {
        if( not( *this ) ) {
          return not rhs;
        }
        if( not rhs ) {
          return false;
        }
        return ( m_state.first == rhs.m_state.first );
      }
 
      [[nodiscard]] constexpr bool
      operator!=( json_lines_iterator const &rhs ) const {
        if( not( *this ) ) {
          return static_cast<bool>( rhs );
        }
        if( not rhs ) {
          return true;
        }
        return m_state.first != rhs.m_state.first;
      }
 
      constexpr std::string_view get_raw_json_document( ) const {
        return std::string_view( m_state.first, m_state.size( ) );
      }
    };
    json_lines_iterator( daw::string_view ) -> json_lines_iterator<>;
 
    template<typename JsonElement = json_value, auto... PolicyFlags>
    struct json_lines_range {
      using ParsePolicy = TryDefaultParsePolicy<BasicParsePolicy<
        options::details::make_parse_flags<PolicyFlags...>( ).value>>;
      using iterator = json_lines_iterator<JsonElement, PolicyFlags...>;
 
    private:
      iterator m_first{ };
      iterator m_last{ };
 
    public:
      explicit json_lines_range( ) = default;
 
      explicit constexpr json_lines_range( iterator first,
                                           iterator last = iterator( ) )
        : m_first( first )
        , m_last( last ) {}
 
      explicit constexpr json_lines_range( daw::string_view json_lines_doc )
        : m_first( json_lines_doc ) {}
 
      [[nodiscard]] constexpr iterator begin( ) const {
        return m_first;
      }
 
      [[nodiscard]] constexpr iterator end( ) const {
        return m_last;
      }
 
      [[nodiscard]] constexpr bool empty( ) const {
        return m_first == m_last;
      }
    };
    json_lines_range( daw::string_view ) -> json_lines_range<>;
 
    template<typename JsonElement, auto... PolicyFlags>
    json_lines_range( json_lines_iterator<JsonElement, PolicyFlags...>,
                      json_lines_iterator<JsonElement, PolicyFlags...> )
      -> json_lines_range<JsonElement, PolicyFlags...>;
 
    template<typename JsonElement = json_value, auto... ParsePolicies>
    std::vector<json_lines_range<JsonElement, ParsePolicies...>>
    partition_jsonl_document( std::size_t num_partitions,
                              daw::string_view jsonl_doc ) {
      using result_t =
        std::vector<json_lines_range<JsonElement, ParsePolicies...>>;
      if( num_partitions <= 1 ) {
        return result_t{
          json_lines_range<JsonElement, ParsePolicies...>( jsonl_doc ) };
      }
      auto approx_segsize = jsonl_doc.size( ) / num_partitions;
      auto result = result_t{ };
      char const *const last = daw::data_end( jsonl_doc );
      while( not jsonl_doc.empty( ) ) {
        char const *tmp = std::data( jsonl_doc ) + approx_segsize;
        if( tmp >= last ) {
          result.emplace_back( jsonl_doc );
          break;
        }
        while( tmp < last and *tmp != '\n' ) {
          ++tmp;
        }
        if( tmp < last ) {
          ++tmp;
        }
        auto sz = static_cast<std::size_t>( tmp - std::data( jsonl_doc ) );
        auto doc = jsonl_doc.pop_front( sz );
        doc.trim_suffix( );
        if( not doc.empty( ) ) {
          result.emplace_back( doc );
        }
      }
      return result;
    }
  } // namespace DAW_JSON_VER
} // namespace daw::json